@markitdownjs/chunking 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/classifier.d.ts.map +1 -1
- package/dist/classifier.js.map +1 -1
- package/dist/fingerprint.d.ts.map +1 -1
- package/dist/fingerprint.js +3 -5
- package/dist/fingerprint.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/strategies/heading-strategy.d.ts.map +1 -1
- package/dist/strategies/heading-strategy.js +4 -1
- package/dist/strategies/heading-strategy.js.map +1 -1
- package/dist/tokenizers/tiktoken.d.ts.map +1 -1
- package/dist/tokenizers/tiktoken.js +1 -6
- package/dist/tokenizers/tiktoken.js.map +1 -1
- package/package.json +14 -2
package/dist/classifier.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../src/classifier.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEnE,6CAA6C;AAC7C,MAAM,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../src/classifier.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEnE,6CAA6C;AAC7C,MAAM,MAAM,WAAW,GACnB,WAAW,GACX,OAAO,GACP,MAAM,GACN,MAAM,GACN,SAAS,GACT,cAAc,GACd,OAAO,CAAC;AAEZ;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,aAAa,GAAG,WAAW,CAuB1E;AAqCD;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,aAAa,EAAE,CAWvE"}
|
package/dist/classifier.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"classifier.js","sourceRoot":"","sources":["../src/classifier.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"classifier.js","sourceRoot":"","sources":["../src/classifier.ts"],"names":[],"mappings":"AAYA;;;;;;;;;;GAUG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAoB;IAC3D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAe,CAAC;IAErC,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAEnC,uEAAuE;IACvE,2CAA2C;IAC3C,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACnB,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC5B,CAAC;IAED,oCAAoC;IACpC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC;IAC/B,CAAC;IAED,4BAA4B;IAC5B,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,2CAA2C;IAC3C,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAa,EAAE,KAAuB;IAC9D,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,OAAO;YACV,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,MAAM;QACR,KAAK,MAAM;YACT,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClB,MAAM;QACR,KAAK,MAAM,CAAC;QACZ,KAAK,WAAW;YACd,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClB,MAAM;QACR,KAAK,SAAS;YACZ,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACrB,MAAM;QACR,KAAK,SAAS;YACZ,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC1B,MAAM;QACR,KAAK,WAAW,CAAC;QACjB,KAAK,MAAM;YACT,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvB,MAAM;IACV,CAAC;IAED,yBAAyB;IACzB,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAAuB;IACpD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,MAAM,WAAW,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QACpD,OAAO;YACL,GAAG,KAAK;YACR,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,WAAW;aACZ;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEnE,oCAAoC;AACpC,MAAM,WAAW,gBAAgB;IAC/B,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,CAAC;CACvB;AAcD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEnE,oCAAoC;AACpC,MAAM,WAAW,gBAAgB;IAC/B,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,CAAC;CACvB;AAcD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGvD;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAI1D;AAeD;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,aAAa,EAAE,GACtB,CAAC,aAAa,GAAG;IAAE,WAAW,EAAE,gBAAgB,CAAA;CAAE,CAAC,EAAE,CAQvD;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE;IAAE,WAAW,EAAE,gBAAgB,CAAA;CAAE,EAAE,EAC9C,SAAS,EAAE;IAAE,WAAW,EAAE,gBAAgB,CAAA;CAAE,EAAE,GAC7C,MAAM,EAAE,CAmBV"}
|
package/dist/fingerprint.js
CHANGED
|
@@ -14,10 +14,7 @@ function djb2Hash(str) {
|
|
|
14
14
|
* Normalizes the content by lowercasing and collapsing whitespace before hashing.
|
|
15
15
|
*/
|
|
16
16
|
export function computeContentHash(text) {
|
|
17
|
-
const normalized = text
|
|
18
|
-
.toLowerCase()
|
|
19
|
-
.replace(/\s+/g, " ")
|
|
20
|
-
.trim();
|
|
17
|
+
const normalized = text.toLowerCase().replace(/\s+/g, " ").trim();
|
|
21
18
|
return `djb2:${djb2Hash(normalized)}`;
|
|
22
19
|
}
|
|
23
20
|
/**
|
|
@@ -67,7 +64,8 @@ export function detectChangedChunks(oldChunks, newChunks) {
|
|
|
67
64
|
if (!oldFp || !newFp) {
|
|
68
65
|
changed.push(i);
|
|
69
66
|
}
|
|
70
|
-
else if (oldFp.contentHash !== newFp.contentHash ||
|
|
67
|
+
else if (oldFp.contentHash !== newFp.contentHash ||
|
|
68
|
+
oldFp.structureHash !== newFp.structureHash) {
|
|
71
69
|
changed.push(i);
|
|
72
70
|
}
|
|
73
71
|
}
|
package/dist/fingerprint.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAUA;;;GAGG;AACH,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,UAAU,GAAG,IAAI
|
|
1
|
+
{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAUA;;;GAGG;AACH,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,OAAO,QAAQ,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAa;IAChD,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,OAAO,QAAQ,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;AACxC,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAa;IACrC,MAAM,KAAK,GAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAuB;IAEvB,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,GAAG,KAAK;QACR,WAAW,EAAE;YACX,WAAW,EAAE,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC;YAC9C,aAAa,EAAE,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC;SAC/C;KACF,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,SAA8C,EAC9C,SAA8C;IAE9C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAE5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC;QACxC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC;QAExC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;aAAM,IACL,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW;YACvC,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa,EAC3C,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ export { SemanticChunkingStrategy } from "./strategies/semantic-strategy.js";
|
|
|
6
6
|
export { FixedChunkingStrategy } from "./strategies/fixed-strategy.js";
|
|
7
7
|
export { classifyChunkContentType, classifyChunks } from "./classifier.js";
|
|
8
8
|
export type { ContentType } from "./classifier.js";
|
|
9
|
-
export { computeContentHash, computeStructureHash, fingerprintChunks, detectChangedChunks } from "./fingerprint.js";
|
|
9
|
+
export { computeContentHash, computeStructureHash, fingerprintChunks, detectChangedChunks, } from "./fingerprint.js";
|
|
10
10
|
export type { ChunkFingerprint } from "./fingerprint.js";
|
|
11
11
|
export type { Tokenizer } from "./tokenizers/types.js";
|
|
12
12
|
export { FastBPETokenizer } from "./tokenizers/fast-bpe.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAGvE,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC3E,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGnD,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAGvE,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC3E,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGnD,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,YAAY,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzF,OAAO,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -6,7 +6,7 @@ export { FixedChunkingStrategy } from "./strategies/fixed-strategy.js";
|
|
|
6
6
|
// Content type classification
|
|
7
7
|
export { classifyChunkContentType, classifyChunks } from "./classifier.js";
|
|
8
8
|
// Deduplication fingerprinting
|
|
9
|
-
export { computeContentHash, computeStructureHash, fingerprintChunks, detectChangedChunks } from "./fingerprint.js";
|
|
9
|
+
export { computeContentHash, computeStructureHash, fingerprintChunks, detectChangedChunks, } from "./fingerprint.js";
|
|
10
10
|
export { FastBPETokenizer } from "./tokenizers/fast-bpe.js";
|
|
11
11
|
export { TikTokenizer, Cl100kTokenizer, O200kTokenizer } from "./tokenizers/tiktoken.js";
|
|
12
12
|
export { TokenizerRegistry, getDefaultTokenizerRegistry } from "./tokenizers/registry.js";
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAEvE,8BAA8B;AAC9B,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAG3E,+BAA+B;AAC/B,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAEvE,8BAA8B;AAC9B,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAG3E,+BAA+B;AAC/B,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,kBAAkB,CAAC;AAK1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzF,OAAO,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"heading-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAe,MAAM,sBAAsB,CAAC;AAEjG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAWtD,8CAA8C;AAC9C,qBAAa,uBAAwB,YAAW,gBAAgB;IAC9D,IAAI,SAAa;IAEjB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;
|
|
1
|
+
{"version":3,"file":"heading-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAe,MAAM,sBAAsB,CAAC;AAEjG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAWtD,8CAA8C;AAC9C,qBAAa,uBAAwB,YAAW,gBAAgB;IAC9D,IAAI,SAAa;IAEjB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA2F/D"}
|
|
@@ -27,7 +27,10 @@ export class HeadingChunkingStrategy {
|
|
|
27
27
|
// Prepend overlap from previous chunk if configured.
|
|
28
28
|
let nodesToChunk = [...currentNodes];
|
|
29
29
|
if (overlapText && overlapTokens > 0) {
|
|
30
|
-
const overlapNode = {
|
|
30
|
+
const overlapNode = {
|
|
31
|
+
type: "paragraph",
|
|
32
|
+
children: [{ type: "text", value: overlapText }],
|
|
33
|
+
};
|
|
31
34
|
nodesToChunk = [overlapNode, ...nodesToChunk];
|
|
32
35
|
}
|
|
33
36
|
const chunk = createChunk(nodesToChunk, [...headingPath], options, startIndex, globalOffset);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"heading-strategy.js","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,0EAA0E;AAC1E,SAAS,kBAAkB,CAAC,OAAwB;IAClD,IAAI,OAAO,CAAC,WAAW;QAAE,OAAO,OAAO,CAAC,WAAW,CAAC;IACpD,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;AACpE,CAAC;AAED,8CAA8C;AAC9C,MAAM,OAAO,uBAAuB;IAClC,IAAI,GAAG,SAAS,CAAC;IAEjB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,MAAM,UAAU,GAAG,GAAS,EAAE;YAC5B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAEtC,qDAAqD;YACrD,IAAI,YAAY,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;YACrC,IAAI,WAAW,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,WAAW,GAAY,
|
|
1
|
+
{"version":3,"file":"heading-strategy.js","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,0EAA0E;AAC1E,SAAS,kBAAkB,CAAC,OAAwB;IAClD,IAAI,OAAO,CAAC,WAAW;QAAE,OAAO,OAAO,CAAC,WAAW,CAAC;IACpD,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;AACpE,CAAC;AAED,8CAA8C;AAC9C,MAAM,OAAO,uBAAuB;IAClC,IAAI,GAAG,SAAS,CAAC;IAEjB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,MAAM,UAAU,GAAG,GAAS,EAAE;YAC5B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAEtC,qDAAqD;YACrD,IAAI,YAAY,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;YACrC,IAAI,WAAW,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,WAAW,GAAY;oBAC3B,IAAI,EAAE,WAAW;oBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;iBACjD,CAAC;gBACF,YAAY,GAAG,CAAC,WAAW,EAAE,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;YAED,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,EAAE,CAAC,GAAG,WAAW,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;YAC7F,IAAI,KAAK,EAAE,CAAC;gBACV,wBAAwB;gBACxB,IAAI,WAAW,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;oBACpC,KAAK,CAAC,QAAgB,CAAC,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;gBAC1D,CAAC;gBAED,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;oBAC1C,MAAM,KAAK,GAAG,eAAe,CAC3B,YAAY,EACZ,WAAW,EACX,OAAO,EACP,SAAS,EACT,UAAU,EACV,WAAW,CACZ,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;gBAED,+CAA+C;gBAC/C,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;oBACtB,WAAW,GAAG,mBAAmB,CAAC,YAAY,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;gBAC9E,CAAC;YACH,CAAC;YAED,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAEhB,UAAU,EAAE,CAAC;QAEb,OAAO,MAAM,CAAC;QAEd,SAAS,IAAI,CAAC,IAAa,EAAE,MAAsB;YACjD,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;gBAC/C,UAAU,EAAE,CAAC;gBACb,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YAClD,IACE,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,CAAC,CAAC,MAAM,IAAI,YAAY,CAAc,MAAM,EAAE,SAAS,CAAC,CAAC,EACzD,CAAC;gBACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,YAAY,IAAI,UAAU,CAAC;YAC7B,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,EAAE,WAAW,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC/E,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,KAAgB,EAChB,aAAqB,EACrB,WAAqC;IAErC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,8EAA8E;IAC9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,WAAW,GAAG,aAAa,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,MAAM,GAAG,WAAW,EAAE,CAAC;YACzB,sCAAsC;YACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,WAAW,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;gBACzB,WAAW,EAAE,CAAC;YAChB,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/B,MAAM;QACR,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpB,WAAW,IAAI,MAAM,CAAC;IACxB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAgB,EAAE,WAAqC;IAC/E,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB,EAClB,WAAqC;IAErC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;QAClD,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAE3D,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;YACvF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,IAAI,UAAU,GAAG,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YAC9E,MAAM,SAAS,GAAG,cAAc,CAC9B,IAAe,EACf,WAAW,EACX,OAAO,EACP,SAAS,EACT,MAAM,EACN,WAAW,CACZ,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YAC1B,MAAM,IAAI,UAAU,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;QACvF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CACrB,IAAa,EACb,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB,EAClB,WAAqC;IAErC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAG,UAAU,CAAC;IACxB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,cAAc,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,YAAY,GAAG,cAAc,GAAG,SAAS,IAAI,MAAM,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;gBACvB,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE;oBACR,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;oBAC5B,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;oBAC7B,UAAU,EAAE,YAAY;oBACxB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,UAAU,EAAE,MAAM;oBAClB,QAAQ,EAAE,MAAM,GAAG,YAAY;iBAChC;gBACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE;aACxE,CAAC,CAAC;YACH,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;YACZ,YAAY,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;QACrD,YAAY,IAAI,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,OAAO,EAAE,MAAM;YACf,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;gBAC5B,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;gBAC7B,UAAU,EAAE,YAAY;gBACxB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,MAAM,GAAG,YAAY;aAChC;YACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE;SACxE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tiktoken.d.ts","sourceRoot":"","sources":["../../src/tokenizers/tiktoken.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;;;GAIG;AACH,qBAAa,YAAa,YAAW,SAAS;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;gBAEd,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;IAKjE,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAK9B,OAAO,CAAC,kBAAkB;IAgB1B,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM;IAIjC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAKjC,8CAA8C;IAC9C,OAAO,CAAC,gBAAgB;CAgBzB;AAED,8CAA8C;AAC9C,qBAAa,eAAgB,SAAQ,YAAY;;CAYhD;AAED,qDAAqD;AACrD,qBAAa,cAAe,SAAQ,YAAY;;
|
|
1
|
+
{"version":3,"file":"tiktoken.d.ts","sourceRoot":"","sources":["../../src/tokenizers/tiktoken.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;;;GAIG;AACH,qBAAa,YAAa,YAAW,SAAS;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;gBAEd,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;IAKjE,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAK9B,OAAO,CAAC,kBAAkB;IAgB1B,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM;IAIjC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAKjC,8CAA8C;IAC9C,OAAO,CAAC,gBAAgB;CAgBzB;AAED,8CAA8C;AAC9C,qBAAa,eAAgB,SAAQ,YAAY;;CAYhD;AAED,qDAAqD;AACrD,qBAAa,cAAe,SAAQ,YAAY;;CAI/C"}
|
|
@@ -76,12 +76,7 @@ export class Cl100kTokenizer extends TikTokenizer {
|
|
|
76
76
|
/** GPT-4o, GPT-4o-mini tokenizer (newer encoding) */
|
|
77
77
|
export class O200kTokenizer extends TikTokenizer {
|
|
78
78
|
constructor() {
|
|
79
|
-
super("o200k_base", "o200k_base", [
|
|
80
|
-
"gpt-4o",
|
|
81
|
-
"gpt-4o-mini",
|
|
82
|
-
"gpt-4.1",
|
|
83
|
-
"gpt-4.1-mini",
|
|
84
|
-
]);
|
|
79
|
+
super("o200k_base", "o200k_base", ["gpt-4o", "gpt-4o-mini", "gpt-4.1", "gpt-4.1-mini"]);
|
|
85
80
|
}
|
|
86
81
|
}
|
|
87
82
|
//# sourceMappingURL=tiktoken.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tiktoken.js","sourceRoot":"","sources":["../../src/tokenizers/tiktoken.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,OAAO,YAAY;IACd,IAAI,CAAS;IACb,MAAM,CAAW;IAE1B,YAAY,IAAY,EAAE,aAAqB,EAAE,MAAgB;QAC/D,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,MAAM,CAAC,IAAY;QACjB,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAEO,kBAAkB,CAAC,IAAY;QACrC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,sBAAsB,CAAC;QACrC,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACtF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,OAAiB;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,WAAW,CAAC,IAAY;QACtB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,8CAA8C;IACtC,gBAAgB,CAAC,IAAY;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,sBAAsB,CAAC;QACrC,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3D,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;CACF;AAED,8CAA8C;AAC9C,MAAM,OAAO,eAAgB,SAAQ,YAAY;IAC/C;QACE,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE;YAClC,OAAO;YACP,aAAa;YACb,QAAQ;YACR,aAAa;YACb,wBAAwB;YACxB,wBAAwB;YACxB,wBAAwB;SACzB,CAAC,CAAC;IACL,CAAC;CACF;AAED,qDAAqD;AACrD,MAAM,OAAO,cAAe,SAAQ,YAAY;IAC9C;QACE,KAAK,CAAC,YAAY,EAAE,YAAY,EAAE
|
|
1
|
+
{"version":3,"file":"tiktoken.js","sourceRoot":"","sources":["../../src/tokenizers/tiktoken.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,OAAO,YAAY;IACd,IAAI,CAAS;IACb,MAAM,CAAW;IAE1B,YAAY,IAAY,EAAE,aAAqB,EAAE,MAAgB;QAC/D,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,MAAM,CAAC,IAAY;QACjB,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAEO,kBAAkB,CAAC,IAAY;QACrC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,sBAAsB,CAAC;QACrC,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACtF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,OAAiB;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,WAAW,CAAC,IAAY;QACtB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,8CAA8C;IACtC,gBAAgB,CAAC,IAAY;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,sBAAsB,CAAC;QACrC,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3D,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;CACF;AAED,8CAA8C;AAC9C,MAAM,OAAO,eAAgB,SAAQ,YAAY;IAC/C;QACE,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE;YAClC,OAAO;YACP,aAAa;YACb,QAAQ;YACR,aAAa;YACb,wBAAwB;YACxB,wBAAwB;YACxB,wBAAwB;SACzB,CAAC,CAAC;IACL,CAAC;CACF;AAED,qDAAqD;AACrD,MAAM,OAAO,cAAe,SAAQ,YAAY;IAC9C;QACE,KAAK,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC;IAC1F,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@markitdownjs/chunking",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "AI-ready document chunking strategies for MarkItDownJS",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -17,11 +17,23 @@
|
|
|
17
17
|
"dist"
|
|
18
18
|
],
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@markitdownjs/shared": "0.
|
|
20
|
+
"@markitdownjs/shared": "0.4.0"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|
|
23
23
|
"typescript": "^5.5.0"
|
|
24
24
|
},
|
|
25
|
+
"keywords": [
|
|
26
|
+
"chunking",
|
|
27
|
+
"converter",
|
|
28
|
+
"document-chunks",
|
|
29
|
+
"document-conversion",
|
|
30
|
+
"embedding",
|
|
31
|
+
"markdown",
|
|
32
|
+
"markitdownjs",
|
|
33
|
+
"rag",
|
|
34
|
+
"typescript",
|
|
35
|
+
"vector-search"
|
|
36
|
+
],
|
|
25
37
|
"scripts": {
|
|
26
38
|
"build": "tsc --project tsconfig.json",
|
|
27
39
|
"dev": "tsc --watch --project tsconfig.json",
|