mdream 0.13.1 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/splitter.mjs +3 -3
- package/package.json +1 -1
package/dist/splitter.mjs
CHANGED
|
@@ -63,7 +63,7 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
|
|
|
63
63
|
let lineNumber = 1;
|
|
64
64
|
let lastChunkEndPosition = 0;
|
|
65
65
|
let lastSplitPosition = 0;
|
|
66
|
-
function flushChunk(endPosition) {
|
|
66
|
+
function flushChunk(endPosition, applyOverlap = false) {
|
|
67
67
|
const currentMd = getCurrentMarkdown(processor.state);
|
|
68
68
|
const chunkEnd = endPosition ?? currentMd.length;
|
|
69
69
|
const chunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
|
|
@@ -89,7 +89,7 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
|
|
|
89
89
|
chunks.push(chunk);
|
|
90
90
|
currentChunkCodeLanguage = "";
|
|
91
91
|
lastSplitPosition = chunkEnd;
|
|
92
|
-
if (opts.chunkOverlap > 0) {
|
|
92
|
+
if (applyOverlap && opts.chunkOverlap > 0) {
|
|
93
93
|
const maxOverlap = Math.max(0, chunkContent.length - 1);
|
|
94
94
|
const actualOverlap = Math.min(opts.chunkOverlap, maxOverlap);
|
|
95
95
|
lastChunkEndPosition = chunkEnd - actualOverlap;
|
|
@@ -168,7 +168,7 @@ function htmlToMarkdownSplitChunks(html, options = {}) {
|
|
|
168
168
|
}
|
|
169
169
|
}
|
|
170
170
|
if (splitPosition === -1 || splitPosition <= lastChunkEndPosition) splitPosition = currentMd.length;
|
|
171
|
-
flushChunk(splitPosition);
|
|
171
|
+
flushChunk(splitPosition, true);
|
|
172
172
|
}
|
|
173
173
|
}
|
|
174
174
|
});
|