@mdream/js 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/parse.mjs +1 -1
- package/dist/splitter.mjs +39 -8
- package/package.json +2 -2
package/dist/_chunks/parse.mjs
CHANGED
|
@@ -70,7 +70,7 @@ function isAsciiAlpha(code) {
|
|
|
70
70
|
}
|
|
71
71
|
function shouldProtectDecodedEntityReferences(state) {
|
|
72
72
|
const depthMap = state.depthMap;
|
|
73
|
-
return !state.plainText && !depthMap[34] && !depthMap[23] && !isInsideRawHtmlBlock(depthMap);
|
|
73
|
+
return !state.plainText && !depthMap[4] && !depthMap[34] && !depthMap[23] && !isInsideRawHtmlBlock(depthMap);
|
|
74
74
|
}
|
|
75
75
|
const SCRIPT_DATA = 0;
|
|
76
76
|
const SCRIPT_DATA_ESCAPED = 1;
|
package/dist/splitter.mjs
CHANGED
|
@@ -84,6 +84,7 @@ function createOptions(options) {
|
|
|
84
84
|
chunkSize: options.chunkSize ?? 1e3,
|
|
85
85
|
chunkOverlap: options.chunkOverlap ?? 200,
|
|
86
86
|
lengthFunction: options.lengthFunction ?? ((text) => text.length),
|
|
87
|
+
customLength: options.lengthFunction !== void 0,
|
|
87
88
|
keepSeparator: options.keepSeparator ?? false,
|
|
88
89
|
resolvedPlugins: resolvePlugins(options.plugins),
|
|
89
90
|
tagOverrideHandlers: options.tagOverrides ? buildTagOverrideHandlers(options.tagOverrides, tagHandlers) : void 0
|
|
@@ -119,6 +120,27 @@ function lastIndexBetween(markdown, separator, floor, from) {
|
|
|
119
120
|
const index = markdown.slice(floor, from + separator.length).lastIndexOf(separator);
|
|
120
121
|
return index === -1 ? -1 : floor + index;
|
|
121
122
|
}
|
|
123
|
+
function largestFit(max, firstProbe, fits) {
|
|
124
|
+
let good = 0;
|
|
125
|
+
let bad = -1;
|
|
126
|
+
let probe = firstProbe >= 1 ? Math.floor(firstProbe) : 1;
|
|
127
|
+
while (good < max) {
|
|
128
|
+
if (probe > max) probe = max;
|
|
129
|
+
if (!fits(probe)) {
|
|
130
|
+
bad = probe;
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
good = probe;
|
|
134
|
+
probe *= 2;
|
|
135
|
+
}
|
|
136
|
+
if (bad === -1) return good;
|
|
137
|
+
while (bad - good > 1) {
|
|
138
|
+
const middle = Math.floor((good + bad) / 2);
|
|
139
|
+
if (fits(middle)) good = middle;
|
|
140
|
+
else bad = middle;
|
|
141
|
+
}
|
|
142
|
+
return good;
|
|
143
|
+
}
|
|
122
144
|
function codeRegions(markdown) {
|
|
123
145
|
const regions = [];
|
|
124
146
|
let open;
|
|
@@ -271,10 +293,9 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
|
271
293
|
language
|
|
272
294
|
});
|
|
273
295
|
codes.sort((a, b) => a.start - b.start);
|
|
274
|
-
|
|
275
|
-
yield* chunkSections(markdown, sections, codes, opts, lengthFunction ? (start, end) => lengthFunction(markdown.slice(start, end)) : (start, end) => end - start);
|
|
296
|
+
yield* chunkSections(markdown, sections, codes, opts);
|
|
276
297
|
}
|
|
277
|
-
function* chunkSections(markdown, sections, codes, opts
|
|
298
|
+
function* chunkSections(markdown, sections, codes, opts) {
|
|
278
299
|
const regions = codeRegions(markdown);
|
|
279
300
|
const newlines = [];
|
|
280
301
|
for (let index = markdown.indexOf("\n"); index !== -1; index = markdown.indexOf("\n", index + 1)) newlines.push(index);
|
|
@@ -303,12 +324,21 @@ function* chunkSections(markdown, sections, codes, opts, measure) {
|
|
|
303
324
|
const region = lower > 0 ? regions[lower - 1] : void 0;
|
|
304
325
|
return region && position < region.end ? region : void 0;
|
|
305
326
|
}
|
|
327
|
+
function measure(start, end) {
|
|
328
|
+
return opts.lengthFunction(markdown.slice(start, end));
|
|
329
|
+
}
|
|
306
330
|
function advanceTo(position) {
|
|
307
331
|
const next = contentStart(markdown, position, markdown.length);
|
|
308
332
|
if (next > chunkStart) chunkStart = next;
|
|
309
333
|
}
|
|
310
|
-
function
|
|
311
|
-
|
|
334
|
+
function sizeLimit(sectionEnd) {
|
|
335
|
+
if (!opts.customLength) return Math.min(sectionEnd, chunkStart + opts.chunkSize);
|
|
336
|
+
const rest = sectionEnd - chunkStart;
|
|
337
|
+
if (rest <= 0) return sectionEnd;
|
|
338
|
+
const fit = largestFit(rest, opts.chunkSize, (length) => !(measure(chunkStart, chunkStart + length) > opts.chunkSize));
|
|
339
|
+
return chunkStart + Math.max(fit, 1);
|
|
340
|
+
}
|
|
341
|
+
function splitPosition(sectionEnd, idealSplitPos) {
|
|
312
342
|
const floor = Math.max(chunkStart, lastEnd);
|
|
313
343
|
let heldRegion;
|
|
314
344
|
for (const separator of SEPARATORS) {
|
|
@@ -389,7 +419,8 @@ function* chunkSections(markdown, sections, codes, opts, measure) {
|
|
|
389
419
|
lastEnd = end;
|
|
390
420
|
let next = chunkEnd;
|
|
391
421
|
if (applyOverlap && opts.chunkOverlap > 0) {
|
|
392
|
-
|
|
422
|
+
const overlapRoom = end - start - 1;
|
|
423
|
+
let overlapStart = end - (opts.customLength ? largestFit(overlapRoom, opts.chunkOverlap, (length) => !(measure(end - length, end) > opts.chunkOverlap)) : Math.min(opts.chunkOverlap, overlapRoom));
|
|
393
424
|
while (overlapStart > start + 1 && !isOutputWhitespace(markdown.charCodeAt(overlapStart - 1))) overlapStart--;
|
|
394
425
|
const region = codeRegionAt(overlapStart);
|
|
395
426
|
if (region) overlapStart = Math.min(region.end, end);
|
|
@@ -399,9 +430,9 @@ function* chunkSections(markdown, sections, codes, opts, measure) {
|
|
|
399
430
|
}
|
|
400
431
|
for (const section of sections) {
|
|
401
432
|
headers = section.headers;
|
|
402
|
-
if (!opts.returnEachLine)
|
|
433
|
+
if (!opts.returnEachLine) for (let limit = sizeLimit(section.end); limit < section.end; limit = sizeLimit(section.end)) {
|
|
403
434
|
const previousStart = chunkStart;
|
|
404
|
-
yield* flush(splitPosition(section.end), true);
|
|
435
|
+
yield* flush(splitPosition(section.end, limit), true);
|
|
405
436
|
if (chunkStart <= previousStart) break;
|
|
406
437
|
}
|
|
407
438
|
yield* flush(section.end, false);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/js",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "2.0.
|
|
4
|
+
"version": "2.0.1",
|
|
5
5
|
"description": "Tree-shakable JavaScript converters and plugins for mdream.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -117,7 +117,7 @@
|
|
|
117
117
|
"dist"
|
|
118
118
|
],
|
|
119
119
|
"dependencies": {
|
|
120
|
-
"cac": "^7.0.
|
|
120
|
+
"cac": "^7.0.1",
|
|
121
121
|
"pathe": "^2.0.3"
|
|
122
122
|
},
|
|
123
123
|
"scripts": {
|