@mdream/js 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -70,7 +70,7 @@ function isAsciiAlpha(code) {
70
70
  }
71
71
  function shouldProtectDecodedEntityReferences(state) {
72
72
  const depthMap = state.depthMap;
73
- return !state.plainText && !depthMap[34] && !depthMap[23] && !isInsideRawHtmlBlock(depthMap);
73
+ return !state.plainText && !depthMap[4] && !depthMap[34] && !depthMap[23] && !isInsideRawHtmlBlock(depthMap);
74
74
  }
75
75
  const SCRIPT_DATA = 0;
76
76
  const SCRIPT_DATA_ESCAPED = 1;
package/dist/splitter.mjs CHANGED
@@ -84,6 +84,7 @@ function createOptions(options) {
84
84
  chunkSize: options.chunkSize ?? 1e3,
85
85
  chunkOverlap: options.chunkOverlap ?? 200,
86
86
  lengthFunction: options.lengthFunction ?? ((text) => text.length),
87
+ customLength: options.lengthFunction !== void 0,
87
88
  keepSeparator: options.keepSeparator ?? false,
88
89
  resolvedPlugins: resolvePlugins(options.plugins),
89
90
  tagOverrideHandlers: options.tagOverrides ? buildTagOverrideHandlers(options.tagOverrides, tagHandlers) : void 0
@@ -119,6 +120,27 @@ function lastIndexBetween(markdown, separator, floor, from) {
119
120
  const index = markdown.slice(floor, from + separator.length).lastIndexOf(separator);
120
121
  return index === -1 ? -1 : floor + index;
121
122
  }
123
+ function largestFit(max, firstProbe, fits) {
124
+ let good = 0;
125
+ let bad = -1;
126
+ let probe = firstProbe >= 1 ? Math.floor(firstProbe) : 1;
127
+ while (good < max) {
128
+ if (probe > max) probe = max;
129
+ if (!fits(probe)) {
130
+ bad = probe;
131
+ break;
132
+ }
133
+ good = probe;
134
+ probe *= 2;
135
+ }
136
+ if (bad === -1) return good;
137
+ while (bad - good > 1) {
138
+ const middle = Math.floor((good + bad) / 2);
139
+ if (fits(middle)) good = middle;
140
+ else bad = middle;
141
+ }
142
+ return good;
143
+ }
122
144
  function codeRegions(markdown) {
123
145
  const regions = [];
124
146
  let open;
@@ -271,10 +293,9 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
271
293
  language
272
294
  });
273
295
  codes.sort((a, b) => a.start - b.start);
274
- const lengthFunction = options.lengthFunction;
275
- yield* chunkSections(markdown, sections, codes, opts, lengthFunction ? (start, end) => lengthFunction(markdown.slice(start, end)) : (start, end) => end - start);
296
+ yield* chunkSections(markdown, sections, codes, opts);
276
297
  }
277
- function* chunkSections(markdown, sections, codes, opts, measure) {
298
+ function* chunkSections(markdown, sections, codes, opts) {
278
299
  const regions = codeRegions(markdown);
279
300
  const newlines = [];
280
301
  for (let index = markdown.indexOf("\n"); index !== -1; index = markdown.indexOf("\n", index + 1)) newlines.push(index);
@@ -303,12 +324,21 @@ function* chunkSections(markdown, sections, codes, opts, measure) {
303
324
  const region = lower > 0 ? regions[lower - 1] : void 0;
304
325
  return region && position < region.end ? region : void 0;
305
326
  }
327
+ function measure(start, end) {
328
+ return opts.lengthFunction(markdown.slice(start, end));
329
+ }
306
330
  function advanceTo(position) {
307
331
  const next = contentStart(markdown, position, markdown.length);
308
332
  if (next > chunkStart) chunkStart = next;
309
333
  }
310
- function splitPosition(sectionEnd) {
311
- const idealSplitPos = Math.min(sectionEnd, chunkStart + opts.chunkSize);
334
+ function sizeLimit(sectionEnd) {
335
+ if (!opts.customLength) return Math.min(sectionEnd, chunkStart + opts.chunkSize);
336
+ const rest = sectionEnd - chunkStart;
337
+ if (rest <= 0) return sectionEnd;
338
+ const fit = largestFit(rest, opts.chunkSize, (length) => !(measure(chunkStart, chunkStart + length) > opts.chunkSize));
339
+ return chunkStart + Math.max(fit, 1);
340
+ }
341
+ function splitPosition(sectionEnd, idealSplitPos) {
312
342
  const floor = Math.max(chunkStart, lastEnd);
313
343
  let heldRegion;
314
344
  for (const separator of SEPARATORS) {
@@ -389,7 +419,8 @@ function* chunkSections(markdown, sections, codes, opts, measure) {
389
419
  lastEnd = end;
390
420
  let next = chunkEnd;
391
421
  if (applyOverlap && opts.chunkOverlap > 0) {
392
- let overlapStart = end - Math.min(opts.chunkOverlap, end - start - 1);
422
+ const overlapRoom = end - start - 1;
423
+ let overlapStart = end - (opts.customLength ? largestFit(overlapRoom, opts.chunkOverlap, (length) => !(measure(end - length, end) > opts.chunkOverlap)) : Math.min(opts.chunkOverlap, overlapRoom));
393
424
  while (overlapStart > start + 1 && !isOutputWhitespace(markdown.charCodeAt(overlapStart - 1))) overlapStart--;
394
425
  const region = codeRegionAt(overlapStart);
395
426
  if (region) overlapStart = Math.min(region.end, end);
@@ -399,9 +430,9 @@ function* chunkSections(markdown, sections, codes, opts, measure) {
399
430
  }
400
431
  for (const section of sections) {
401
432
  headers = section.headers;
402
- if (!opts.returnEachLine) while (measure(chunkStart, section.end) > opts.chunkSize) {
433
+ if (!opts.returnEachLine) for (let limit = sizeLimit(section.end); limit < section.end; limit = sizeLimit(section.end)) {
403
434
  const previousStart = chunkStart;
404
- yield* flush(splitPosition(section.end), true);
435
+ yield* flush(splitPosition(section.end, limit), true);
405
436
  if (chunkStart <= previousStart) break;
406
437
  }
407
438
  yield* flush(section.end, false);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/js",
3
3
  "type": "module",
4
- "version": "2.0.0",
4
+ "version": "2.0.1",
5
5
  "description": "Tree-shakable JavaScript converters and plugins for mdream.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -117,7 +117,7 @@
117
117
  "dist"
118
118
  ],
119
119
  "dependencies": {
120
- "cac": "^7.0.0",
120
+ "cac": "^7.0.1",
121
121
  "pathe": "^2.0.3"
122
122
  },
123
123
  "scripts": {