@mdream/js 2.0.0-beta.1 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +402 -126
  2. package/dist/_chunks/clean.mjs +526 -0
  3. package/dist/_chunks/const.mjs +3 -7
  4. package/dist/_chunks/extraction.d.mts +14 -0
  5. package/dist/_chunks/html.mjs +260 -0
  6. package/dist/_chunks/markdown-processor.mjs +1077 -828
  7. package/dist/_chunks/minimal.mjs +29 -0
  8. package/dist/_chunks/output-runner.mjs +54 -0
  9. package/dist/_chunks/parse.mjs +134 -1501
  10. package/dist/_chunks/plugin.mjs +14 -0
  11. package/dist/_chunks/src.mjs +291 -0
  12. package/dist/_chunks/tag-overrides.mjs +138 -0
  13. package/dist/_chunks/{plugins.mjs → tailwind.mjs} +145 -292
  14. package/dist/_chunks/text.mjs +583 -0
  15. package/dist/_chunks/types.d.mts +136 -126
  16. package/dist/_chunks/url.mjs +82 -0
  17. package/dist/_chunks/utils.mjs +659 -0
  18. package/dist/clean.d.mts +10 -0
  19. package/dist/clean.mjs +2 -0
  20. package/dist/cli.mjs +47 -8
  21. package/dist/html.d.mts +6 -0
  22. package/dist/html.mjs +2 -0
  23. package/dist/index.d.mts +5 -4
  24. package/dist/index.mjs +4 -58
  25. package/dist/parse.d.mts +15 -10
  26. package/dist/plugins.d.mts +13 -28
  27. package/dist/plugins.mjs +52 -2
  28. package/dist/preset/minimal.d.mts +14 -2
  29. package/dist/preset/minimal.mjs +1 -32
  30. package/dist/splitter.d.mts +12 -4
  31. package/dist/splitter.mjs +386 -113
  32. package/dist/text.d.mts +6 -0
  33. package/dist/text.mjs +2 -0
  34. package/package.json +34 -8
  35. package/dist/_chunks/index.d.mts +0 -9
  36. package/dist/_chunks/minimal.d.mts +0 -6
  37. package/dist/_chunks/resolve-plugins.mjs +0 -44
  38. package/dist/_chunks/stream.mjs +0 -37
  39. package/dist/core.d.mts +0 -15
  40. package/dist/core.mjs +0 -28
package/dist/splitter.mjs CHANGED
@@ -1,109 +1,240 @@
1
1
  import "./_chunks/const.mjs";
2
- import { parseHtmlStream } from "./_chunks/parse.mjs";
3
- import { createMarkdownProcessor, processPluginsForEvent } from "./_chunks/markdown-processor.mjs";
4
- import { resolvePlugins } from "./_chunks/resolve-plugins.mjs";
2
+ import { trimOutputStart } from "./_chunks/utils.mjs";
3
+ import { createMarkdownProcessor, tagHandlers, trimAsciiWhitespaceEnd } from "./_chunks/markdown-processor.mjs";
4
+ import { assertEngineOptions, buildTagOverrideHandlers, checkClean, endPlugins, processPluginsForEvent } from "./_chunks/tag-overrides.mjs";
5
+ import { finalizeParse, parseHtmlStream } from "./_chunks/parse.mjs";
6
+ import { resolvePlugins } from "./_chunks/plugin.mjs";
7
+ function createOutputPositions(buffer) {
8
+ const positions = [];
9
+ return {
10
+ capture(fragment = Math.max(0, buffer.length - 1), offset = buffer[fragment]?.length ?? 0, kind = "boundary") {
11
+ const position = {
12
+ fragment,
13
+ offset,
14
+ kind
15
+ };
16
+ positions.push(position);
17
+ return position;
18
+ },
19
+ replace(start, end, replacement, mapOffset) {
20
+ if (positions.length === 0) return;
21
+ const fragmentDelta = replacement.length - (end - start);
22
+ let lower = 0;
23
+ let upper = positions.length;
24
+ while (lower < upper) {
25
+ const middle = lower + upper >>> 1;
26
+ if (positions[middle].fragment < start) lower = middle + 1;
27
+ else upper = middle;
28
+ }
29
+ let oldPrefix;
30
+ let newPrefix;
31
+ for (let index = lower; index < positions.length; index++) {
32
+ const position = positions[index];
33
+ if (position.fragment >= end) {
34
+ if (fragmentDelta === 0) break;
35
+ position.fragment += fragmentDelta;
36
+ continue;
37
+ }
38
+ if (!oldPrefix) {
39
+ oldPrefix = [0];
40
+ for (let index = start; index < end; index++) oldPrefix.push(oldPrefix.at(-1) + buffer[index].length);
41
+ newPrefix = [0];
42
+ for (const fragment of replacement) newPrefix.push(newPrefix.at(-1) + fragment.length);
43
+ }
44
+ const oldOffset = oldPrefix[position.fragment - start] + position.offset;
45
+ const endOffset = newPrefix.at(-1);
46
+ const offset = Math.max(0, Math.min(mapOffset ? mapOffset(oldOffset, position.kind) : oldOffset, endOffset));
47
+ if (replacement.length === 0 || position.kind === "boundary" && offset === 0 && start > 0) {
48
+ position.fragment = Math.max(0, start - 1);
49
+ position.offset = start > 0 ? buffer[start - 1].length : 0;
50
+ continue;
51
+ }
52
+ let fragment = 0;
53
+ while (fragment + 1 < replacement.length && (newPrefix[fragment + 1] < offset || position.kind === "content" && newPrefix[fragment + 1] === offset)) fragment++;
54
+ position.fragment = start + fragment;
55
+ position.offset = offset - newPrefix[fragment];
56
+ }
57
+ },
58
+ finish() {
59
+ const prefix = [0];
60
+ for (const fragment of buffer) prefix.push(prefix.at(-1) + fragment.length);
61
+ return (position) => prefix[Math.min(position.fragment, buffer.length)] + Math.min(position.offset, buffer[position.fragment]?.length ?? 0);
62
+ }
63
+ };
64
+ }
5
65
  const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
6
- const NEWLINE_RE = /\n/g;
7
- const DEFAULT_HEADERS_TO_SPLIT_ON = [
8
- 8,
9
- 9,
10
- 10,
11
- 11,
12
- 12
13
- ];
66
+ const DEFAULT_SPLIT_HEADINGS = 62;
67
+ const HEADING_LEVELS_ERROR = "headersToSplitOn takes heading levels from 1 to 6, such as [2, 3]. @mdream/js no longer exports the TAG_H* constants.";
68
+ function headingSplitMask(levels) {
69
+ if (levels == null) return DEFAULT_SPLIT_HEADINGS;
70
+ if (!Array.isArray(levels)) throw new TypeError(HEADING_LEVELS_ERROR);
71
+ let mask = 0;
72
+ for (let index = 0; index < levels.length; index++) {
73
+ const level = levels[index];
74
+ if (!Number.isInteger(level) || level < 1 || level > 6) throw new TypeError(HEADING_LEVELS_ERROR);
75
+ mask |= 1 << level - 1;
76
+ }
77
+ return mask;
78
+ }
14
79
  function createOptions(options) {
15
80
  return {
16
- headersToSplitOn: options.headersToSplitOn ?? DEFAULT_HEADERS_TO_SPLIT_ON,
81
+ splitHeadings: headingSplitMask(options.headersToSplitOn),
17
82
  returnEachLine: options.returnEachLine ?? false,
18
83
  stripHeaders: options.stripHeaders ?? true,
19
84
  chunkSize: options.chunkSize ?? 1e3,
20
85
  chunkOverlap: options.chunkOverlap ?? 200,
21
86
  lengthFunction: options.lengthFunction ?? ((text) => text.length),
87
+ customLength: options.lengthFunction !== void 0,
22
88
  keepSeparator: options.keepSeparator ?? false,
23
- resolvedPlugins: resolvePlugins(options).plugins
89
+ resolvedPlugins: resolvePlugins(options.plugins),
90
+ tagOverrideHandlers: options.tagOverrides ? buildTagOverrideHandlers(options.tagOverrides, tagHandlers) : void 0
24
91
  };
25
92
  }
26
93
  function shouldSplitOnHeader(tagId, options) {
27
- return options.headersToSplitOn.includes(tagId);
94
+ return (options.splitHeadings & 1 << tagId - 7) !== 0;
28
95
  }
29
- function getCurrentMarkdown(state) {
30
- return state.buffer.join("").trimStart();
96
+ function isOutputWhitespace(code) {
97
+ return code === 32 || code >= 9 && code <= 13;
98
+ }
99
+ const SEPARATORS = [
100
+ "\n\n",
101
+ "```\n",
102
+ "\n",
103
+ " "
104
+ ];
105
+ function contentStart(markdown, start, end) {
106
+ let lineStart = start === 0 || markdown.charCodeAt(start - 1) === 10 ? start : -1;
107
+ for (let index = start; index < end; index++) {
108
+ const code = markdown.charCodeAt(index);
109
+ if (code === 10) lineStart = index + 1;
110
+ else if (!isOutputWhitespace(code)) return lineStart === -1 ? index : lineStart;
111
+ }
112
+ return end;
113
+ }
114
+ function contentEnd(markdown, start, end) {
115
+ while (end > start && isOutputWhitespace(markdown.charCodeAt(end - 1))) end--;
116
+ return end;
117
+ }
118
+ function lastIndexBetween(markdown, separator, floor, from) {
119
+ if (from < floor) return -1;
120
+ const index = markdown.slice(floor, from + separator.length).lastIndexOf(separator);
121
+ return index === -1 ? -1 : floor + index;
122
+ }
123
+ function largestFit(max, firstProbe, fits) {
124
+ let good = 0;
125
+ let bad = -1;
126
+ let probe = firstProbe >= 1 ? Math.floor(firstProbe) : 1;
127
+ while (good < max) {
128
+ if (probe > max) probe = max;
129
+ if (!fits(probe)) {
130
+ bad = probe;
131
+ break;
132
+ }
133
+ good = probe;
134
+ probe *= 2;
135
+ }
136
+ if (bad === -1) return good;
137
+ while (bad - good > 1) {
138
+ const middle = Math.floor((good + bad) / 2);
139
+ if (fits(middle)) good = middle;
140
+ else bad = middle;
141
+ }
142
+ return good;
143
+ }
144
+ function codeRegions(markdown) {
145
+ const regions = [];
146
+ let open;
147
+ let lineStart = 0;
148
+ while (lineStart < markdown.length) {
149
+ const newline = markdown.indexOf("\n", lineStart);
150
+ const lineEnd = newline === -1 ? markdown.length : newline;
151
+ let markerStart = lineStart;
152
+ let prefixStart = lineStart;
153
+ let prefix = "";
154
+ while (true) {
155
+ while (markdown.charCodeAt(markerStart) === 32 || markdown.charCodeAt(markerStart) === 62) markerStart++;
156
+ let listEnd = markerStart;
157
+ const first = markdown.charCodeAt(markerStart);
158
+ if ((first === 45 || first === 43 || first === 42) && markdown.charCodeAt(markerStart + 1) === 32) listEnd++;
159
+ else if (first >= 48 && first <= 57) {
160
+ let end = markerStart + 1;
161
+ while (markdown.charCodeAt(end) >= 48 && markdown.charCodeAt(end) <= 57) end++;
162
+ const delimiter = markdown.charCodeAt(end);
163
+ if ((delimiter === 46 || delimiter === 41) && markdown.charCodeAt(end + 1) === 32) listEnd = end + 1;
164
+ }
165
+ if (listEnd === markerStart) break;
166
+ prefix += markdown.slice(prefixStart, markerStart) + " ".repeat(listEnd - markerStart);
167
+ markerStart = listEnd;
168
+ prefixStart = listEnd;
169
+ }
170
+ prefix += markdown.slice(prefixStart, markerStart);
171
+ const marker = markdown.charCodeAt(markerStart);
172
+ if (marker === 96 || marker === 126) {
173
+ let markerEnd = markerStart + 1;
174
+ while (markdown.charCodeAt(markerEnd) === marker) markerEnd++;
175
+ const run = markerEnd - markerStart;
176
+ if (!open && run >= 3) open = {
177
+ start: lineStart,
178
+ markerStart,
179
+ marker,
180
+ run,
181
+ prefix
182
+ };
183
+ else if (open && marker === open.marker && run >= open.run && prefix === open.prefix) {
184
+ let tail = markerEnd;
185
+ while (tail < lineEnd && isOutputWhitespace(markdown.charCodeAt(tail))) tail++;
186
+ if (tail === lineEnd) {
187
+ regions.push({
188
+ start: open.start,
189
+ end: lineEnd,
190
+ markerStart: open.markerStart
191
+ });
192
+ open = void 0;
193
+ }
194
+ }
195
+ }
196
+ lineStart = lineEnd + 1;
197
+ }
198
+ if (open) regions.push({
199
+ start: open.start,
200
+ end: markdown.length,
201
+ markerStart: open.markerStart
202
+ });
203
+ return regions;
31
204
  }
32
205
  function* htmlToMarkdownSplitChunksStream(html, options = {}) {
206
+ assertEngineOptions(options, "splitter");
207
+ checkClean(options);
33
208
  const opts = createOptions(options);
34
209
  if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
35
- let currentChunkCodeLanguage = "";
36
- const processor = createMarkdownProcessor(options, opts.resolvedPlugins);
210
+ const processor = createMarkdownProcessor(options);
211
+ const positions = createOutputPositions(processor.state.buffer);
212
+ processor.state.outputPositions = positions;
213
+ const codePositions = [];
37
214
  processor.state.onCodeFenceOpen = (language) => {
38
- if (language && !currentChunkCodeLanguage) currentChunkCodeLanguage = language;
215
+ if (language) {
216
+ const fence = processor.state.codeFence;
217
+ codePositions.push({
218
+ position: positions.capture(fence.fragment, fence.markerOffset, "content"),
219
+ language
220
+ });
221
+ }
39
222
  };
40
223
  const headerHierarchy = /* @__PURE__ */ new Map();
41
224
  const seenSplitHeaders = /* @__PURE__ */ new Set();
42
225
  let collectingHeaderText = false;
43
226
  let currentHeaderTagId = null;
44
227
  let currentHeaderText = "";
45
- let lineNumber = 1;
46
- let lastChunkEndPosition = 0;
47
- let lastSplitPosition = 0;
48
- function* flushChunk(endPosition, applyOverlap = false) {
49
- const currentMd = getCurrentMarkdown(processor.state);
50
- const chunkEnd = endPosition ?? currentMd.length;
51
- const originalChunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
52
- if (!originalChunkContent.trim()) {
53
- lastChunkEndPosition = chunkEnd;
54
- return;
55
- }
56
- let chunkContent = originalChunkContent;
57
- if (opts.stripHeaders) {
58
- chunkContent = chunkContent.split("\n").filter((line) => !MARKDOWN_HEADER_LINE_RE.test(line)).join("\n").trim();
59
- if (!chunkContent) {
60
- lastChunkEndPosition = chunkEnd;
61
- return;
62
- }
63
- }
64
- const chunk = {
65
- content: chunkContent.trimEnd(),
66
- metadata: { loc: { lines: {
67
- from: lineNumber,
68
- to: lineNumber + (originalChunkContent.match(NEWLINE_RE) || []).length
69
- } } }
70
- };
71
- if (headerHierarchy.size > 0) {
72
- chunk.metadata.headers = {};
73
- for (const [tagId, text] of headerHierarchy.entries()) {
74
- const level = `h${tagId - 7 + 1}`;
75
- chunk.metadata.headers[level] = text;
76
- }
77
- }
78
- if (currentChunkCodeLanguage) chunk.metadata.code = currentChunkCodeLanguage;
79
- yield chunk;
80
- currentChunkCodeLanguage = "";
81
- lastSplitPosition = chunkEnd;
82
- if (applyOverlap && opts.chunkOverlap > 0) {
83
- const maxOverlap = Math.max(0, originalChunkContent.length - 1);
84
- lastChunkEndPosition = chunkEnd - Math.min(opts.chunkOverlap, maxOverlap);
85
- } else lastChunkEndPosition = chunkEnd;
86
- lineNumber += (originalChunkContent.match(NEWLINE_RE) || []).length;
87
- }
88
- const parseState = {
89
- depthMap: processor.state.depthMap,
90
- depth: 0,
91
- resolvedPlugins: opts.resolvedPlugins
228
+ const sectionEnds = [];
229
+ const endSection = () => {
230
+ sectionEnds.push({
231
+ position: positions.capture(),
232
+ headers: new Map(headerHierarchy)
233
+ });
92
234
  };
93
- const eventBuffer = [];
94
235
  const processResolvedEvent = opts.resolvedPlugins?.length ? (event) => processPluginsForEvent(event, opts.resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
95
- parseHtmlStream(html, parseState, (event) => {
96
- eventBuffer.push(event);
97
- });
98
- for (const event of eventBuffer) {
236
+ const handleEvent = (event) => {
99
237
  const { type: eventType, node } = event;
100
- const eventDepthMap = node.type === 1 ? node.depthMap : node.parent?.depthMap;
101
- if (eventDepthMap) processor.state.depthMap.set(eventDepthMap);
102
- else processor.state.depthMap.fill(0);
103
- if (eventType === 1 && node.type === 1) {
104
- const tagId = node.tagId;
105
- if (tagId !== void 0 && tagId >= 0 && tagId < processor.state.depthMap.length) processor.state.depthMap[tagId] = Math.max(0, processor.state.depthMap[tagId] - 1);
106
- }
107
238
  if (node.type === 1) {
108
239
  const tagId = node.tagId;
109
240
  if (tagId && tagId >= 7 && tagId <= 12) {
@@ -113,7 +244,7 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
113
244
  currentHeaderText = "";
114
245
  if (shouldSplitOnHeader(tagId, opts)) {
115
246
  if (seenSplitHeaders.has(tagId)) {
116
- yield* flushChunk();
247
+ endSection();
117
248
  for (let i = tagId; i <= 12; i++) headerHierarchy.delete(i);
118
249
  }
119
250
  seenSplitHeaders.add(tagId);
@@ -124,51 +255,193 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
124
255
  currentHeaderTagId = null;
125
256
  }
126
257
  }
127
- if (tagId === 13 && eventType === 0) yield* flushChunk();
128
- }
129
- if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
258
+ if (tagId === 13 && eventType === 0) endSection();
259
+ } else if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
130
260
  processResolvedEvent(event);
131
- if (!opts.returnEachLine) {
132
- const currentMd = getCurrentMarkdown(processor.state);
133
- if (opts.lengthFunction(currentMd.slice(lastChunkEndPosition)) > opts.chunkSize) {
134
- const idealSplitPos = lastChunkEndPosition + opts.chunkSize;
135
- const separators = [
136
- "\n\n",
137
- "```\n",
138
- "\n",
139
- " "
140
- ];
141
- let splitPosition = -1;
142
- for (const sep of separators) {
143
- const idx = currentMd.lastIndexOf(sep, idealSplitPos);
144
- const candidateSplitPos = idx + sep.length;
145
- if (idx >= 0) {
146
- const beforeSplit = currentMd.slice(0, candidateSplitPos);
147
- let backtickCount = 0;
148
- let pos = beforeSplit.indexOf("```", 0);
149
- while (pos !== -1) {
150
- backtickCount++;
151
- pos = beforeSplit.indexOf("```", pos + 3);
261
+ };
262
+ const parseState = {
263
+ depthMap: processor.state.depthMap,
264
+ depth: 0,
265
+ resolvedPlugins: opts.resolvedPlugins,
266
+ tagHandlers,
267
+ tagOverrideHandlers: opts.tagOverrideHandlers
268
+ };
269
+ const leftover = parseHtmlStream(html, parseState, handleEvent);
270
+ finalizeParse(leftover, parseState, handleEvent);
271
+ endPlugins(opts.resolvedPlugins, processor.state);
272
+ endSection();
273
+ const positionOffset = positions.finish();
274
+ const joined = processor.state.buffer.join("");
275
+ const started = trimOutputStart(joined);
276
+ const leadingTrim = joined.length - started.length;
277
+ const raw = trimAsciiWhitespaceEnd(started);
278
+ const view = processor.finishOutput?.(raw);
279
+ const markdown = view ? view.markdown : raw;
280
+ const mapPosition = view ? view.mapPosition ?? ((position) => processor.finishOutput(raw.slice(0, position)).markdown.length) : void 0;
281
+ const offsetOf = (position) => {
282
+ const offset = Math.min(Math.max(0, positionOffset(position) - leadingTrim), raw.length);
283
+ return offset === raw.length ? markdown.length : mapPosition ? mapPosition(offset) : offset;
284
+ };
285
+ const sections = [];
286
+ for (const { position, headers } of sectionEnds) sections.push({
287
+ end: offsetOf(position),
288
+ headers
289
+ });
290
+ const codes = [];
291
+ for (const { position, language } of codePositions) codes.push({
292
+ start: offsetOf(position),
293
+ language
294
+ });
295
+ codes.sort((a, b) => a.start - b.start);
296
+ yield* chunkSections(markdown, sections, codes, opts);
297
+ }
298
+ function* chunkSections(markdown, sections, codes, opts) {
299
+ const regions = codeRegions(markdown);
300
+ const newlines = [];
301
+ for (let index = markdown.indexOf("\n"); index !== -1; index = markdown.indexOf("\n", index + 1)) newlines.push(index);
302
+ let chunkStart = 0;
303
+ let lastEnd = 0;
304
+ let codeIndex = 0;
305
+ let headers = /* @__PURE__ */ new Map();
306
+ function lineAt(offset) {
307
+ let lower = 0;
308
+ let upper = newlines.length;
309
+ while (lower < upper) {
310
+ const middle = lower + upper >>> 1;
311
+ if (newlines[middle] < offset) lower = middle + 1;
312
+ else upper = middle;
313
+ }
314
+ return lower + 1;
315
+ }
316
+ function codeRegionAt(position) {
317
+ let lower = 0;
318
+ let upper = regions.length;
319
+ while (lower < upper) {
320
+ const middle = lower + upper >>> 1;
321
+ if (regions[middle].start < position) lower = middle + 1;
322
+ else upper = middle;
323
+ }
324
+ const region = lower > 0 ? regions[lower - 1] : void 0;
325
+ return region && position < region.end ? region : void 0;
326
+ }
327
+ function measure(start, end) {
328
+ return opts.lengthFunction(markdown.slice(start, end));
329
+ }
330
+ function advanceTo(position) {
331
+ const next = contentStart(markdown, position, markdown.length);
332
+ if (next > chunkStart) chunkStart = next;
333
+ }
334
+ function sizeLimit(sectionEnd) {
335
+ if (!opts.customLength) return Math.min(sectionEnd, chunkStart + opts.chunkSize);
336
+ const rest = sectionEnd - chunkStart;
337
+ if (rest <= 0) return sectionEnd;
338
+ const fit = largestFit(rest, opts.chunkSize, (length) => !(measure(chunkStart, chunkStart + length) > opts.chunkSize));
339
+ return chunkStart + Math.max(fit, 1);
340
+ }
341
+ function splitPosition(sectionEnd, idealSplitPos) {
342
+ const floor = Math.max(chunkStart, lastEnd);
343
+ let heldRegion;
344
+ for (const separator of SEPARATORS) {
345
+ const index = lastIndexBetween(markdown, separator, Math.max(0, floor - separator.length + 1), idealSplitPos);
346
+ const candidate = index + separator.length;
347
+ if (index < 0 || candidate > sectionEnd || contentEnd(markdown, floor, candidate) <= floor) continue;
348
+ const region = codeRegionAt(candidate);
349
+ if (!region) return candidate;
350
+ heldRegion = region;
351
+ }
352
+ heldRegion ??= codeRegionAt(idealSplitPos);
353
+ if (heldRegion) return Math.min(heldRegion.end, sectionEnd);
354
+ let next = idealSplitPos;
355
+ while (next < sectionEnd) {
356
+ const code = markdown.charCodeAt(next);
357
+ if (code === 32 || code === 10) break;
358
+ next++;
359
+ }
360
+ return next > chunkStart ? next : sectionEnd;
361
+ }
362
+ function* flush(endPosition, applyOverlap) {
363
+ const chunkEnd = Math.min(endPosition, markdown.length);
364
+ const start = chunkStart;
365
+ const end = contentEnd(markdown, start, chunkEnd);
366
+ if (end <= start || end <= lastEnd) {
367
+ advanceTo(chunkEnd);
368
+ return;
369
+ }
370
+ let content = markdown.slice(start, end);
371
+ let contentFrom = start;
372
+ let contentTo = end;
373
+ if (opts.stripHeaders && (content.charCodeAt(0) === 35 || content.includes("\n#"))) {
374
+ const kept = [];
375
+ let firstKept = -1;
376
+ let lastKept = -1;
377
+ let offset = 0;
378
+ for (const line of content.split("\n")) {
379
+ if (line.charCodeAt(0) !== 35 || !MARKDOWN_HEADER_LINE_RE.test(line)) {
380
+ const lineEnd = contentEnd(line, 0, line.length);
381
+ if (lineEnd > 0) {
382
+ if (firstKept === -1) {
383
+ firstKept = kept.length;
384
+ contentFrom = start + offset;
152
385
  }
153
- if (backtickCount % 2 === 1) continue;
154
- }
155
- if (idx >= 0 && candidateSplitPos > lastSplitPosition) {
156
- splitPosition = candidateSplitPos;
157
- break;
386
+ lastKept = kept.length;
387
+ contentTo = start + offset + lineEnd;
158
388
  }
389
+ kept.push(line);
159
390
  }
160
- if (splitPosition === -1 || splitPosition <= lastChunkEndPosition) splitPosition = currentMd.length;
161
- yield* flushChunk(splitPosition, true);
391
+ offset += line.length + 1;
162
392
  }
393
+ if (firstKept === -1) {
394
+ lastEnd = end;
395
+ advanceTo(chunkEnd);
396
+ return;
397
+ }
398
+ const lastLine = kept[lastKept];
399
+ kept[lastKept] = lastLine.slice(0, contentEnd(lastLine, 0, lastLine.length));
400
+ content = kept.slice(firstKept, lastKept + 1).join("\n");
163
401
  }
402
+ const chunk = {
403
+ content,
404
+ metadata: { loc: { lines: {
405
+ from: lineAt(contentFrom),
406
+ to: lineAt(contentTo - 1)
407
+ } } }
408
+ };
409
+ if (headers.size > 0) {
410
+ chunk.metadata.headers = {};
411
+ for (const [tagId, text] of headers.entries()) {
412
+ const level = `h${tagId - 7 + 1}`;
413
+ chunk.metadata.headers[level] = text;
414
+ }
415
+ }
416
+ while (codeIndex < codes.length && codes[codeIndex].start < start) codeIndex++;
417
+ if (codeIndex < codes.length && codes[codeIndex].start < end) chunk.metadata.code = codes[codeIndex].language;
418
+ yield chunk;
419
+ lastEnd = end;
420
+ let next = chunkEnd;
421
+ if (applyOverlap && opts.chunkOverlap > 0) {
422
+ const overlapRoom = end - start - 1;
423
+ let overlapStart = end - (opts.customLength ? largestFit(overlapRoom, opts.chunkOverlap, (length) => !(measure(end - length, end) > opts.chunkOverlap)) : Math.min(opts.chunkOverlap, overlapRoom));
424
+ while (overlapStart > start + 1 && !isOutputWhitespace(markdown.charCodeAt(overlapStart - 1))) overlapStart--;
425
+ const region = codeRegionAt(overlapStart);
426
+ if (region) overlapStart = Math.min(region.end, end);
427
+ if (overlapStart > start + 1) next = overlapStart;
428
+ }
429
+ advanceTo(next);
430
+ }
431
+ for (const section of sections) {
432
+ headers = section.headers;
433
+ if (!opts.returnEachLine) for (let limit = sizeLimit(section.end); limit < section.end; limit = sizeLimit(section.end)) {
434
+ const previousStart = chunkStart;
435
+ yield* flush(splitPosition(section.end, limit), true);
436
+ if (chunkStart <= previousStart) break;
437
+ }
438
+ yield* flush(section.end, false);
164
439
  }
165
- yield* flushChunk();
166
440
  }
167
441
  function htmlToMarkdownSplitChunks(html, options = {}) {
168
- const opts = createOptions(options);
169
442
  const chunks = [];
170
443
  for (const chunk of htmlToMarkdownSplitChunksStream(html, options)) chunks.push(chunk);
171
- if (opts.returnEachLine && chunks.length > 0) {
444
+ if (options.returnEachLine && chunks.length > 0) {
172
445
  const lineChunks = [];
173
446
  for (const chunk of chunks) {
174
447
  const lines = chunk.content.split("\n");
@@ -0,0 +1,6 @@
1
+ import { MdreamOptions } from "./_chunks/types.mjs";
2
+ /** Convert HTML to readable plain text. */
3
+ export declare function htmlToText(html: string, options?: Partial<MdreamOptions>): string;
4
+ /** Stream HTML as readable plain text. */
5
+ export declare function streamHtmlToText(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
6
+ export type { MdreamOptions };
package/dist/text.mjs ADDED
@@ -0,0 +1,2 @@
1
+ import { htmlToText, streamHtmlToText } from "./_chunks/text.mjs";
2
+ export { htmlToText, streamHtmlToText };
package/package.json CHANGED
@@ -1,13 +1,23 @@
1
1
  {
2
2
  "name": "@mdream/js",
3
3
  "type": "module",
4
- "version": "2.0.0-beta.1",
5
- "description": "JavaScript HTML-to-Markdown engine for mdream. Escape hatch for hooks and edge runtimes.",
4
+ "version": "2.0.1",
5
+ "description": "Tree-shakable JavaScript converters and plugins for mdream.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
8
8
  "email": "harlan@harlanzw.com",
9
9
  "url": "https://harlanzw.com/"
10
10
  },
11
+ "contributors": [
12
+ {
13
+ "name": "André Cruz",
14
+ "url": "https://github.com/edevil"
15
+ },
16
+ {
17
+ "name": "Bernardo Pereira",
18
+ "url": "https://github.com/BernardoPe"
19
+ }
20
+ ],
11
21
  "license": "MIT",
12
22
  "repository": {
13
23
  "type": "git",
@@ -24,13 +34,29 @@
24
34
  },
25
35
  "default": "./dist/index.mjs"
26
36
  },
27
- "./core": {
28
- "types": "./dist/core.d.mts",
37
+ "./text": {
38
+ "types": "./dist/text.d.mts",
39
+ "import": {
40
+ "types": "./dist/text.d.mts",
41
+ "default": "./dist/text.mjs"
42
+ },
43
+ "default": "./dist/text.mjs"
44
+ },
45
+ "./html": {
46
+ "types": "./dist/html.d.mts",
47
+ "import": {
48
+ "types": "./dist/html.d.mts",
49
+ "default": "./dist/html.mjs"
50
+ },
51
+ "default": "./dist/html.mjs"
52
+ },
53
+ "./clean": {
54
+ "types": "./dist/clean.d.mts",
29
55
  "import": {
30
- "types": "./dist/core.d.mts",
31
- "default": "./dist/core.mjs"
56
+ "types": "./dist/clean.d.mts",
57
+ "default": "./dist/clean.mjs"
32
58
  },
33
- "default": "./dist/core.mjs"
59
+ "default": "./dist/clean.mjs"
34
60
  },
35
61
  "./negotiate": {
36
62
  "types": "./dist/negotiate.d.mts",
@@ -91,7 +117,7 @@
91
117
  "dist"
92
118
  ],
93
119
  "dependencies": {
94
- "cac": "^7.0.0",
120
+ "cac": "^7.0.1",
95
121
  "pathe": "^2.0.3"
96
122
  },
97
123
  "scripts": {
@@ -1,9 +0,0 @@
1
- import { MdreamOptions, TransformPlugin } from "./types.mjs";
2
- import "./minimal.mjs";
3
- /**
4
- * Create a plugin with type-safe hook definitions.
5
- * All TransformPlugin fields are optional, so this is a typed identity function.
6
- */
7
- export declare function createPlugin(plugin: TransformPlugin): TransformPlugin;
8
- export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
9
- export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
@@ -1,6 +0,0 @@
1
- import { EngineOptions } from "./types.mjs";
2
- /**
3
- * Creates a configurable minimal preset with advanced options.
4
- * Returns declarative plugin config that works with both JS and Rust engines.
5
- */
6
- export declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;