@mdream/js 1.7.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +402 -126
  2. package/dist/_chunks/clean.mjs +526 -0
  3. package/dist/_chunks/const.mjs +3 -7
  4. package/dist/_chunks/extraction.d.mts +14 -0
  5. package/dist/_chunks/html.mjs +260 -0
  6. package/dist/_chunks/markdown-processor.mjs +1077 -827
  7. package/dist/_chunks/minimal.mjs +29 -0
  8. package/dist/_chunks/output-runner.mjs +54 -0
  9. package/dist/_chunks/parse.mjs +136 -1490
  10. package/dist/_chunks/plugin.mjs +14 -0
  11. package/dist/_chunks/src.mjs +291 -0
  12. package/dist/_chunks/tag-overrides.mjs +138 -0
  13. package/dist/_chunks/{plugins.mjs → tailwind.mjs} +145 -292
  14. package/dist/_chunks/text.mjs +583 -0
  15. package/dist/_chunks/types.d.mts +136 -122
  16. package/dist/_chunks/url.mjs +82 -0
  17. package/dist/_chunks/utils.mjs +659 -0
  18. package/dist/clean.d.mts +10 -0
  19. package/dist/clean.mjs +2 -0
  20. package/dist/cli.mjs +47 -8
  21. package/dist/html.d.mts +6 -0
  22. package/dist/html.mjs +2 -0
  23. package/dist/index.d.mts +5 -4
  24. package/dist/index.mjs +4 -34
  25. package/dist/parse.d.mts +15 -10
  26. package/dist/plugins.d.mts +13 -28
  27. package/dist/plugins.mjs +52 -2
  28. package/dist/preset/minimal.d.mts +14 -2
  29. package/dist/preset/minimal.mjs +1 -27
  30. package/dist/splitter.d.mts +12 -4
  31. package/dist/splitter.mjs +355 -113
  32. package/dist/text.d.mts +6 -0
  33. package/dist/text.mjs +2 -0
  34. package/package.json +33 -7
  35. package/dist/_chunks/index.d.mts +0 -9
  36. package/dist/_chunks/minimal.d.mts +0 -6
  37. package/dist/_chunks/resolve-plugins.mjs +0 -44
  38. package/dist/_chunks/stream.mjs +0 -37
  39. package/dist/core.d.mts +0 -15
  40. package/dist/core.mjs +0 -28
package/dist/splitter.mjs CHANGED
@@ -1,109 +1,218 @@
1
1
  import "./_chunks/const.mjs";
2
- import { parseHtmlStream } from "./_chunks/parse.mjs";
3
- import { createMarkdownProcessor, processPluginsForEvent } from "./_chunks/markdown-processor.mjs";
4
- import { resolvePlugins } from "./_chunks/resolve-plugins.mjs";
2
+ import { trimOutputStart } from "./_chunks/utils.mjs";
3
+ import { createMarkdownProcessor, tagHandlers, trimAsciiWhitespaceEnd } from "./_chunks/markdown-processor.mjs";
4
+ import { assertEngineOptions, buildTagOverrideHandlers, checkClean, endPlugins, processPluginsForEvent } from "./_chunks/tag-overrides.mjs";
5
+ import { finalizeParse, parseHtmlStream } from "./_chunks/parse.mjs";
6
+ import { resolvePlugins } from "./_chunks/plugin.mjs";
7
+ function createOutputPositions(buffer) {
8
+ const positions = [];
9
+ return {
10
+ capture(fragment = Math.max(0, buffer.length - 1), offset = buffer[fragment]?.length ?? 0, kind = "boundary") {
11
+ const position = {
12
+ fragment,
13
+ offset,
14
+ kind
15
+ };
16
+ positions.push(position);
17
+ return position;
18
+ },
19
+ replace(start, end, replacement, mapOffset) {
20
+ if (positions.length === 0) return;
21
+ const fragmentDelta = replacement.length - (end - start);
22
+ let lower = 0;
23
+ let upper = positions.length;
24
+ while (lower < upper) {
25
+ const middle = lower + upper >>> 1;
26
+ if (positions[middle].fragment < start) lower = middle + 1;
27
+ else upper = middle;
28
+ }
29
+ let oldPrefix;
30
+ let newPrefix;
31
+ for (let index = lower; index < positions.length; index++) {
32
+ const position = positions[index];
33
+ if (position.fragment >= end) {
34
+ if (fragmentDelta === 0) break;
35
+ position.fragment += fragmentDelta;
36
+ continue;
37
+ }
38
+ if (!oldPrefix) {
39
+ oldPrefix = [0];
40
+ for (let index = start; index < end; index++) oldPrefix.push(oldPrefix.at(-1) + buffer[index].length);
41
+ newPrefix = [0];
42
+ for (const fragment of replacement) newPrefix.push(newPrefix.at(-1) + fragment.length);
43
+ }
44
+ const oldOffset = oldPrefix[position.fragment - start] + position.offset;
45
+ const endOffset = newPrefix.at(-1);
46
+ const offset = Math.max(0, Math.min(mapOffset ? mapOffset(oldOffset, position.kind) : oldOffset, endOffset));
47
+ if (replacement.length === 0 || position.kind === "boundary" && offset === 0 && start > 0) {
48
+ position.fragment = Math.max(0, start - 1);
49
+ position.offset = start > 0 ? buffer[start - 1].length : 0;
50
+ continue;
51
+ }
52
+ let fragment = 0;
53
+ while (fragment + 1 < replacement.length && (newPrefix[fragment + 1] < offset || position.kind === "content" && newPrefix[fragment + 1] === offset)) fragment++;
54
+ position.fragment = start + fragment;
55
+ position.offset = offset - newPrefix[fragment];
56
+ }
57
+ },
58
+ finish() {
59
+ const prefix = [0];
60
+ for (const fragment of buffer) prefix.push(prefix.at(-1) + fragment.length);
61
+ return (position) => prefix[Math.min(position.fragment, buffer.length)] + Math.min(position.offset, buffer[position.fragment]?.length ?? 0);
62
+ }
63
+ };
64
+ }
5
65
  const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
6
- const NEWLINE_RE = /\n/g;
7
- const DEFAULT_HEADERS_TO_SPLIT_ON = [
8
- 8,
9
- 9,
10
- 10,
11
- 11,
12
- 12
13
- ];
66
+ const DEFAULT_SPLIT_HEADINGS = 62;
67
+ const HEADING_LEVELS_ERROR = "headersToSplitOn takes heading levels from 1 to 6, such as [2, 3]. @mdream/js no longer exports the TAG_H* constants.";
68
+ function headingSplitMask(levels) {
69
+ if (levels == null) return DEFAULT_SPLIT_HEADINGS;
70
+ if (!Array.isArray(levels)) throw new TypeError(HEADING_LEVELS_ERROR);
71
+ let mask = 0;
72
+ for (let index = 0; index < levels.length; index++) {
73
+ const level = levels[index];
74
+ if (!Number.isInteger(level) || level < 1 || level > 6) throw new TypeError(HEADING_LEVELS_ERROR);
75
+ mask |= 1 << level - 1;
76
+ }
77
+ return mask;
78
+ }
14
79
  function createOptions(options) {
15
80
  return {
16
- headersToSplitOn: options.headersToSplitOn ?? DEFAULT_HEADERS_TO_SPLIT_ON,
81
+ splitHeadings: headingSplitMask(options.headersToSplitOn),
17
82
  returnEachLine: options.returnEachLine ?? false,
18
83
  stripHeaders: options.stripHeaders ?? true,
19
84
  chunkSize: options.chunkSize ?? 1e3,
20
85
  chunkOverlap: options.chunkOverlap ?? 200,
21
86
  lengthFunction: options.lengthFunction ?? ((text) => text.length),
22
87
  keepSeparator: options.keepSeparator ?? false,
23
- resolvedPlugins: resolvePlugins(options).plugins
88
+ resolvedPlugins: resolvePlugins(options.plugins),
89
+ tagOverrideHandlers: options.tagOverrides ? buildTagOverrideHandlers(options.tagOverrides, tagHandlers) : void 0
24
90
  };
25
91
  }
26
92
  function shouldSplitOnHeader(tagId, options) {
27
- return options.headersToSplitOn.includes(tagId);
93
+ return (options.splitHeadings & 1 << tagId - 7) !== 0;
94
+ }
95
+ function isOutputWhitespace(code) {
96
+ return code === 32 || code >= 9 && code <= 13;
97
+ }
98
+ const SEPARATORS = [
99
+ "\n\n",
100
+ "```\n",
101
+ "\n",
102
+ " "
103
+ ];
104
+ function contentStart(markdown, start, end) {
105
+ let lineStart = start === 0 || markdown.charCodeAt(start - 1) === 10 ? start : -1;
106
+ for (let index = start; index < end; index++) {
107
+ const code = markdown.charCodeAt(index);
108
+ if (code === 10) lineStart = index + 1;
109
+ else if (!isOutputWhitespace(code)) return lineStart === -1 ? index : lineStart;
110
+ }
111
+ return end;
112
+ }
113
+ function contentEnd(markdown, start, end) {
114
+ while (end > start && isOutputWhitespace(markdown.charCodeAt(end - 1))) end--;
115
+ return end;
28
116
  }
29
- function getCurrentMarkdown(state) {
30
- return state.buffer.join("").trimStart();
117
+ function lastIndexBetween(markdown, separator, floor, from) {
118
+ if (from < floor) return -1;
119
+ const index = markdown.slice(floor, from + separator.length).lastIndexOf(separator);
120
+ return index === -1 ? -1 : floor + index;
121
+ }
122
+ function codeRegions(markdown) {
123
+ const regions = [];
124
+ let open;
125
+ let lineStart = 0;
126
+ while (lineStart < markdown.length) {
127
+ const newline = markdown.indexOf("\n", lineStart);
128
+ const lineEnd = newline === -1 ? markdown.length : newline;
129
+ let markerStart = lineStart;
130
+ let prefixStart = lineStart;
131
+ let prefix = "";
132
+ while (true) {
133
+ while (markdown.charCodeAt(markerStart) === 32 || markdown.charCodeAt(markerStart) === 62) markerStart++;
134
+ let listEnd = markerStart;
135
+ const first = markdown.charCodeAt(markerStart);
136
+ if ((first === 45 || first === 43 || first === 42) && markdown.charCodeAt(markerStart + 1) === 32) listEnd++;
137
+ else if (first >= 48 && first <= 57) {
138
+ let end = markerStart + 1;
139
+ while (markdown.charCodeAt(end) >= 48 && markdown.charCodeAt(end) <= 57) end++;
140
+ const delimiter = markdown.charCodeAt(end);
141
+ if ((delimiter === 46 || delimiter === 41) && markdown.charCodeAt(end + 1) === 32) listEnd = end + 1;
142
+ }
143
+ if (listEnd === markerStart) break;
144
+ prefix += markdown.slice(prefixStart, markerStart) + " ".repeat(listEnd - markerStart);
145
+ markerStart = listEnd;
146
+ prefixStart = listEnd;
147
+ }
148
+ prefix += markdown.slice(prefixStart, markerStart);
149
+ const marker = markdown.charCodeAt(markerStart);
150
+ if (marker === 96 || marker === 126) {
151
+ let markerEnd = markerStart + 1;
152
+ while (markdown.charCodeAt(markerEnd) === marker) markerEnd++;
153
+ const run = markerEnd - markerStart;
154
+ if (!open && run >= 3) open = {
155
+ start: lineStart,
156
+ markerStart,
157
+ marker,
158
+ run,
159
+ prefix
160
+ };
161
+ else if (open && marker === open.marker && run >= open.run && prefix === open.prefix) {
162
+ let tail = markerEnd;
163
+ while (tail < lineEnd && isOutputWhitespace(markdown.charCodeAt(tail))) tail++;
164
+ if (tail === lineEnd) {
165
+ regions.push({
166
+ start: open.start,
167
+ end: lineEnd,
168
+ markerStart: open.markerStart
169
+ });
170
+ open = void 0;
171
+ }
172
+ }
173
+ }
174
+ lineStart = lineEnd + 1;
175
+ }
176
+ if (open) regions.push({
177
+ start: open.start,
178
+ end: markdown.length,
179
+ markerStart: open.markerStart
180
+ });
181
+ return regions;
31
182
  }
32
183
  function* htmlToMarkdownSplitChunksStream(html, options = {}) {
184
+ assertEngineOptions(options, "splitter");
185
+ checkClean(options);
33
186
  const opts = createOptions(options);
34
187
  if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
35
- let currentChunkCodeLanguage = "";
36
- const processor = createMarkdownProcessor(options, opts.resolvedPlugins);
188
+ const processor = createMarkdownProcessor(options);
189
+ const positions = createOutputPositions(processor.state.buffer);
190
+ processor.state.outputPositions = positions;
191
+ const codePositions = [];
37
192
  processor.state.onCodeFenceOpen = (language) => {
38
- if (language && !currentChunkCodeLanguage) currentChunkCodeLanguage = language;
193
+ if (language) {
194
+ const fence = processor.state.codeFence;
195
+ codePositions.push({
196
+ position: positions.capture(fence.fragment, fence.markerOffset, "content"),
197
+ language
198
+ });
199
+ }
39
200
  };
40
201
  const headerHierarchy = /* @__PURE__ */ new Map();
41
202
  const seenSplitHeaders = /* @__PURE__ */ new Set();
42
203
  let collectingHeaderText = false;
43
204
  let currentHeaderTagId = null;
44
205
  let currentHeaderText = "";
45
- let lineNumber = 1;
46
- let lastChunkEndPosition = 0;
47
- let lastSplitPosition = 0;
48
- function* flushChunk(endPosition, applyOverlap = false) {
49
- const currentMd = getCurrentMarkdown(processor.state);
50
- const chunkEnd = endPosition ?? currentMd.length;
51
- const originalChunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
52
- if (!originalChunkContent.trim()) {
53
- lastChunkEndPosition = chunkEnd;
54
- return;
55
- }
56
- let chunkContent = originalChunkContent;
57
- if (opts.stripHeaders) {
58
- chunkContent = chunkContent.split("\n").filter((line) => !MARKDOWN_HEADER_LINE_RE.test(line)).join("\n").trim();
59
- if (!chunkContent) {
60
- lastChunkEndPosition = chunkEnd;
61
- return;
62
- }
63
- }
64
- const chunk = {
65
- content: chunkContent.trimEnd(),
66
- metadata: { loc: { lines: {
67
- from: lineNumber,
68
- to: lineNumber + (originalChunkContent.match(NEWLINE_RE) || []).length
69
- } } }
70
- };
71
- if (headerHierarchy.size > 0) {
72
- chunk.metadata.headers = {};
73
- for (const [tagId, text] of headerHierarchy.entries()) {
74
- const level = `h${tagId - 7 + 1}`;
75
- chunk.metadata.headers[level] = text;
76
- }
77
- }
78
- if (currentChunkCodeLanguage) chunk.metadata.code = currentChunkCodeLanguage;
79
- yield chunk;
80
- currentChunkCodeLanguage = "";
81
- lastSplitPosition = chunkEnd;
82
- if (applyOverlap && opts.chunkOverlap > 0) {
83
- const maxOverlap = Math.max(0, originalChunkContent.length - 1);
84
- lastChunkEndPosition = chunkEnd - Math.min(opts.chunkOverlap, maxOverlap);
85
- } else lastChunkEndPosition = chunkEnd;
86
- lineNumber += (originalChunkContent.match(NEWLINE_RE) || []).length;
87
- }
88
- const parseState = {
89
- depthMap: processor.state.depthMap,
90
- depth: 0,
91
- resolvedPlugins: opts.resolvedPlugins
206
+ const sectionEnds = [];
207
+ const endSection = () => {
208
+ sectionEnds.push({
209
+ position: positions.capture(),
210
+ headers: new Map(headerHierarchy)
211
+ });
92
212
  };
93
- const eventBuffer = [];
94
213
  const processResolvedEvent = opts.resolvedPlugins?.length ? (event) => processPluginsForEvent(event, opts.resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
95
- parseHtmlStream(html, parseState, (event) => {
96
- eventBuffer.push(event);
97
- });
98
- for (const event of eventBuffer) {
214
+ const handleEvent = (event) => {
99
215
  const { type: eventType, node } = event;
100
- const eventDepthMap = node.type === 1 ? node.depthMap : node.parent?.depthMap;
101
- if (eventDepthMap) processor.state.depthMap.set(eventDepthMap);
102
- else processor.state.depthMap.fill(0);
103
- if (eventType === 1 && node.type === 1) {
104
- const tagId = node.tagId;
105
- if (tagId !== void 0 && tagId >= 0 && tagId < processor.state.depthMap.length) processor.state.depthMap[tagId] = Math.max(0, processor.state.depthMap[tagId] - 1);
106
- }
107
216
  if (node.type === 1) {
108
217
  const tagId = node.tagId;
109
218
  if (tagId && tagId >= 7 && tagId <= 12) {
@@ -113,7 +222,7 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
113
222
  currentHeaderText = "";
114
223
  if (shouldSplitOnHeader(tagId, opts)) {
115
224
  if (seenSplitHeaders.has(tagId)) {
116
- yield* flushChunk();
225
+ endSection();
117
226
  for (let i = tagId; i <= 12; i++) headerHierarchy.delete(i);
118
227
  }
119
228
  seenSplitHeaders.add(tagId);
@@ -124,51 +233,184 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
124
233
  currentHeaderTagId = null;
125
234
  }
126
235
  }
127
- if (tagId === 13 && eventType === 0) yield* flushChunk();
128
- }
129
- if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
236
+ if (tagId === 13 && eventType === 0) endSection();
237
+ } else if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
130
238
  processResolvedEvent(event);
131
- if (!opts.returnEachLine) {
132
- const currentMd = getCurrentMarkdown(processor.state);
133
- if (opts.lengthFunction(currentMd.slice(lastChunkEndPosition)) > opts.chunkSize) {
134
- const idealSplitPos = lastChunkEndPosition + opts.chunkSize;
135
- const separators = [
136
- "\n\n",
137
- "```\n",
138
- "\n",
139
- " "
140
- ];
141
- let splitPosition = -1;
142
- for (const sep of separators) {
143
- const idx = currentMd.lastIndexOf(sep, idealSplitPos);
144
- const candidateSplitPos = idx + sep.length;
145
- if (idx >= 0) {
146
- const beforeSplit = currentMd.slice(0, candidateSplitPos);
147
- let backtickCount = 0;
148
- let pos = beforeSplit.indexOf("```", 0);
149
- while (pos !== -1) {
150
- backtickCount++;
151
- pos = beforeSplit.indexOf("```", pos + 3);
239
+ };
240
+ const parseState = {
241
+ depthMap: processor.state.depthMap,
242
+ depth: 0,
243
+ resolvedPlugins: opts.resolvedPlugins,
244
+ tagHandlers,
245
+ tagOverrideHandlers: opts.tagOverrideHandlers
246
+ };
247
+ const leftover = parseHtmlStream(html, parseState, handleEvent);
248
+ finalizeParse(leftover, parseState, handleEvent);
249
+ endPlugins(opts.resolvedPlugins, processor.state);
250
+ endSection();
251
+ const positionOffset = positions.finish();
252
+ const joined = processor.state.buffer.join("");
253
+ const started = trimOutputStart(joined);
254
+ const leadingTrim = joined.length - started.length;
255
+ const raw = trimAsciiWhitespaceEnd(started);
256
+ const view = processor.finishOutput?.(raw);
257
+ const markdown = view ? view.markdown : raw;
258
+ const mapPosition = view ? view.mapPosition ?? ((position) => processor.finishOutput(raw.slice(0, position)).markdown.length) : void 0;
259
+ const offsetOf = (position) => {
260
+ const offset = Math.min(Math.max(0, positionOffset(position) - leadingTrim), raw.length);
261
+ return offset === raw.length ? markdown.length : mapPosition ? mapPosition(offset) : offset;
262
+ };
263
+ const sections = [];
264
+ for (const { position, headers } of sectionEnds) sections.push({
265
+ end: offsetOf(position),
266
+ headers
267
+ });
268
+ const codes = [];
269
+ for (const { position, language } of codePositions) codes.push({
270
+ start: offsetOf(position),
271
+ language
272
+ });
273
+ codes.sort((a, b) => a.start - b.start);
274
+ const lengthFunction = options.lengthFunction;
275
+ yield* chunkSections(markdown, sections, codes, opts, lengthFunction ? (start, end) => lengthFunction(markdown.slice(start, end)) : (start, end) => end - start);
276
+ }
277
+ function* chunkSections(markdown, sections, codes, opts, measure) {
278
+ const regions = codeRegions(markdown);
279
+ const newlines = [];
280
+ for (let index = markdown.indexOf("\n"); index !== -1; index = markdown.indexOf("\n", index + 1)) newlines.push(index);
281
+ let chunkStart = 0;
282
+ let lastEnd = 0;
283
+ let codeIndex = 0;
284
+ let headers = /* @__PURE__ */ new Map();
285
+ function lineAt(offset) {
286
+ let lower = 0;
287
+ let upper = newlines.length;
288
+ while (lower < upper) {
289
+ const middle = lower + upper >>> 1;
290
+ if (newlines[middle] < offset) lower = middle + 1;
291
+ else upper = middle;
292
+ }
293
+ return lower + 1;
294
+ }
295
+ function codeRegionAt(position) {
296
+ let lower = 0;
297
+ let upper = regions.length;
298
+ while (lower < upper) {
299
+ const middle = lower + upper >>> 1;
300
+ if (regions[middle].start < position) lower = middle + 1;
301
+ else upper = middle;
302
+ }
303
+ const region = lower > 0 ? regions[lower - 1] : void 0;
304
+ return region && position < region.end ? region : void 0;
305
+ }
306
+ function advanceTo(position) {
307
+ const next = contentStart(markdown, position, markdown.length);
308
+ if (next > chunkStart) chunkStart = next;
309
+ }
310
+ function splitPosition(sectionEnd) {
311
+ const idealSplitPos = Math.min(sectionEnd, chunkStart + opts.chunkSize);
312
+ const floor = Math.max(chunkStart, lastEnd);
313
+ let heldRegion;
314
+ for (const separator of SEPARATORS) {
315
+ const index = lastIndexBetween(markdown, separator, Math.max(0, floor - separator.length + 1), idealSplitPos);
316
+ const candidate = index + separator.length;
317
+ if (index < 0 || candidate > sectionEnd || contentEnd(markdown, floor, candidate) <= floor) continue;
318
+ const region = codeRegionAt(candidate);
319
+ if (!region) return candidate;
320
+ heldRegion = region;
321
+ }
322
+ heldRegion ??= codeRegionAt(idealSplitPos);
323
+ if (heldRegion) return Math.min(heldRegion.end, sectionEnd);
324
+ let next = idealSplitPos;
325
+ while (next < sectionEnd) {
326
+ const code = markdown.charCodeAt(next);
327
+ if (code === 32 || code === 10) break;
328
+ next++;
329
+ }
330
+ return next > chunkStart ? next : sectionEnd;
331
+ }
332
+ function* flush(endPosition, applyOverlap) {
333
+ const chunkEnd = Math.min(endPosition, markdown.length);
334
+ const start = chunkStart;
335
+ const end = contentEnd(markdown, start, chunkEnd);
336
+ if (end <= start || end <= lastEnd) {
337
+ advanceTo(chunkEnd);
338
+ return;
339
+ }
340
+ let content = markdown.slice(start, end);
341
+ let contentFrom = start;
342
+ let contentTo = end;
343
+ if (opts.stripHeaders && (content.charCodeAt(0) === 35 || content.includes("\n#"))) {
344
+ const kept = [];
345
+ let firstKept = -1;
346
+ let lastKept = -1;
347
+ let offset = 0;
348
+ for (const line of content.split("\n")) {
349
+ if (line.charCodeAt(0) !== 35 || !MARKDOWN_HEADER_LINE_RE.test(line)) {
350
+ const lineEnd = contentEnd(line, 0, line.length);
351
+ if (lineEnd > 0) {
352
+ if (firstKept === -1) {
353
+ firstKept = kept.length;
354
+ contentFrom = start + offset;
152
355
  }
153
- if (backtickCount % 2 === 1) continue;
154
- }
155
- if (idx >= 0 && candidateSplitPos > lastSplitPosition) {
156
- splitPosition = candidateSplitPos;
157
- break;
356
+ lastKept = kept.length;
357
+ contentTo = start + offset + lineEnd;
158
358
  }
359
+ kept.push(line);
159
360
  }
160
- if (splitPosition === -1 || splitPosition <= lastChunkEndPosition) splitPosition = currentMd.length;
161
- yield* flushChunk(splitPosition, true);
361
+ offset += line.length + 1;
362
+ }
363
+ if (firstKept === -1) {
364
+ lastEnd = end;
365
+ advanceTo(chunkEnd);
366
+ return;
162
367
  }
368
+ const lastLine = kept[lastKept];
369
+ kept[lastKept] = lastLine.slice(0, contentEnd(lastLine, 0, lastLine.length));
370
+ content = kept.slice(firstKept, lastKept + 1).join("\n");
163
371
  }
372
+ const chunk = {
373
+ content,
374
+ metadata: { loc: { lines: {
375
+ from: lineAt(contentFrom),
376
+ to: lineAt(contentTo - 1)
377
+ } } }
378
+ };
379
+ if (headers.size > 0) {
380
+ chunk.metadata.headers = {};
381
+ for (const [tagId, text] of headers.entries()) {
382
+ const level = `h${tagId - 7 + 1}`;
383
+ chunk.metadata.headers[level] = text;
384
+ }
385
+ }
386
+ while (codeIndex < codes.length && codes[codeIndex].start < start) codeIndex++;
387
+ if (codeIndex < codes.length && codes[codeIndex].start < end) chunk.metadata.code = codes[codeIndex].language;
388
+ yield chunk;
389
+ lastEnd = end;
390
+ let next = chunkEnd;
391
+ if (applyOverlap && opts.chunkOverlap > 0) {
392
+ let overlapStart = end - Math.min(opts.chunkOverlap, end - start - 1);
393
+ while (overlapStart > start + 1 && !isOutputWhitespace(markdown.charCodeAt(overlapStart - 1))) overlapStart--;
394
+ const region = codeRegionAt(overlapStart);
395
+ if (region) overlapStart = Math.min(region.end, end);
396
+ if (overlapStart > start + 1) next = overlapStart;
397
+ }
398
+ advanceTo(next);
399
+ }
400
+ for (const section of sections) {
401
+ headers = section.headers;
402
+ if (!opts.returnEachLine) while (measure(chunkStart, section.end) > opts.chunkSize) {
403
+ const previousStart = chunkStart;
404
+ yield* flush(splitPosition(section.end), true);
405
+ if (chunkStart <= previousStart) break;
406
+ }
407
+ yield* flush(section.end, false);
164
408
  }
165
- yield* flushChunk();
166
409
  }
167
410
  function htmlToMarkdownSplitChunks(html, options = {}) {
168
- const opts = createOptions(options);
169
411
  const chunks = [];
170
412
  for (const chunk of htmlToMarkdownSplitChunksStream(html, options)) chunks.push(chunk);
171
- if (opts.returnEachLine && chunks.length > 0) {
413
+ if (options.returnEachLine && chunks.length > 0) {
172
414
  const lineChunks = [];
173
415
  for (const chunk of chunks) {
174
416
  const lines = chunk.content.split("\n");
@@ -0,0 +1,6 @@
1
+ import { MdreamOptions } from "./_chunks/types.mjs";
2
+ /** Convert HTML to readable plain text. */
3
+ export declare function htmlToText(html: string, options?: Partial<MdreamOptions>): string;
4
+ /** Stream HTML as readable plain text. */
5
+ export declare function streamHtmlToText(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
6
+ export type { MdreamOptions };
package/dist/text.mjs ADDED
@@ -0,0 +1,2 @@
1
+ import { htmlToText, streamHtmlToText } from "./_chunks/text.mjs";
2
+ export { htmlToText, streamHtmlToText };
package/package.json CHANGED
@@ -1,13 +1,23 @@
1
1
  {
2
2
  "name": "@mdream/js",
3
3
  "type": "module",
4
- "version": "1.7.2",
5
- "description": "JavaScript HTML-to-Markdown engine for mdream. Escape hatch for hooks and edge runtimes.",
4
+ "version": "2.0.0",
5
+ "description": "Tree-shakable JavaScript converters and plugins for mdream.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
8
8
  "email": "harlan@harlanzw.com",
9
9
  "url": "https://harlanzw.com/"
10
10
  },
11
+ "contributors": [
12
+ {
13
+ "name": "André Cruz",
14
+ "url": "https://github.com/edevil"
15
+ },
16
+ {
17
+ "name": "Bernardo Pereira",
18
+ "url": "https://github.com/BernardoPe"
19
+ }
20
+ ],
11
21
  "license": "MIT",
12
22
  "repository": {
13
23
  "type": "git",
@@ -24,13 +34,29 @@
24
34
  },
25
35
  "default": "./dist/index.mjs"
26
36
  },
27
- "./core": {
28
- "types": "./dist/core.d.mts",
37
+ "./text": {
38
+ "types": "./dist/text.d.mts",
39
+ "import": {
40
+ "types": "./dist/text.d.mts",
41
+ "default": "./dist/text.mjs"
42
+ },
43
+ "default": "./dist/text.mjs"
44
+ },
45
+ "./html": {
46
+ "types": "./dist/html.d.mts",
47
+ "import": {
48
+ "types": "./dist/html.d.mts",
49
+ "default": "./dist/html.mjs"
50
+ },
51
+ "default": "./dist/html.mjs"
52
+ },
53
+ "./clean": {
54
+ "types": "./dist/clean.d.mts",
29
55
  "import": {
30
- "types": "./dist/core.d.mts",
31
- "default": "./dist/core.mjs"
56
+ "types": "./dist/clean.d.mts",
57
+ "default": "./dist/clean.mjs"
32
58
  },
33
- "default": "./dist/core.mjs"
59
+ "default": "./dist/clean.mjs"
34
60
  },
35
61
  "./negotiate": {
36
62
  "types": "./dist/negotiate.d.mts",
@@ -1,9 +0,0 @@
1
- import { MdreamOptions, TransformPlugin } from "./types.mjs";
2
- import "./minimal.mjs";
3
- /**
4
- * Create a plugin with type-safe hook definitions.
5
- * All TransformPlugin fields are optional, so this is a typed identity function.
6
- */
7
- export declare function createPlugin(plugin: TransformPlugin): TransformPlugin;
8
- export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
9
- export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
@@ -1,6 +0,0 @@
1
- import { EngineOptions } from "./types.mjs";
2
- /**
3
- * Creates a configurable minimal preset with advanced options.
4
- * Returns declarative plugin config that works with both JS and Rust engines.
5
- */
6
- export declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
@@ -1,44 +0,0 @@
1
- import { extractionCollectorPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins.mjs";
2
- function resolveFrontmatterOpt(opt) {
3
- if (typeof opt === "function") return {
4
- config: {},
5
- callback: opt
6
- };
7
- if (typeof opt === "object") return {
8
- config: opt,
9
- callback: opt.onExtract
10
- };
11
- return { config: {} };
12
- }
13
- function resolvePlugins(options, hooks) {
14
- const plugins = [];
15
- let callExtractionHandlers;
16
- let getFrontmatter;
17
- let frontmatterCallback;
18
- const config = options.plugins;
19
- if (config) {
20
- if (config.frontmatter) {
21
- const fm = resolveFrontmatterOpt(config.frontmatter);
22
- const fmPlugin = frontmatterPlugin(fm.config);
23
- plugins.push(fmPlugin);
24
- getFrontmatter = fmPlugin.getFrontmatter;
25
- frontmatterCallback = fm.callback;
26
- }
27
- if (config.isolateMain) plugins.push(isolateMainPlugin());
28
- if (config.tailwind) plugins.push(tailwindPlugin());
29
- if (config.filter) plugins.push(filterPlugin(config.filter));
30
- if (config.extraction) {
31
- const collector = extractionCollectorPlugin(config.extraction);
32
- plugins.push(collector.plugin);
33
- callExtractionHandlers = collector.callHandlers;
34
- }
35
- }
36
- if (hooks) plugins.push(...hooks);
37
- return {
38
- plugins,
39
- callExtractionHandlers,
40
- getFrontmatter,
41
- frontmatterCallback
42
- };
43
- }
44
- export { resolvePlugins };