@mdream/js 1.7.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +402 -126
- package/dist/_chunks/clean.mjs +526 -0
- package/dist/_chunks/const.mjs +3 -7
- package/dist/_chunks/extraction.d.mts +14 -0
- package/dist/_chunks/html.mjs +260 -0
- package/dist/_chunks/markdown-processor.mjs +1077 -827
- package/dist/_chunks/minimal.mjs +29 -0
- package/dist/_chunks/output-runner.mjs +54 -0
- package/dist/_chunks/parse.mjs +136 -1490
- package/dist/_chunks/plugin.mjs +14 -0
- package/dist/_chunks/src.mjs +291 -0
- package/dist/_chunks/tag-overrides.mjs +138 -0
- package/dist/_chunks/{plugins.mjs → tailwind.mjs} +145 -292
- package/dist/_chunks/text.mjs +583 -0
- package/dist/_chunks/types.d.mts +136 -122
- package/dist/_chunks/url.mjs +82 -0
- package/dist/_chunks/utils.mjs +659 -0
- package/dist/clean.d.mts +10 -0
- package/dist/clean.mjs +2 -0
- package/dist/cli.mjs +47 -8
- package/dist/html.d.mts +6 -0
- package/dist/html.mjs +2 -0
- package/dist/index.d.mts +5 -4
- package/dist/index.mjs +4 -34
- package/dist/parse.d.mts +15 -10
- package/dist/plugins.d.mts +13 -28
- package/dist/plugins.mjs +52 -2
- package/dist/preset/minimal.d.mts +14 -2
- package/dist/preset/minimal.mjs +1 -27
- package/dist/splitter.d.mts +12 -4
- package/dist/splitter.mjs +355 -113
- package/dist/text.d.mts +6 -0
- package/dist/text.mjs +2 -0
- package/package.json +33 -7
- package/dist/_chunks/index.d.mts +0 -9
- package/dist/_chunks/minimal.d.mts +0 -6
- package/dist/_chunks/resolve-plugins.mjs +0 -44
- package/dist/_chunks/stream.mjs +0 -37
- package/dist/core.d.mts +0 -15
- package/dist/core.mjs +0 -28
package/dist/splitter.mjs
CHANGED
|
@@ -1,109 +1,218 @@
|
|
|
1
1
|
import "./_chunks/const.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import { createMarkdownProcessor,
|
|
4
|
-
import {
|
|
2
|
+
import { trimOutputStart } from "./_chunks/utils.mjs";
|
|
3
|
+
import { createMarkdownProcessor, tagHandlers, trimAsciiWhitespaceEnd } from "./_chunks/markdown-processor.mjs";
|
|
4
|
+
import { assertEngineOptions, buildTagOverrideHandlers, checkClean, endPlugins, processPluginsForEvent } from "./_chunks/tag-overrides.mjs";
|
|
5
|
+
import { finalizeParse, parseHtmlStream } from "./_chunks/parse.mjs";
|
|
6
|
+
import { resolvePlugins } from "./_chunks/plugin.mjs";
|
|
7
|
+
function createOutputPositions(buffer) {
|
|
8
|
+
const positions = [];
|
|
9
|
+
return {
|
|
10
|
+
capture(fragment = Math.max(0, buffer.length - 1), offset = buffer[fragment]?.length ?? 0, kind = "boundary") {
|
|
11
|
+
const position = {
|
|
12
|
+
fragment,
|
|
13
|
+
offset,
|
|
14
|
+
kind
|
|
15
|
+
};
|
|
16
|
+
positions.push(position);
|
|
17
|
+
return position;
|
|
18
|
+
},
|
|
19
|
+
replace(start, end, replacement, mapOffset) {
|
|
20
|
+
if (positions.length === 0) return;
|
|
21
|
+
const fragmentDelta = replacement.length - (end - start);
|
|
22
|
+
let lower = 0;
|
|
23
|
+
let upper = positions.length;
|
|
24
|
+
while (lower < upper) {
|
|
25
|
+
const middle = lower + upper >>> 1;
|
|
26
|
+
if (positions[middle].fragment < start) lower = middle + 1;
|
|
27
|
+
else upper = middle;
|
|
28
|
+
}
|
|
29
|
+
let oldPrefix;
|
|
30
|
+
let newPrefix;
|
|
31
|
+
for (let index = lower; index < positions.length; index++) {
|
|
32
|
+
const position = positions[index];
|
|
33
|
+
if (position.fragment >= end) {
|
|
34
|
+
if (fragmentDelta === 0) break;
|
|
35
|
+
position.fragment += fragmentDelta;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
if (!oldPrefix) {
|
|
39
|
+
oldPrefix = [0];
|
|
40
|
+
for (let index = start; index < end; index++) oldPrefix.push(oldPrefix.at(-1) + buffer[index].length);
|
|
41
|
+
newPrefix = [0];
|
|
42
|
+
for (const fragment of replacement) newPrefix.push(newPrefix.at(-1) + fragment.length);
|
|
43
|
+
}
|
|
44
|
+
const oldOffset = oldPrefix[position.fragment - start] + position.offset;
|
|
45
|
+
const endOffset = newPrefix.at(-1);
|
|
46
|
+
const offset = Math.max(0, Math.min(mapOffset ? mapOffset(oldOffset, position.kind) : oldOffset, endOffset));
|
|
47
|
+
if (replacement.length === 0 || position.kind === "boundary" && offset === 0 && start > 0) {
|
|
48
|
+
position.fragment = Math.max(0, start - 1);
|
|
49
|
+
position.offset = start > 0 ? buffer[start - 1].length : 0;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
let fragment = 0;
|
|
53
|
+
while (fragment + 1 < replacement.length && (newPrefix[fragment + 1] < offset || position.kind === "content" && newPrefix[fragment + 1] === offset)) fragment++;
|
|
54
|
+
position.fragment = start + fragment;
|
|
55
|
+
position.offset = offset - newPrefix[fragment];
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
finish() {
|
|
59
|
+
const prefix = [0];
|
|
60
|
+
for (const fragment of buffer) prefix.push(prefix.at(-1) + fragment.length);
|
|
61
|
+
return (position) => prefix[Math.min(position.fragment, buffer.length)] + Math.min(position.offset, buffer[position.fragment]?.length ?? 0);
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
}
|
|
5
65
|
const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
];
|
|
66
|
+
const DEFAULT_SPLIT_HEADINGS = 62;
|
|
67
|
+
const HEADING_LEVELS_ERROR = "headersToSplitOn takes heading levels from 1 to 6, such as [2, 3]. @mdream/js no longer exports the TAG_H* constants.";
|
|
68
|
+
function headingSplitMask(levels) {
|
|
69
|
+
if (levels == null) return DEFAULT_SPLIT_HEADINGS;
|
|
70
|
+
if (!Array.isArray(levels)) throw new TypeError(HEADING_LEVELS_ERROR);
|
|
71
|
+
let mask = 0;
|
|
72
|
+
for (let index = 0; index < levels.length; index++) {
|
|
73
|
+
const level = levels[index];
|
|
74
|
+
if (!Number.isInteger(level) || level < 1 || level > 6) throw new TypeError(HEADING_LEVELS_ERROR);
|
|
75
|
+
mask |= 1 << level - 1;
|
|
76
|
+
}
|
|
77
|
+
return mask;
|
|
78
|
+
}
|
|
14
79
|
function createOptions(options) {
|
|
15
80
|
return {
|
|
16
|
-
|
|
81
|
+
splitHeadings: headingSplitMask(options.headersToSplitOn),
|
|
17
82
|
returnEachLine: options.returnEachLine ?? false,
|
|
18
83
|
stripHeaders: options.stripHeaders ?? true,
|
|
19
84
|
chunkSize: options.chunkSize ?? 1e3,
|
|
20
85
|
chunkOverlap: options.chunkOverlap ?? 200,
|
|
21
86
|
lengthFunction: options.lengthFunction ?? ((text) => text.length),
|
|
22
87
|
keepSeparator: options.keepSeparator ?? false,
|
|
23
|
-
resolvedPlugins: resolvePlugins(options
|
|
88
|
+
resolvedPlugins: resolvePlugins(options.plugins),
|
|
89
|
+
tagOverrideHandlers: options.tagOverrides ? buildTagOverrideHandlers(options.tagOverrides, tagHandlers) : void 0
|
|
24
90
|
};
|
|
25
91
|
}
|
|
26
92
|
function shouldSplitOnHeader(tagId, options) {
|
|
27
|
-
return options.
|
|
93
|
+
return (options.splitHeadings & 1 << tagId - 7) !== 0;
|
|
94
|
+
}
|
|
95
|
+
function isOutputWhitespace(code) {
|
|
96
|
+
return code === 32 || code >= 9 && code <= 13;
|
|
97
|
+
}
|
|
98
|
+
const SEPARATORS = [
|
|
99
|
+
"\n\n",
|
|
100
|
+
"```\n",
|
|
101
|
+
"\n",
|
|
102
|
+
" "
|
|
103
|
+
];
|
|
104
|
+
function contentStart(markdown, start, end) {
|
|
105
|
+
let lineStart = start === 0 || markdown.charCodeAt(start - 1) === 10 ? start : -1;
|
|
106
|
+
for (let index = start; index < end; index++) {
|
|
107
|
+
const code = markdown.charCodeAt(index);
|
|
108
|
+
if (code === 10) lineStart = index + 1;
|
|
109
|
+
else if (!isOutputWhitespace(code)) return lineStart === -1 ? index : lineStart;
|
|
110
|
+
}
|
|
111
|
+
return end;
|
|
112
|
+
}
|
|
113
|
+
function contentEnd(markdown, start, end) {
|
|
114
|
+
while (end > start && isOutputWhitespace(markdown.charCodeAt(end - 1))) end--;
|
|
115
|
+
return end;
|
|
28
116
|
}
|
|
29
|
-
function
|
|
30
|
-
|
|
117
|
+
function lastIndexBetween(markdown, separator, floor, from) {
|
|
118
|
+
if (from < floor) return -1;
|
|
119
|
+
const index = markdown.slice(floor, from + separator.length).lastIndexOf(separator);
|
|
120
|
+
return index === -1 ? -1 : floor + index;
|
|
121
|
+
}
|
|
122
|
+
function codeRegions(markdown) {
|
|
123
|
+
const regions = [];
|
|
124
|
+
let open;
|
|
125
|
+
let lineStart = 0;
|
|
126
|
+
while (lineStart < markdown.length) {
|
|
127
|
+
const newline = markdown.indexOf("\n", lineStart);
|
|
128
|
+
const lineEnd = newline === -1 ? markdown.length : newline;
|
|
129
|
+
let markerStart = lineStart;
|
|
130
|
+
let prefixStart = lineStart;
|
|
131
|
+
let prefix = "";
|
|
132
|
+
while (true) {
|
|
133
|
+
while (markdown.charCodeAt(markerStart) === 32 || markdown.charCodeAt(markerStart) === 62) markerStart++;
|
|
134
|
+
let listEnd = markerStart;
|
|
135
|
+
const first = markdown.charCodeAt(markerStart);
|
|
136
|
+
if ((first === 45 || first === 43 || first === 42) && markdown.charCodeAt(markerStart + 1) === 32) listEnd++;
|
|
137
|
+
else if (first >= 48 && first <= 57) {
|
|
138
|
+
let end = markerStart + 1;
|
|
139
|
+
while (markdown.charCodeAt(end) >= 48 && markdown.charCodeAt(end) <= 57) end++;
|
|
140
|
+
const delimiter = markdown.charCodeAt(end);
|
|
141
|
+
if ((delimiter === 46 || delimiter === 41) && markdown.charCodeAt(end + 1) === 32) listEnd = end + 1;
|
|
142
|
+
}
|
|
143
|
+
if (listEnd === markerStart) break;
|
|
144
|
+
prefix += markdown.slice(prefixStart, markerStart) + " ".repeat(listEnd - markerStart);
|
|
145
|
+
markerStart = listEnd;
|
|
146
|
+
prefixStart = listEnd;
|
|
147
|
+
}
|
|
148
|
+
prefix += markdown.slice(prefixStart, markerStart);
|
|
149
|
+
const marker = markdown.charCodeAt(markerStart);
|
|
150
|
+
if (marker === 96 || marker === 126) {
|
|
151
|
+
let markerEnd = markerStart + 1;
|
|
152
|
+
while (markdown.charCodeAt(markerEnd) === marker) markerEnd++;
|
|
153
|
+
const run = markerEnd - markerStart;
|
|
154
|
+
if (!open && run >= 3) open = {
|
|
155
|
+
start: lineStart,
|
|
156
|
+
markerStart,
|
|
157
|
+
marker,
|
|
158
|
+
run,
|
|
159
|
+
prefix
|
|
160
|
+
};
|
|
161
|
+
else if (open && marker === open.marker && run >= open.run && prefix === open.prefix) {
|
|
162
|
+
let tail = markerEnd;
|
|
163
|
+
while (tail < lineEnd && isOutputWhitespace(markdown.charCodeAt(tail))) tail++;
|
|
164
|
+
if (tail === lineEnd) {
|
|
165
|
+
regions.push({
|
|
166
|
+
start: open.start,
|
|
167
|
+
end: lineEnd,
|
|
168
|
+
markerStart: open.markerStart
|
|
169
|
+
});
|
|
170
|
+
open = void 0;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
lineStart = lineEnd + 1;
|
|
175
|
+
}
|
|
176
|
+
if (open) regions.push({
|
|
177
|
+
start: open.start,
|
|
178
|
+
end: markdown.length,
|
|
179
|
+
markerStart: open.markerStart
|
|
180
|
+
});
|
|
181
|
+
return regions;
|
|
31
182
|
}
|
|
32
183
|
function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
184
|
+
assertEngineOptions(options, "splitter");
|
|
185
|
+
checkClean(options);
|
|
33
186
|
const opts = createOptions(options);
|
|
34
187
|
if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
|
|
35
|
-
|
|
36
|
-
const
|
|
188
|
+
const processor = createMarkdownProcessor(options);
|
|
189
|
+
const positions = createOutputPositions(processor.state.buffer);
|
|
190
|
+
processor.state.outputPositions = positions;
|
|
191
|
+
const codePositions = [];
|
|
37
192
|
processor.state.onCodeFenceOpen = (language) => {
|
|
38
|
-
if (language
|
|
193
|
+
if (language) {
|
|
194
|
+
const fence = processor.state.codeFence;
|
|
195
|
+
codePositions.push({
|
|
196
|
+
position: positions.capture(fence.fragment, fence.markerOffset, "content"),
|
|
197
|
+
language
|
|
198
|
+
});
|
|
199
|
+
}
|
|
39
200
|
};
|
|
40
201
|
const headerHierarchy = /* @__PURE__ */ new Map();
|
|
41
202
|
const seenSplitHeaders = /* @__PURE__ */ new Set();
|
|
42
203
|
let collectingHeaderText = false;
|
|
43
204
|
let currentHeaderTagId = null;
|
|
44
205
|
let currentHeaderText = "";
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const originalChunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
|
|
52
|
-
if (!originalChunkContent.trim()) {
|
|
53
|
-
lastChunkEndPosition = chunkEnd;
|
|
54
|
-
return;
|
|
55
|
-
}
|
|
56
|
-
let chunkContent = originalChunkContent;
|
|
57
|
-
if (opts.stripHeaders) {
|
|
58
|
-
chunkContent = chunkContent.split("\n").filter((line) => !MARKDOWN_HEADER_LINE_RE.test(line)).join("\n").trim();
|
|
59
|
-
if (!chunkContent) {
|
|
60
|
-
lastChunkEndPosition = chunkEnd;
|
|
61
|
-
return;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
const chunk = {
|
|
65
|
-
content: chunkContent.trimEnd(),
|
|
66
|
-
metadata: { loc: { lines: {
|
|
67
|
-
from: lineNumber,
|
|
68
|
-
to: lineNumber + (originalChunkContent.match(NEWLINE_RE) || []).length
|
|
69
|
-
} } }
|
|
70
|
-
};
|
|
71
|
-
if (headerHierarchy.size > 0) {
|
|
72
|
-
chunk.metadata.headers = {};
|
|
73
|
-
for (const [tagId, text] of headerHierarchy.entries()) {
|
|
74
|
-
const level = `h${tagId - 7 + 1}`;
|
|
75
|
-
chunk.metadata.headers[level] = text;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
if (currentChunkCodeLanguage) chunk.metadata.code = currentChunkCodeLanguage;
|
|
79
|
-
yield chunk;
|
|
80
|
-
currentChunkCodeLanguage = "";
|
|
81
|
-
lastSplitPosition = chunkEnd;
|
|
82
|
-
if (applyOverlap && opts.chunkOverlap > 0) {
|
|
83
|
-
const maxOverlap = Math.max(0, originalChunkContent.length - 1);
|
|
84
|
-
lastChunkEndPosition = chunkEnd - Math.min(opts.chunkOverlap, maxOverlap);
|
|
85
|
-
} else lastChunkEndPosition = chunkEnd;
|
|
86
|
-
lineNumber += (originalChunkContent.match(NEWLINE_RE) || []).length;
|
|
87
|
-
}
|
|
88
|
-
const parseState = {
|
|
89
|
-
depthMap: processor.state.depthMap,
|
|
90
|
-
depth: 0,
|
|
91
|
-
resolvedPlugins: opts.resolvedPlugins
|
|
206
|
+
const sectionEnds = [];
|
|
207
|
+
const endSection = () => {
|
|
208
|
+
sectionEnds.push({
|
|
209
|
+
position: positions.capture(),
|
|
210
|
+
headers: new Map(headerHierarchy)
|
|
211
|
+
});
|
|
92
212
|
};
|
|
93
|
-
const eventBuffer = [];
|
|
94
213
|
const processResolvedEvent = opts.resolvedPlugins?.length ? (event) => processPluginsForEvent(event, opts.resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
|
|
95
|
-
|
|
96
|
-
eventBuffer.push(event);
|
|
97
|
-
});
|
|
98
|
-
for (const event of eventBuffer) {
|
|
214
|
+
const handleEvent = (event) => {
|
|
99
215
|
const { type: eventType, node } = event;
|
|
100
|
-
const eventDepthMap = node.type === 1 ? node.depthMap : node.parent?.depthMap;
|
|
101
|
-
if (eventDepthMap) processor.state.depthMap.set(eventDepthMap);
|
|
102
|
-
else processor.state.depthMap.fill(0);
|
|
103
|
-
if (eventType === 1 && node.type === 1) {
|
|
104
|
-
const tagId = node.tagId;
|
|
105
|
-
if (tagId !== void 0 && tagId >= 0 && tagId < processor.state.depthMap.length) processor.state.depthMap[tagId] = Math.max(0, processor.state.depthMap[tagId] - 1);
|
|
106
|
-
}
|
|
107
216
|
if (node.type === 1) {
|
|
108
217
|
const tagId = node.tagId;
|
|
109
218
|
if (tagId && tagId >= 7 && tagId <= 12) {
|
|
@@ -113,7 +222,7 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
|
113
222
|
currentHeaderText = "";
|
|
114
223
|
if (shouldSplitOnHeader(tagId, opts)) {
|
|
115
224
|
if (seenSplitHeaders.has(tagId)) {
|
|
116
|
-
|
|
225
|
+
endSection();
|
|
117
226
|
for (let i = tagId; i <= 12; i++) headerHierarchy.delete(i);
|
|
118
227
|
}
|
|
119
228
|
seenSplitHeaders.add(tagId);
|
|
@@ -124,51 +233,184 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
|
124
233
|
currentHeaderTagId = null;
|
|
125
234
|
}
|
|
126
235
|
}
|
|
127
|
-
if (tagId === 13 && eventType === 0)
|
|
128
|
-
}
|
|
129
|
-
if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
|
|
236
|
+
if (tagId === 13 && eventType === 0) endSection();
|
|
237
|
+
} else if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
|
|
130
238
|
processResolvedEvent(event);
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
239
|
+
};
|
|
240
|
+
const parseState = {
|
|
241
|
+
depthMap: processor.state.depthMap,
|
|
242
|
+
depth: 0,
|
|
243
|
+
resolvedPlugins: opts.resolvedPlugins,
|
|
244
|
+
tagHandlers,
|
|
245
|
+
tagOverrideHandlers: opts.tagOverrideHandlers
|
|
246
|
+
};
|
|
247
|
+
const leftover = parseHtmlStream(html, parseState, handleEvent);
|
|
248
|
+
finalizeParse(leftover, parseState, handleEvent);
|
|
249
|
+
endPlugins(opts.resolvedPlugins, processor.state);
|
|
250
|
+
endSection();
|
|
251
|
+
const positionOffset = positions.finish();
|
|
252
|
+
const joined = processor.state.buffer.join("");
|
|
253
|
+
const started = trimOutputStart(joined);
|
|
254
|
+
const leadingTrim = joined.length - started.length;
|
|
255
|
+
const raw = trimAsciiWhitespaceEnd(started);
|
|
256
|
+
const view = processor.finishOutput?.(raw);
|
|
257
|
+
const markdown = view ? view.markdown : raw;
|
|
258
|
+
const mapPosition = view ? view.mapPosition ?? ((position) => processor.finishOutput(raw.slice(0, position)).markdown.length) : void 0;
|
|
259
|
+
const offsetOf = (position) => {
|
|
260
|
+
const offset = Math.min(Math.max(0, positionOffset(position) - leadingTrim), raw.length);
|
|
261
|
+
return offset === raw.length ? markdown.length : mapPosition ? mapPosition(offset) : offset;
|
|
262
|
+
};
|
|
263
|
+
const sections = [];
|
|
264
|
+
for (const { position, headers } of sectionEnds) sections.push({
|
|
265
|
+
end: offsetOf(position),
|
|
266
|
+
headers
|
|
267
|
+
});
|
|
268
|
+
const codes = [];
|
|
269
|
+
for (const { position, language } of codePositions) codes.push({
|
|
270
|
+
start: offsetOf(position),
|
|
271
|
+
language
|
|
272
|
+
});
|
|
273
|
+
codes.sort((a, b) => a.start - b.start);
|
|
274
|
+
const lengthFunction = options.lengthFunction;
|
|
275
|
+
yield* chunkSections(markdown, sections, codes, opts, lengthFunction ? (start, end) => lengthFunction(markdown.slice(start, end)) : (start, end) => end - start);
|
|
276
|
+
}
|
|
277
|
+
function* chunkSections(markdown, sections, codes, opts, measure) {
|
|
278
|
+
const regions = codeRegions(markdown);
|
|
279
|
+
const newlines = [];
|
|
280
|
+
for (let index = markdown.indexOf("\n"); index !== -1; index = markdown.indexOf("\n", index + 1)) newlines.push(index);
|
|
281
|
+
let chunkStart = 0;
|
|
282
|
+
let lastEnd = 0;
|
|
283
|
+
let codeIndex = 0;
|
|
284
|
+
let headers = /* @__PURE__ */ new Map();
|
|
285
|
+
function lineAt(offset) {
|
|
286
|
+
let lower = 0;
|
|
287
|
+
let upper = newlines.length;
|
|
288
|
+
while (lower < upper) {
|
|
289
|
+
const middle = lower + upper >>> 1;
|
|
290
|
+
if (newlines[middle] < offset) lower = middle + 1;
|
|
291
|
+
else upper = middle;
|
|
292
|
+
}
|
|
293
|
+
return lower + 1;
|
|
294
|
+
}
|
|
295
|
+
function codeRegionAt(position) {
|
|
296
|
+
let lower = 0;
|
|
297
|
+
let upper = regions.length;
|
|
298
|
+
while (lower < upper) {
|
|
299
|
+
const middle = lower + upper >>> 1;
|
|
300
|
+
if (regions[middle].start < position) lower = middle + 1;
|
|
301
|
+
else upper = middle;
|
|
302
|
+
}
|
|
303
|
+
const region = lower > 0 ? regions[lower - 1] : void 0;
|
|
304
|
+
return region && position < region.end ? region : void 0;
|
|
305
|
+
}
|
|
306
|
+
function advanceTo(position) {
|
|
307
|
+
const next = contentStart(markdown, position, markdown.length);
|
|
308
|
+
if (next > chunkStart) chunkStart = next;
|
|
309
|
+
}
|
|
310
|
+
function splitPosition(sectionEnd) {
|
|
311
|
+
const idealSplitPos = Math.min(sectionEnd, chunkStart + opts.chunkSize);
|
|
312
|
+
const floor = Math.max(chunkStart, lastEnd);
|
|
313
|
+
let heldRegion;
|
|
314
|
+
for (const separator of SEPARATORS) {
|
|
315
|
+
const index = lastIndexBetween(markdown, separator, Math.max(0, floor - separator.length + 1), idealSplitPos);
|
|
316
|
+
const candidate = index + separator.length;
|
|
317
|
+
if (index < 0 || candidate > sectionEnd || contentEnd(markdown, floor, candidate) <= floor) continue;
|
|
318
|
+
const region = codeRegionAt(candidate);
|
|
319
|
+
if (!region) return candidate;
|
|
320
|
+
heldRegion = region;
|
|
321
|
+
}
|
|
322
|
+
heldRegion ??= codeRegionAt(idealSplitPos);
|
|
323
|
+
if (heldRegion) return Math.min(heldRegion.end, sectionEnd);
|
|
324
|
+
let next = idealSplitPos;
|
|
325
|
+
while (next < sectionEnd) {
|
|
326
|
+
const code = markdown.charCodeAt(next);
|
|
327
|
+
if (code === 32 || code === 10) break;
|
|
328
|
+
next++;
|
|
329
|
+
}
|
|
330
|
+
return next > chunkStart ? next : sectionEnd;
|
|
331
|
+
}
|
|
332
|
+
function* flush(endPosition, applyOverlap) {
|
|
333
|
+
const chunkEnd = Math.min(endPosition, markdown.length);
|
|
334
|
+
const start = chunkStart;
|
|
335
|
+
const end = contentEnd(markdown, start, chunkEnd);
|
|
336
|
+
if (end <= start || end <= lastEnd) {
|
|
337
|
+
advanceTo(chunkEnd);
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
let content = markdown.slice(start, end);
|
|
341
|
+
let contentFrom = start;
|
|
342
|
+
let contentTo = end;
|
|
343
|
+
if (opts.stripHeaders && (content.charCodeAt(0) === 35 || content.includes("\n#"))) {
|
|
344
|
+
const kept = [];
|
|
345
|
+
let firstKept = -1;
|
|
346
|
+
let lastKept = -1;
|
|
347
|
+
let offset = 0;
|
|
348
|
+
for (const line of content.split("\n")) {
|
|
349
|
+
if (line.charCodeAt(0) !== 35 || !MARKDOWN_HEADER_LINE_RE.test(line)) {
|
|
350
|
+
const lineEnd = contentEnd(line, 0, line.length);
|
|
351
|
+
if (lineEnd > 0) {
|
|
352
|
+
if (firstKept === -1) {
|
|
353
|
+
firstKept = kept.length;
|
|
354
|
+
contentFrom = start + offset;
|
|
152
355
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
if (idx >= 0 && candidateSplitPos > lastSplitPosition) {
|
|
156
|
-
splitPosition = candidateSplitPos;
|
|
157
|
-
break;
|
|
356
|
+
lastKept = kept.length;
|
|
357
|
+
contentTo = start + offset + lineEnd;
|
|
158
358
|
}
|
|
359
|
+
kept.push(line);
|
|
159
360
|
}
|
|
160
|
-
|
|
161
|
-
|
|
361
|
+
offset += line.length + 1;
|
|
362
|
+
}
|
|
363
|
+
if (firstKept === -1) {
|
|
364
|
+
lastEnd = end;
|
|
365
|
+
advanceTo(chunkEnd);
|
|
366
|
+
return;
|
|
162
367
|
}
|
|
368
|
+
const lastLine = kept[lastKept];
|
|
369
|
+
kept[lastKept] = lastLine.slice(0, contentEnd(lastLine, 0, lastLine.length));
|
|
370
|
+
content = kept.slice(firstKept, lastKept + 1).join("\n");
|
|
163
371
|
}
|
|
372
|
+
const chunk = {
|
|
373
|
+
content,
|
|
374
|
+
metadata: { loc: { lines: {
|
|
375
|
+
from: lineAt(contentFrom),
|
|
376
|
+
to: lineAt(contentTo - 1)
|
|
377
|
+
} } }
|
|
378
|
+
};
|
|
379
|
+
if (headers.size > 0) {
|
|
380
|
+
chunk.metadata.headers = {};
|
|
381
|
+
for (const [tagId, text] of headers.entries()) {
|
|
382
|
+
const level = `h${tagId - 7 + 1}`;
|
|
383
|
+
chunk.metadata.headers[level] = text;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
while (codeIndex < codes.length && codes[codeIndex].start < start) codeIndex++;
|
|
387
|
+
if (codeIndex < codes.length && codes[codeIndex].start < end) chunk.metadata.code = codes[codeIndex].language;
|
|
388
|
+
yield chunk;
|
|
389
|
+
lastEnd = end;
|
|
390
|
+
let next = chunkEnd;
|
|
391
|
+
if (applyOverlap && opts.chunkOverlap > 0) {
|
|
392
|
+
let overlapStart = end - Math.min(opts.chunkOverlap, end - start - 1);
|
|
393
|
+
while (overlapStart > start + 1 && !isOutputWhitespace(markdown.charCodeAt(overlapStart - 1))) overlapStart--;
|
|
394
|
+
const region = codeRegionAt(overlapStart);
|
|
395
|
+
if (region) overlapStart = Math.min(region.end, end);
|
|
396
|
+
if (overlapStart > start + 1) next = overlapStart;
|
|
397
|
+
}
|
|
398
|
+
advanceTo(next);
|
|
399
|
+
}
|
|
400
|
+
for (const section of sections) {
|
|
401
|
+
headers = section.headers;
|
|
402
|
+
if (!opts.returnEachLine) while (measure(chunkStart, section.end) > opts.chunkSize) {
|
|
403
|
+
const previousStart = chunkStart;
|
|
404
|
+
yield* flush(splitPosition(section.end), true);
|
|
405
|
+
if (chunkStart <= previousStart) break;
|
|
406
|
+
}
|
|
407
|
+
yield* flush(section.end, false);
|
|
164
408
|
}
|
|
165
|
-
yield* flushChunk();
|
|
166
409
|
}
|
|
167
410
|
function htmlToMarkdownSplitChunks(html, options = {}) {
|
|
168
|
-
const opts = createOptions(options);
|
|
169
411
|
const chunks = [];
|
|
170
412
|
for (const chunk of htmlToMarkdownSplitChunksStream(html, options)) chunks.push(chunk);
|
|
171
|
-
if (
|
|
413
|
+
if (options.returnEachLine && chunks.length > 0) {
|
|
172
414
|
const lineChunks = [];
|
|
173
415
|
for (const chunk of chunks) {
|
|
174
416
|
const lines = chunk.content.split("\n");
|
package/dist/text.d.mts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { MdreamOptions } from "./_chunks/types.mjs";
|
|
2
|
+
/** Convert HTML to readable plain text. */
|
|
3
|
+
export declare function htmlToText(html: string, options?: Partial<MdreamOptions>): string;
|
|
4
|
+
/** Stream HTML as readable plain text. */
|
|
5
|
+
export declare function streamHtmlToText(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
6
|
+
export type { MdreamOptions };
|
package/dist/text.mjs
ADDED
package/package.json
CHANGED
|
@@ -1,13 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/js",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "
|
|
5
|
-
"description": "
|
|
4
|
+
"version": "2.0.0",
|
|
5
|
+
"description": "Tree-shakable JavaScript converters and plugins for mdream.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
8
8
|
"email": "harlan@harlanzw.com",
|
|
9
9
|
"url": "https://harlanzw.com/"
|
|
10
10
|
},
|
|
11
|
+
"contributors": [
|
|
12
|
+
{
|
|
13
|
+
"name": "André Cruz",
|
|
14
|
+
"url": "https://github.com/edevil"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"name": "Bernardo Pereira",
|
|
18
|
+
"url": "https://github.com/BernardoPe"
|
|
19
|
+
}
|
|
20
|
+
],
|
|
11
21
|
"license": "MIT",
|
|
12
22
|
"repository": {
|
|
13
23
|
"type": "git",
|
|
@@ -24,13 +34,29 @@
|
|
|
24
34
|
},
|
|
25
35
|
"default": "./dist/index.mjs"
|
|
26
36
|
},
|
|
27
|
-
"./
|
|
28
|
-
"types": "./dist/
|
|
37
|
+
"./text": {
|
|
38
|
+
"types": "./dist/text.d.mts",
|
|
39
|
+
"import": {
|
|
40
|
+
"types": "./dist/text.d.mts",
|
|
41
|
+
"default": "./dist/text.mjs"
|
|
42
|
+
},
|
|
43
|
+
"default": "./dist/text.mjs"
|
|
44
|
+
},
|
|
45
|
+
"./html": {
|
|
46
|
+
"types": "./dist/html.d.mts",
|
|
47
|
+
"import": {
|
|
48
|
+
"types": "./dist/html.d.mts",
|
|
49
|
+
"default": "./dist/html.mjs"
|
|
50
|
+
},
|
|
51
|
+
"default": "./dist/html.mjs"
|
|
52
|
+
},
|
|
53
|
+
"./clean": {
|
|
54
|
+
"types": "./dist/clean.d.mts",
|
|
29
55
|
"import": {
|
|
30
|
-
"types": "./dist/
|
|
31
|
-
"default": "./dist/
|
|
56
|
+
"types": "./dist/clean.d.mts",
|
|
57
|
+
"default": "./dist/clean.mjs"
|
|
32
58
|
},
|
|
33
|
-
"default": "./dist/
|
|
59
|
+
"default": "./dist/clean.mjs"
|
|
34
60
|
},
|
|
35
61
|
"./negotiate": {
|
|
36
62
|
"types": "./dist/negotiate.d.mts",
|
package/dist/_chunks/index.d.mts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { MdreamOptions, TransformPlugin } from "./types.mjs";
|
|
2
|
-
import "./minimal.mjs";
|
|
3
|
-
/**
|
|
4
|
-
* Create a plugin with type-safe hook definitions.
|
|
5
|
-
* All TransformPlugin fields are optional, so this is a typed identity function.
|
|
6
|
-
*/
|
|
7
|
-
export declare function createPlugin(plugin: TransformPlugin): TransformPlugin;
|
|
8
|
-
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
9
|
-
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import { EngineOptions } from "./types.mjs";
|
|
2
|
-
/**
|
|
3
|
-
* Creates a configurable minimal preset with advanced options.
|
|
4
|
-
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
5
|
-
*/
|
|
6
|
-
export declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
import { extractionCollectorPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins.mjs";
|
|
2
|
-
function resolveFrontmatterOpt(opt) {
|
|
3
|
-
if (typeof opt === "function") return {
|
|
4
|
-
config: {},
|
|
5
|
-
callback: opt
|
|
6
|
-
};
|
|
7
|
-
if (typeof opt === "object") return {
|
|
8
|
-
config: opt,
|
|
9
|
-
callback: opt.onExtract
|
|
10
|
-
};
|
|
11
|
-
return { config: {} };
|
|
12
|
-
}
|
|
13
|
-
function resolvePlugins(options, hooks) {
|
|
14
|
-
const plugins = [];
|
|
15
|
-
let callExtractionHandlers;
|
|
16
|
-
let getFrontmatter;
|
|
17
|
-
let frontmatterCallback;
|
|
18
|
-
const config = options.plugins;
|
|
19
|
-
if (config) {
|
|
20
|
-
if (config.frontmatter) {
|
|
21
|
-
const fm = resolveFrontmatterOpt(config.frontmatter);
|
|
22
|
-
const fmPlugin = frontmatterPlugin(fm.config);
|
|
23
|
-
plugins.push(fmPlugin);
|
|
24
|
-
getFrontmatter = fmPlugin.getFrontmatter;
|
|
25
|
-
frontmatterCallback = fm.callback;
|
|
26
|
-
}
|
|
27
|
-
if (config.isolateMain) plugins.push(isolateMainPlugin());
|
|
28
|
-
if (config.tailwind) plugins.push(tailwindPlugin());
|
|
29
|
-
if (config.filter) plugins.push(filterPlugin(config.filter));
|
|
30
|
-
if (config.extraction) {
|
|
31
|
-
const collector = extractionCollectorPlugin(config.extraction);
|
|
32
|
-
plugins.push(collector.plugin);
|
|
33
|
-
callExtractionHandlers = collector.callHandlers;
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
if (hooks) plugins.push(...hooks);
|
|
37
|
-
return {
|
|
38
|
-
plugins,
|
|
39
|
-
callExtractionHandlers,
|
|
40
|
-
getFrontmatter,
|
|
41
|
-
frontmatterCallback
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
export { resolvePlugins };
|