@mdream/js 2.0.0-beta.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +402 -126
- package/dist/_chunks/clean.mjs +526 -0
- package/dist/_chunks/const.mjs +3 -7
- package/dist/_chunks/extraction.d.mts +14 -0
- package/dist/_chunks/html.mjs +260 -0
- package/dist/_chunks/markdown-processor.mjs +1077 -828
- package/dist/_chunks/minimal.mjs +29 -0
- package/dist/_chunks/output-runner.mjs +54 -0
- package/dist/_chunks/parse.mjs +134 -1501
- package/dist/_chunks/plugin.mjs +14 -0
- package/dist/_chunks/src.mjs +291 -0
- package/dist/_chunks/tag-overrides.mjs +138 -0
- package/dist/_chunks/{plugins.mjs → tailwind.mjs} +145 -292
- package/dist/_chunks/text.mjs +583 -0
- package/dist/_chunks/types.d.mts +136 -126
- package/dist/_chunks/url.mjs +82 -0
- package/dist/_chunks/utils.mjs +659 -0
- package/dist/clean.d.mts +10 -0
- package/dist/clean.mjs +2 -0
- package/dist/cli.mjs +47 -8
- package/dist/html.d.mts +6 -0
- package/dist/html.mjs +2 -0
- package/dist/index.d.mts +5 -4
- package/dist/index.mjs +4 -58
- package/dist/parse.d.mts +15 -10
- package/dist/plugins.d.mts +13 -28
- package/dist/plugins.mjs +52 -2
- package/dist/preset/minimal.d.mts +14 -2
- package/dist/preset/minimal.mjs +1 -32
- package/dist/splitter.d.mts +12 -4
- package/dist/splitter.mjs +386 -113
- package/dist/text.d.mts +6 -0
- package/dist/text.mjs +2 -0
- package/package.json +34 -8
- package/dist/_chunks/index.d.mts +0 -9
- package/dist/_chunks/minimal.d.mts +0 -6
- package/dist/_chunks/resolve-plugins.mjs +0 -44
- package/dist/_chunks/stream.mjs +0 -37
- package/dist/core.d.mts +0 -15
- package/dist/core.mjs +0 -28
package/dist/splitter.mjs
CHANGED
|
@@ -1,109 +1,240 @@
|
|
|
1
1
|
import "./_chunks/const.mjs";
|
|
2
|
-
import {
|
|
3
|
-
import { createMarkdownProcessor,
|
|
4
|
-
import {
|
|
2
|
+
import { trimOutputStart } from "./_chunks/utils.mjs";
|
|
3
|
+
import { createMarkdownProcessor, tagHandlers, trimAsciiWhitespaceEnd } from "./_chunks/markdown-processor.mjs";
|
|
4
|
+
import { assertEngineOptions, buildTagOverrideHandlers, checkClean, endPlugins, processPluginsForEvent } from "./_chunks/tag-overrides.mjs";
|
|
5
|
+
import { finalizeParse, parseHtmlStream } from "./_chunks/parse.mjs";
|
|
6
|
+
import { resolvePlugins } from "./_chunks/plugin.mjs";
|
|
7
|
+
function createOutputPositions(buffer) {
|
|
8
|
+
const positions = [];
|
|
9
|
+
return {
|
|
10
|
+
capture(fragment = Math.max(0, buffer.length - 1), offset = buffer[fragment]?.length ?? 0, kind = "boundary") {
|
|
11
|
+
const position = {
|
|
12
|
+
fragment,
|
|
13
|
+
offset,
|
|
14
|
+
kind
|
|
15
|
+
};
|
|
16
|
+
positions.push(position);
|
|
17
|
+
return position;
|
|
18
|
+
},
|
|
19
|
+
replace(start, end, replacement, mapOffset) {
|
|
20
|
+
if (positions.length === 0) return;
|
|
21
|
+
const fragmentDelta = replacement.length - (end - start);
|
|
22
|
+
let lower = 0;
|
|
23
|
+
let upper = positions.length;
|
|
24
|
+
while (lower < upper) {
|
|
25
|
+
const middle = lower + upper >>> 1;
|
|
26
|
+
if (positions[middle].fragment < start) lower = middle + 1;
|
|
27
|
+
else upper = middle;
|
|
28
|
+
}
|
|
29
|
+
let oldPrefix;
|
|
30
|
+
let newPrefix;
|
|
31
|
+
for (let index = lower; index < positions.length; index++) {
|
|
32
|
+
const position = positions[index];
|
|
33
|
+
if (position.fragment >= end) {
|
|
34
|
+
if (fragmentDelta === 0) break;
|
|
35
|
+
position.fragment += fragmentDelta;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
if (!oldPrefix) {
|
|
39
|
+
oldPrefix = [0];
|
|
40
|
+
for (let index = start; index < end; index++) oldPrefix.push(oldPrefix.at(-1) + buffer[index].length);
|
|
41
|
+
newPrefix = [0];
|
|
42
|
+
for (const fragment of replacement) newPrefix.push(newPrefix.at(-1) + fragment.length);
|
|
43
|
+
}
|
|
44
|
+
const oldOffset = oldPrefix[position.fragment - start] + position.offset;
|
|
45
|
+
const endOffset = newPrefix.at(-1);
|
|
46
|
+
const offset = Math.max(0, Math.min(mapOffset ? mapOffset(oldOffset, position.kind) : oldOffset, endOffset));
|
|
47
|
+
if (replacement.length === 0 || position.kind === "boundary" && offset === 0 && start > 0) {
|
|
48
|
+
position.fragment = Math.max(0, start - 1);
|
|
49
|
+
position.offset = start > 0 ? buffer[start - 1].length : 0;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
let fragment = 0;
|
|
53
|
+
while (fragment + 1 < replacement.length && (newPrefix[fragment + 1] < offset || position.kind === "content" && newPrefix[fragment + 1] === offset)) fragment++;
|
|
54
|
+
position.fragment = start + fragment;
|
|
55
|
+
position.offset = offset - newPrefix[fragment];
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
finish() {
|
|
59
|
+
const prefix = [0];
|
|
60
|
+
for (const fragment of buffer) prefix.push(prefix.at(-1) + fragment.length);
|
|
61
|
+
return (position) => prefix[Math.min(position.fragment, buffer.length)] + Math.min(position.offset, buffer[position.fragment]?.length ?? 0);
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
}
|
|
5
65
|
const MARKDOWN_HEADER_LINE_RE = /^#{1,6}\s+/;
|
|
6
|
-
const
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
];
|
|
66
|
+
const DEFAULT_SPLIT_HEADINGS = 62;
|
|
67
|
+
const HEADING_LEVELS_ERROR = "headersToSplitOn takes heading levels from 1 to 6, such as [2, 3]. @mdream/js no longer exports the TAG_H* constants.";
|
|
68
|
+
function headingSplitMask(levels) {
|
|
69
|
+
if (levels == null) return DEFAULT_SPLIT_HEADINGS;
|
|
70
|
+
if (!Array.isArray(levels)) throw new TypeError(HEADING_LEVELS_ERROR);
|
|
71
|
+
let mask = 0;
|
|
72
|
+
for (let index = 0; index < levels.length; index++) {
|
|
73
|
+
const level = levels[index];
|
|
74
|
+
if (!Number.isInteger(level) || level < 1 || level > 6) throw new TypeError(HEADING_LEVELS_ERROR);
|
|
75
|
+
mask |= 1 << level - 1;
|
|
76
|
+
}
|
|
77
|
+
return mask;
|
|
78
|
+
}
|
|
14
79
|
function createOptions(options) {
|
|
15
80
|
return {
|
|
16
|
-
|
|
81
|
+
splitHeadings: headingSplitMask(options.headersToSplitOn),
|
|
17
82
|
returnEachLine: options.returnEachLine ?? false,
|
|
18
83
|
stripHeaders: options.stripHeaders ?? true,
|
|
19
84
|
chunkSize: options.chunkSize ?? 1e3,
|
|
20
85
|
chunkOverlap: options.chunkOverlap ?? 200,
|
|
21
86
|
lengthFunction: options.lengthFunction ?? ((text) => text.length),
|
|
87
|
+
customLength: options.lengthFunction !== void 0,
|
|
22
88
|
keepSeparator: options.keepSeparator ?? false,
|
|
23
|
-
resolvedPlugins: resolvePlugins(options
|
|
89
|
+
resolvedPlugins: resolvePlugins(options.plugins),
|
|
90
|
+
tagOverrideHandlers: options.tagOverrides ? buildTagOverrideHandlers(options.tagOverrides, tagHandlers) : void 0
|
|
24
91
|
};
|
|
25
92
|
}
|
|
26
93
|
function shouldSplitOnHeader(tagId, options) {
|
|
27
|
-
return options.
|
|
94
|
+
return (options.splitHeadings & 1 << tagId - 7) !== 0;
|
|
28
95
|
}
|
|
29
|
-
function
|
|
30
|
-
return
|
|
96
|
+
function isOutputWhitespace(code) {
|
|
97
|
+
return code === 32 || code >= 9 && code <= 13;
|
|
98
|
+
}
|
|
99
|
+
const SEPARATORS = [
|
|
100
|
+
"\n\n",
|
|
101
|
+
"```\n",
|
|
102
|
+
"\n",
|
|
103
|
+
" "
|
|
104
|
+
];
|
|
105
|
+
function contentStart(markdown, start, end) {
|
|
106
|
+
let lineStart = start === 0 || markdown.charCodeAt(start - 1) === 10 ? start : -1;
|
|
107
|
+
for (let index = start; index < end; index++) {
|
|
108
|
+
const code = markdown.charCodeAt(index);
|
|
109
|
+
if (code === 10) lineStart = index + 1;
|
|
110
|
+
else if (!isOutputWhitespace(code)) return lineStart === -1 ? index : lineStart;
|
|
111
|
+
}
|
|
112
|
+
return end;
|
|
113
|
+
}
|
|
114
|
+
function contentEnd(markdown, start, end) {
|
|
115
|
+
while (end > start && isOutputWhitespace(markdown.charCodeAt(end - 1))) end--;
|
|
116
|
+
return end;
|
|
117
|
+
}
|
|
118
|
+
function lastIndexBetween(markdown, separator, floor, from) {
|
|
119
|
+
if (from < floor) return -1;
|
|
120
|
+
const index = markdown.slice(floor, from + separator.length).lastIndexOf(separator);
|
|
121
|
+
return index === -1 ? -1 : floor + index;
|
|
122
|
+
}
|
|
123
|
+
function largestFit(max, firstProbe, fits) {
|
|
124
|
+
let good = 0;
|
|
125
|
+
let bad = -1;
|
|
126
|
+
let probe = firstProbe >= 1 ? Math.floor(firstProbe) : 1;
|
|
127
|
+
while (good < max) {
|
|
128
|
+
if (probe > max) probe = max;
|
|
129
|
+
if (!fits(probe)) {
|
|
130
|
+
bad = probe;
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
good = probe;
|
|
134
|
+
probe *= 2;
|
|
135
|
+
}
|
|
136
|
+
if (bad === -1) return good;
|
|
137
|
+
while (bad - good > 1) {
|
|
138
|
+
const middle = Math.floor((good + bad) / 2);
|
|
139
|
+
if (fits(middle)) good = middle;
|
|
140
|
+
else bad = middle;
|
|
141
|
+
}
|
|
142
|
+
return good;
|
|
143
|
+
}
|
|
144
|
+
function codeRegions(markdown) {
|
|
145
|
+
const regions = [];
|
|
146
|
+
let open;
|
|
147
|
+
let lineStart = 0;
|
|
148
|
+
while (lineStart < markdown.length) {
|
|
149
|
+
const newline = markdown.indexOf("\n", lineStart);
|
|
150
|
+
const lineEnd = newline === -1 ? markdown.length : newline;
|
|
151
|
+
let markerStart = lineStart;
|
|
152
|
+
let prefixStart = lineStart;
|
|
153
|
+
let prefix = "";
|
|
154
|
+
while (true) {
|
|
155
|
+
while (markdown.charCodeAt(markerStart) === 32 || markdown.charCodeAt(markerStart) === 62) markerStart++;
|
|
156
|
+
let listEnd = markerStart;
|
|
157
|
+
const first = markdown.charCodeAt(markerStart);
|
|
158
|
+
if ((first === 45 || first === 43 || first === 42) && markdown.charCodeAt(markerStart + 1) === 32) listEnd++;
|
|
159
|
+
else if (first >= 48 && first <= 57) {
|
|
160
|
+
let end = markerStart + 1;
|
|
161
|
+
while (markdown.charCodeAt(end) >= 48 && markdown.charCodeAt(end) <= 57) end++;
|
|
162
|
+
const delimiter = markdown.charCodeAt(end);
|
|
163
|
+
if ((delimiter === 46 || delimiter === 41) && markdown.charCodeAt(end + 1) === 32) listEnd = end + 1;
|
|
164
|
+
}
|
|
165
|
+
if (listEnd === markerStart) break;
|
|
166
|
+
prefix += markdown.slice(prefixStart, markerStart) + " ".repeat(listEnd - markerStart);
|
|
167
|
+
markerStart = listEnd;
|
|
168
|
+
prefixStart = listEnd;
|
|
169
|
+
}
|
|
170
|
+
prefix += markdown.slice(prefixStart, markerStart);
|
|
171
|
+
const marker = markdown.charCodeAt(markerStart);
|
|
172
|
+
if (marker === 96 || marker === 126) {
|
|
173
|
+
let markerEnd = markerStart + 1;
|
|
174
|
+
while (markdown.charCodeAt(markerEnd) === marker) markerEnd++;
|
|
175
|
+
const run = markerEnd - markerStart;
|
|
176
|
+
if (!open && run >= 3) open = {
|
|
177
|
+
start: lineStart,
|
|
178
|
+
markerStart,
|
|
179
|
+
marker,
|
|
180
|
+
run,
|
|
181
|
+
prefix
|
|
182
|
+
};
|
|
183
|
+
else if (open && marker === open.marker && run >= open.run && prefix === open.prefix) {
|
|
184
|
+
let tail = markerEnd;
|
|
185
|
+
while (tail < lineEnd && isOutputWhitespace(markdown.charCodeAt(tail))) tail++;
|
|
186
|
+
if (tail === lineEnd) {
|
|
187
|
+
regions.push({
|
|
188
|
+
start: open.start,
|
|
189
|
+
end: lineEnd,
|
|
190
|
+
markerStart: open.markerStart
|
|
191
|
+
});
|
|
192
|
+
open = void 0;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
lineStart = lineEnd + 1;
|
|
197
|
+
}
|
|
198
|
+
if (open) regions.push({
|
|
199
|
+
start: open.start,
|
|
200
|
+
end: markdown.length,
|
|
201
|
+
markerStart: open.markerStart
|
|
202
|
+
});
|
|
203
|
+
return regions;
|
|
31
204
|
}
|
|
32
205
|
function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
206
|
+
assertEngineOptions(options, "splitter");
|
|
207
|
+
checkClean(options);
|
|
33
208
|
const opts = createOptions(options);
|
|
34
209
|
if (opts.chunkOverlap >= opts.chunkSize) throw new Error("chunkOverlap must be less than chunkSize");
|
|
35
|
-
|
|
36
|
-
const
|
|
210
|
+
const processor = createMarkdownProcessor(options);
|
|
211
|
+
const positions = createOutputPositions(processor.state.buffer);
|
|
212
|
+
processor.state.outputPositions = positions;
|
|
213
|
+
const codePositions = [];
|
|
37
214
|
processor.state.onCodeFenceOpen = (language) => {
|
|
38
|
-
if (language
|
|
215
|
+
if (language) {
|
|
216
|
+
const fence = processor.state.codeFence;
|
|
217
|
+
codePositions.push({
|
|
218
|
+
position: positions.capture(fence.fragment, fence.markerOffset, "content"),
|
|
219
|
+
language
|
|
220
|
+
});
|
|
221
|
+
}
|
|
39
222
|
};
|
|
40
223
|
const headerHierarchy = /* @__PURE__ */ new Map();
|
|
41
224
|
const seenSplitHeaders = /* @__PURE__ */ new Set();
|
|
42
225
|
let collectingHeaderText = false;
|
|
43
226
|
let currentHeaderTagId = null;
|
|
44
227
|
let currentHeaderText = "";
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const originalChunkContent = currentMd.slice(lastChunkEndPosition, chunkEnd);
|
|
52
|
-
if (!originalChunkContent.trim()) {
|
|
53
|
-
lastChunkEndPosition = chunkEnd;
|
|
54
|
-
return;
|
|
55
|
-
}
|
|
56
|
-
let chunkContent = originalChunkContent;
|
|
57
|
-
if (opts.stripHeaders) {
|
|
58
|
-
chunkContent = chunkContent.split("\n").filter((line) => !MARKDOWN_HEADER_LINE_RE.test(line)).join("\n").trim();
|
|
59
|
-
if (!chunkContent) {
|
|
60
|
-
lastChunkEndPosition = chunkEnd;
|
|
61
|
-
return;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
const chunk = {
|
|
65
|
-
content: chunkContent.trimEnd(),
|
|
66
|
-
metadata: { loc: { lines: {
|
|
67
|
-
from: lineNumber,
|
|
68
|
-
to: lineNumber + (originalChunkContent.match(NEWLINE_RE) || []).length
|
|
69
|
-
} } }
|
|
70
|
-
};
|
|
71
|
-
if (headerHierarchy.size > 0) {
|
|
72
|
-
chunk.metadata.headers = {};
|
|
73
|
-
for (const [tagId, text] of headerHierarchy.entries()) {
|
|
74
|
-
const level = `h${tagId - 7 + 1}`;
|
|
75
|
-
chunk.metadata.headers[level] = text;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
if (currentChunkCodeLanguage) chunk.metadata.code = currentChunkCodeLanguage;
|
|
79
|
-
yield chunk;
|
|
80
|
-
currentChunkCodeLanguage = "";
|
|
81
|
-
lastSplitPosition = chunkEnd;
|
|
82
|
-
if (applyOverlap && opts.chunkOverlap > 0) {
|
|
83
|
-
const maxOverlap = Math.max(0, originalChunkContent.length - 1);
|
|
84
|
-
lastChunkEndPosition = chunkEnd - Math.min(opts.chunkOverlap, maxOverlap);
|
|
85
|
-
} else lastChunkEndPosition = chunkEnd;
|
|
86
|
-
lineNumber += (originalChunkContent.match(NEWLINE_RE) || []).length;
|
|
87
|
-
}
|
|
88
|
-
const parseState = {
|
|
89
|
-
depthMap: processor.state.depthMap,
|
|
90
|
-
depth: 0,
|
|
91
|
-
resolvedPlugins: opts.resolvedPlugins
|
|
228
|
+
const sectionEnds = [];
|
|
229
|
+
const endSection = () => {
|
|
230
|
+
sectionEnds.push({
|
|
231
|
+
position: positions.capture(),
|
|
232
|
+
headers: new Map(headerHierarchy)
|
|
233
|
+
});
|
|
92
234
|
};
|
|
93
|
-
const eventBuffer = [];
|
|
94
235
|
const processResolvedEvent = opts.resolvedPlugins?.length ? (event) => processPluginsForEvent(event, opts.resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
|
|
95
|
-
|
|
96
|
-
eventBuffer.push(event);
|
|
97
|
-
});
|
|
98
|
-
for (const event of eventBuffer) {
|
|
236
|
+
const handleEvent = (event) => {
|
|
99
237
|
const { type: eventType, node } = event;
|
|
100
|
-
const eventDepthMap = node.type === 1 ? node.depthMap : node.parent?.depthMap;
|
|
101
|
-
if (eventDepthMap) processor.state.depthMap.set(eventDepthMap);
|
|
102
|
-
else processor.state.depthMap.fill(0);
|
|
103
|
-
if (eventType === 1 && node.type === 1) {
|
|
104
|
-
const tagId = node.tagId;
|
|
105
|
-
if (tagId !== void 0 && tagId >= 0 && tagId < processor.state.depthMap.length) processor.state.depthMap[tagId] = Math.max(0, processor.state.depthMap[tagId] - 1);
|
|
106
|
-
}
|
|
107
238
|
if (node.type === 1) {
|
|
108
239
|
const tagId = node.tagId;
|
|
109
240
|
if (tagId && tagId >= 7 && tagId <= 12) {
|
|
@@ -113,7 +244,7 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
|
113
244
|
currentHeaderText = "";
|
|
114
245
|
if (shouldSplitOnHeader(tagId, opts)) {
|
|
115
246
|
if (seenSplitHeaders.has(tagId)) {
|
|
116
|
-
|
|
247
|
+
endSection();
|
|
117
248
|
for (let i = tagId; i <= 12; i++) headerHierarchy.delete(i);
|
|
118
249
|
}
|
|
119
250
|
seenSplitHeaders.add(tagId);
|
|
@@ -124,51 +255,193 @@ function* htmlToMarkdownSplitChunksStream(html, options = {}) {
|
|
|
124
255
|
currentHeaderTagId = null;
|
|
125
256
|
}
|
|
126
257
|
}
|
|
127
|
-
if (tagId === 13 && eventType === 0)
|
|
128
|
-
}
|
|
129
|
-
if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
|
|
258
|
+
if (tagId === 13 && eventType === 0) endSection();
|
|
259
|
+
} else if (collectingHeaderText && node.type === 2) currentHeaderText += node.value;
|
|
130
260
|
processResolvedEvent(event);
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
261
|
+
};
|
|
262
|
+
const parseState = {
|
|
263
|
+
depthMap: processor.state.depthMap,
|
|
264
|
+
depth: 0,
|
|
265
|
+
resolvedPlugins: opts.resolvedPlugins,
|
|
266
|
+
tagHandlers,
|
|
267
|
+
tagOverrideHandlers: opts.tagOverrideHandlers
|
|
268
|
+
};
|
|
269
|
+
const leftover = parseHtmlStream(html, parseState, handleEvent);
|
|
270
|
+
finalizeParse(leftover, parseState, handleEvent);
|
|
271
|
+
endPlugins(opts.resolvedPlugins, processor.state);
|
|
272
|
+
endSection();
|
|
273
|
+
const positionOffset = positions.finish();
|
|
274
|
+
const joined = processor.state.buffer.join("");
|
|
275
|
+
const started = trimOutputStart(joined);
|
|
276
|
+
const leadingTrim = joined.length - started.length;
|
|
277
|
+
const raw = trimAsciiWhitespaceEnd(started);
|
|
278
|
+
const view = processor.finishOutput?.(raw);
|
|
279
|
+
const markdown = view ? view.markdown : raw;
|
|
280
|
+
const mapPosition = view ? view.mapPosition ?? ((position) => processor.finishOutput(raw.slice(0, position)).markdown.length) : void 0;
|
|
281
|
+
const offsetOf = (position) => {
|
|
282
|
+
const offset = Math.min(Math.max(0, positionOffset(position) - leadingTrim), raw.length);
|
|
283
|
+
return offset === raw.length ? markdown.length : mapPosition ? mapPosition(offset) : offset;
|
|
284
|
+
};
|
|
285
|
+
const sections = [];
|
|
286
|
+
for (const { position, headers } of sectionEnds) sections.push({
|
|
287
|
+
end: offsetOf(position),
|
|
288
|
+
headers
|
|
289
|
+
});
|
|
290
|
+
const codes = [];
|
|
291
|
+
for (const { position, language } of codePositions) codes.push({
|
|
292
|
+
start: offsetOf(position),
|
|
293
|
+
language
|
|
294
|
+
});
|
|
295
|
+
codes.sort((a, b) => a.start - b.start);
|
|
296
|
+
yield* chunkSections(markdown, sections, codes, opts);
|
|
297
|
+
}
|
|
298
|
+
function* chunkSections(markdown, sections, codes, opts) {
|
|
299
|
+
const regions = codeRegions(markdown);
|
|
300
|
+
const newlines = [];
|
|
301
|
+
for (let index = markdown.indexOf("\n"); index !== -1; index = markdown.indexOf("\n", index + 1)) newlines.push(index);
|
|
302
|
+
let chunkStart = 0;
|
|
303
|
+
let lastEnd = 0;
|
|
304
|
+
let codeIndex = 0;
|
|
305
|
+
let headers = /* @__PURE__ */ new Map();
|
|
306
|
+
function lineAt(offset) {
|
|
307
|
+
let lower = 0;
|
|
308
|
+
let upper = newlines.length;
|
|
309
|
+
while (lower < upper) {
|
|
310
|
+
const middle = lower + upper >>> 1;
|
|
311
|
+
if (newlines[middle] < offset) lower = middle + 1;
|
|
312
|
+
else upper = middle;
|
|
313
|
+
}
|
|
314
|
+
return lower + 1;
|
|
315
|
+
}
|
|
316
|
+
function codeRegionAt(position) {
|
|
317
|
+
let lower = 0;
|
|
318
|
+
let upper = regions.length;
|
|
319
|
+
while (lower < upper) {
|
|
320
|
+
const middle = lower + upper >>> 1;
|
|
321
|
+
if (regions[middle].start < position) lower = middle + 1;
|
|
322
|
+
else upper = middle;
|
|
323
|
+
}
|
|
324
|
+
const region = lower > 0 ? regions[lower - 1] : void 0;
|
|
325
|
+
return region && position < region.end ? region : void 0;
|
|
326
|
+
}
|
|
327
|
+
function measure(start, end) {
|
|
328
|
+
return opts.lengthFunction(markdown.slice(start, end));
|
|
329
|
+
}
|
|
330
|
+
function advanceTo(position) {
|
|
331
|
+
const next = contentStart(markdown, position, markdown.length);
|
|
332
|
+
if (next > chunkStart) chunkStart = next;
|
|
333
|
+
}
|
|
334
|
+
function sizeLimit(sectionEnd) {
|
|
335
|
+
if (!opts.customLength) return Math.min(sectionEnd, chunkStart + opts.chunkSize);
|
|
336
|
+
const rest = sectionEnd - chunkStart;
|
|
337
|
+
if (rest <= 0) return sectionEnd;
|
|
338
|
+
const fit = largestFit(rest, opts.chunkSize, (length) => !(measure(chunkStart, chunkStart + length) > opts.chunkSize));
|
|
339
|
+
return chunkStart + Math.max(fit, 1);
|
|
340
|
+
}
|
|
341
|
+
function splitPosition(sectionEnd, idealSplitPos) {
|
|
342
|
+
const floor = Math.max(chunkStart, lastEnd);
|
|
343
|
+
let heldRegion;
|
|
344
|
+
for (const separator of SEPARATORS) {
|
|
345
|
+
const index = lastIndexBetween(markdown, separator, Math.max(0, floor - separator.length + 1), idealSplitPos);
|
|
346
|
+
const candidate = index + separator.length;
|
|
347
|
+
if (index < 0 || candidate > sectionEnd || contentEnd(markdown, floor, candidate) <= floor) continue;
|
|
348
|
+
const region = codeRegionAt(candidate);
|
|
349
|
+
if (!region) return candidate;
|
|
350
|
+
heldRegion = region;
|
|
351
|
+
}
|
|
352
|
+
heldRegion ??= codeRegionAt(idealSplitPos);
|
|
353
|
+
if (heldRegion) return Math.min(heldRegion.end, sectionEnd);
|
|
354
|
+
let next = idealSplitPos;
|
|
355
|
+
while (next < sectionEnd) {
|
|
356
|
+
const code = markdown.charCodeAt(next);
|
|
357
|
+
if (code === 32 || code === 10) break;
|
|
358
|
+
next++;
|
|
359
|
+
}
|
|
360
|
+
return next > chunkStart ? next : sectionEnd;
|
|
361
|
+
}
|
|
362
|
+
function* flush(endPosition, applyOverlap) {
|
|
363
|
+
const chunkEnd = Math.min(endPosition, markdown.length);
|
|
364
|
+
const start = chunkStart;
|
|
365
|
+
const end = contentEnd(markdown, start, chunkEnd);
|
|
366
|
+
if (end <= start || end <= lastEnd) {
|
|
367
|
+
advanceTo(chunkEnd);
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
let content = markdown.slice(start, end);
|
|
371
|
+
let contentFrom = start;
|
|
372
|
+
let contentTo = end;
|
|
373
|
+
if (opts.stripHeaders && (content.charCodeAt(0) === 35 || content.includes("\n#"))) {
|
|
374
|
+
const kept = [];
|
|
375
|
+
let firstKept = -1;
|
|
376
|
+
let lastKept = -1;
|
|
377
|
+
let offset = 0;
|
|
378
|
+
for (const line of content.split("\n")) {
|
|
379
|
+
if (line.charCodeAt(0) !== 35 || !MARKDOWN_HEADER_LINE_RE.test(line)) {
|
|
380
|
+
const lineEnd = contentEnd(line, 0, line.length);
|
|
381
|
+
if (lineEnd > 0) {
|
|
382
|
+
if (firstKept === -1) {
|
|
383
|
+
firstKept = kept.length;
|
|
384
|
+
contentFrom = start + offset;
|
|
152
385
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
if (idx >= 0 && candidateSplitPos > lastSplitPosition) {
|
|
156
|
-
splitPosition = candidateSplitPos;
|
|
157
|
-
break;
|
|
386
|
+
lastKept = kept.length;
|
|
387
|
+
contentTo = start + offset + lineEnd;
|
|
158
388
|
}
|
|
389
|
+
kept.push(line);
|
|
159
390
|
}
|
|
160
|
-
|
|
161
|
-
yield* flushChunk(splitPosition, true);
|
|
391
|
+
offset += line.length + 1;
|
|
162
392
|
}
|
|
393
|
+
if (firstKept === -1) {
|
|
394
|
+
lastEnd = end;
|
|
395
|
+
advanceTo(chunkEnd);
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
const lastLine = kept[lastKept];
|
|
399
|
+
kept[lastKept] = lastLine.slice(0, contentEnd(lastLine, 0, lastLine.length));
|
|
400
|
+
content = kept.slice(firstKept, lastKept + 1).join("\n");
|
|
163
401
|
}
|
|
402
|
+
const chunk = {
|
|
403
|
+
content,
|
|
404
|
+
metadata: { loc: { lines: {
|
|
405
|
+
from: lineAt(contentFrom),
|
|
406
|
+
to: lineAt(contentTo - 1)
|
|
407
|
+
} } }
|
|
408
|
+
};
|
|
409
|
+
if (headers.size > 0) {
|
|
410
|
+
chunk.metadata.headers = {};
|
|
411
|
+
for (const [tagId, text] of headers.entries()) {
|
|
412
|
+
const level = `h${tagId - 7 + 1}`;
|
|
413
|
+
chunk.metadata.headers[level] = text;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
while (codeIndex < codes.length && codes[codeIndex].start < start) codeIndex++;
|
|
417
|
+
if (codeIndex < codes.length && codes[codeIndex].start < end) chunk.metadata.code = codes[codeIndex].language;
|
|
418
|
+
yield chunk;
|
|
419
|
+
lastEnd = end;
|
|
420
|
+
let next = chunkEnd;
|
|
421
|
+
if (applyOverlap && opts.chunkOverlap > 0) {
|
|
422
|
+
const overlapRoom = end - start - 1;
|
|
423
|
+
let overlapStart = end - (opts.customLength ? largestFit(overlapRoom, opts.chunkOverlap, (length) => !(measure(end - length, end) > opts.chunkOverlap)) : Math.min(opts.chunkOverlap, overlapRoom));
|
|
424
|
+
while (overlapStart > start + 1 && !isOutputWhitespace(markdown.charCodeAt(overlapStart - 1))) overlapStart--;
|
|
425
|
+
const region = codeRegionAt(overlapStart);
|
|
426
|
+
if (region) overlapStart = Math.min(region.end, end);
|
|
427
|
+
if (overlapStart > start + 1) next = overlapStart;
|
|
428
|
+
}
|
|
429
|
+
advanceTo(next);
|
|
430
|
+
}
|
|
431
|
+
for (const section of sections) {
|
|
432
|
+
headers = section.headers;
|
|
433
|
+
if (!opts.returnEachLine) for (let limit = sizeLimit(section.end); limit < section.end; limit = sizeLimit(section.end)) {
|
|
434
|
+
const previousStart = chunkStart;
|
|
435
|
+
yield* flush(splitPosition(section.end, limit), true);
|
|
436
|
+
if (chunkStart <= previousStart) break;
|
|
437
|
+
}
|
|
438
|
+
yield* flush(section.end, false);
|
|
164
439
|
}
|
|
165
|
-
yield* flushChunk();
|
|
166
440
|
}
|
|
167
441
|
function htmlToMarkdownSplitChunks(html, options = {}) {
|
|
168
|
-
const opts = createOptions(options);
|
|
169
442
|
const chunks = [];
|
|
170
443
|
for (const chunk of htmlToMarkdownSplitChunksStream(html, options)) chunks.push(chunk);
|
|
171
|
-
if (
|
|
444
|
+
if (options.returnEachLine && chunks.length > 0) {
|
|
172
445
|
const lineChunks = [];
|
|
173
446
|
for (const chunk of chunks) {
|
|
174
447
|
const lines = chunk.content.split("\n");
|
package/dist/text.d.mts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { MdreamOptions } from "./_chunks/types.mjs";
|
|
2
|
+
/** Convert HTML to readable plain text. */
|
|
3
|
+
export declare function htmlToText(html: string, options?: Partial<MdreamOptions>): string;
|
|
4
|
+
/** Stream HTML as readable plain text. */
|
|
5
|
+
export declare function streamHtmlToText(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
6
|
+
export type { MdreamOptions };
|
package/dist/text.mjs
ADDED
package/package.json
CHANGED
|
@@ -1,13 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mdream/js",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "2.0.
|
|
5
|
-
"description": "
|
|
4
|
+
"version": "2.0.1",
|
|
5
|
+
"description": "Tree-shakable JavaScript converters and plugins for mdream.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
8
8
|
"email": "harlan@harlanzw.com",
|
|
9
9
|
"url": "https://harlanzw.com/"
|
|
10
10
|
},
|
|
11
|
+
"contributors": [
|
|
12
|
+
{
|
|
13
|
+
"name": "André Cruz",
|
|
14
|
+
"url": "https://github.com/edevil"
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"name": "Bernardo Pereira",
|
|
18
|
+
"url": "https://github.com/BernardoPe"
|
|
19
|
+
}
|
|
20
|
+
],
|
|
11
21
|
"license": "MIT",
|
|
12
22
|
"repository": {
|
|
13
23
|
"type": "git",
|
|
@@ -24,13 +34,29 @@
|
|
|
24
34
|
},
|
|
25
35
|
"default": "./dist/index.mjs"
|
|
26
36
|
},
|
|
27
|
-
"./
|
|
28
|
-
"types": "./dist/
|
|
37
|
+
"./text": {
|
|
38
|
+
"types": "./dist/text.d.mts",
|
|
39
|
+
"import": {
|
|
40
|
+
"types": "./dist/text.d.mts",
|
|
41
|
+
"default": "./dist/text.mjs"
|
|
42
|
+
},
|
|
43
|
+
"default": "./dist/text.mjs"
|
|
44
|
+
},
|
|
45
|
+
"./html": {
|
|
46
|
+
"types": "./dist/html.d.mts",
|
|
47
|
+
"import": {
|
|
48
|
+
"types": "./dist/html.d.mts",
|
|
49
|
+
"default": "./dist/html.mjs"
|
|
50
|
+
},
|
|
51
|
+
"default": "./dist/html.mjs"
|
|
52
|
+
},
|
|
53
|
+
"./clean": {
|
|
54
|
+
"types": "./dist/clean.d.mts",
|
|
29
55
|
"import": {
|
|
30
|
-
"types": "./dist/
|
|
31
|
-
"default": "./dist/
|
|
56
|
+
"types": "./dist/clean.d.mts",
|
|
57
|
+
"default": "./dist/clean.mjs"
|
|
32
58
|
},
|
|
33
|
-
"default": "./dist/
|
|
59
|
+
"default": "./dist/clean.mjs"
|
|
34
60
|
},
|
|
35
61
|
"./negotiate": {
|
|
36
62
|
"types": "./dist/negotiate.d.mts",
|
|
@@ -91,7 +117,7 @@
|
|
|
91
117
|
"dist"
|
|
92
118
|
],
|
|
93
119
|
"dependencies": {
|
|
94
|
-
"cac": "^7.0.
|
|
120
|
+
"cac": "^7.0.1",
|
|
95
121
|
"pathe": "^2.0.3"
|
|
96
122
|
},
|
|
97
123
|
"scripts": {
|
package/dist/_chunks/index.d.mts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { MdreamOptions, TransformPlugin } from "./types.mjs";
|
|
2
|
-
import "./minimal.mjs";
|
|
3
|
-
/**
|
|
4
|
-
* Create a plugin with type-safe hook definitions.
|
|
5
|
-
* All TransformPlugin fields are optional, so this is a typed identity function.
|
|
6
|
-
*/
|
|
7
|
-
export declare function createPlugin(plugin: TransformPlugin): TransformPlugin;
|
|
8
|
-
export declare function htmlToMarkdown(html: string, options?: Partial<MdreamOptions>): string;
|
|
9
|
-
export declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: Partial<MdreamOptions>): AsyncIterable<string>;
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import { EngineOptions } from "./types.mjs";
|
|
2
|
-
/**
|
|
3
|
-
* Creates a configurable minimal preset with advanced options.
|
|
4
|
-
* Returns declarative plugin config that works with both JS and Rust engines.
|
|
5
|
-
*/
|
|
6
|
-
export declare function withMinimalPreset<T extends EngineOptions>(options?: T): T;
|