@mdream/js 1.4.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -7
- package/dist/_chunks/const.mjs +2 -2
- package/dist/_chunks/index.d.mts +1 -0
- package/dist/_chunks/markdown-processor.mjs +404 -0
- package/dist/_chunks/parse.mjs +946 -452
- package/dist/_chunks/plugins.mjs +18 -19
- package/dist/_chunks/resolve-plugins.mjs +1 -339
- package/dist/_chunks/src.mjs +5 -35
- package/dist/_chunks/stream.mjs +35 -0
- package/dist/_chunks/types.d.mts +23 -6
- package/dist/cli.mjs +9 -2
- package/dist/core.d.mts +16 -0
- package/dist/core.mjs +26 -0
- package/dist/parse.d.mts +26 -9
- package/dist/parse.mjs +2 -2
- package/dist/plugins.d.mts +6 -5
- package/dist/splitter.mjs +11 -2
- package/package.json +12 -2
package/dist/_chunks/plugins.mjs
CHANGED
|
@@ -226,30 +226,24 @@ function filterPlugin(options = {}) {
|
|
|
226
226
|
const includeSelectors = options.include?.map((selector) => compileSelector(selector)) || [];
|
|
227
227
|
const excludeSelectors = options.exclude?.map((selector) => compileSelector(selector)) || [];
|
|
228
228
|
const processChildren = options.processChildren !== false;
|
|
229
|
-
const
|
|
229
|
+
const skippedSubtrees = /* @__PURE__ */ new WeakSet();
|
|
230
230
|
return createPlugin({ beforeNodeProcess(event) {
|
|
231
231
|
const { node } = event;
|
|
232
232
|
if (node.type === 2) {
|
|
233
233
|
const parent = node.parent;
|
|
234
|
-
if (parent &&
|
|
235
|
-
let currentParent = parent;
|
|
236
|
-
while (currentParent && excludeSelectors.length) {
|
|
237
|
-
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
238
|
-
currentParent = currentParent.parent;
|
|
239
|
-
}
|
|
234
|
+
if (parent && skippedSubtrees.has(parent)) return { skip: true };
|
|
240
235
|
return;
|
|
241
236
|
}
|
|
242
237
|
if (node.type !== 1) return;
|
|
243
238
|
const element = node;
|
|
244
|
-
if (
|
|
245
|
-
|
|
239
|
+
if (skippedSubtrees.has(element)) return { skip: true };
|
|
240
|
+
if ((element.parent ? skippedSubtrees.has(element.parent) : false) || isHidden(element)) {
|
|
241
|
+
skippedSubtrees.add(element);
|
|
246
242
|
return { skip: true };
|
|
247
243
|
}
|
|
248
|
-
if (excludeSelectors.length && excludeSelectors.some((selector) => selector.matches(element)))
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
if (excludeSelectors.some((selector) => selector.matches(currentParent))) return { skip: true };
|
|
252
|
-
currentParent = currentParent.parent;
|
|
244
|
+
if (excludeSelectors.length && excludeSelectors.some((selector) => selector.matches(element))) {
|
|
245
|
+
skippedSubtrees.add(element);
|
|
246
|
+
return { skip: true };
|
|
253
247
|
}
|
|
254
248
|
if (includeSelectors.length) {
|
|
255
249
|
let currentElement = element;
|
|
@@ -267,7 +261,7 @@ const DOUBLE_QUOTE_RE = /"/g;
|
|
|
267
261
|
const ESCAPED_DOUBLE_QUOTE_RE = /\\"/g;
|
|
268
262
|
function frontmatterPlugin(options = {}) {
|
|
269
263
|
const additionalFields = options.additionalFields || {};
|
|
270
|
-
const metaFields = new Set([
|
|
264
|
+
const metaFields = /* @__PURE__ */ new Set([
|
|
271
265
|
"description",
|
|
272
266
|
"keywords",
|
|
273
267
|
"author",
|
|
@@ -305,6 +299,7 @@ function frontmatterPlugin(options = {}) {
|
|
|
305
299
|
}
|
|
306
300
|
const plugin = createPlugin({
|
|
307
301
|
onNodeEnter(node) {
|
|
302
|
+
if (node.excludedFromMarkdown) return;
|
|
308
303
|
if (node.tagId === 1) {
|
|
309
304
|
inHead = true;
|
|
310
305
|
return;
|
|
@@ -318,8 +313,10 @@ function frontmatterPlugin(options = {}) {
|
|
|
318
313
|
}
|
|
319
314
|
},
|
|
320
315
|
onNodeExit(node, state) {
|
|
316
|
+
if (node.excludedFromMarkdown) return void 0;
|
|
321
317
|
if (node.type === 1 && node.tagId === 1) {
|
|
322
318
|
inHead = false;
|
|
319
|
+
if (state.options?.format === "text") return void 0;
|
|
323
320
|
if (Object.keys(frontmatter).length > 0) {
|
|
324
321
|
const frontmatterContent = generateFrontmatter();
|
|
325
322
|
if (frontmatterContent) {
|
|
@@ -330,6 +327,7 @@ function frontmatterPlugin(options = {}) {
|
|
|
330
327
|
}
|
|
331
328
|
},
|
|
332
329
|
processTextNode(node) {
|
|
330
|
+
if (node.parent?.excludedFromMarkdown) return;
|
|
333
331
|
if (!inHead) return;
|
|
334
332
|
const parent = node.parent;
|
|
335
333
|
if (parent && parent.tagId === 4 && node.value) {
|
|
@@ -366,7 +364,7 @@ function isolateMainPlugin() {
|
|
|
366
364
|
let mainElement = null;
|
|
367
365
|
let firstHeaderElement = null;
|
|
368
366
|
let afterFooter = false;
|
|
369
|
-
const headerTagIds = new Set([
|
|
367
|
+
const headerTagIds = /* @__PURE__ */ new Set([
|
|
370
368
|
7,
|
|
371
369
|
8,
|
|
372
370
|
9,
|
|
@@ -376,6 +374,7 @@ function isolateMainPlugin() {
|
|
|
376
374
|
]);
|
|
377
375
|
return createPlugin({ beforeNodeProcess(event) {
|
|
378
376
|
const { node } = event;
|
|
377
|
+
if (node.excludedFromMarkdown || node.parent?.excludedFromMarkdown) return;
|
|
379
378
|
if (node.type === 1) {
|
|
380
379
|
const element = node;
|
|
381
380
|
if (!mainElement && element.tagId === 104 && element.depth <= 5) {
|
|
@@ -641,7 +640,7 @@ function tailwindPlugin() {
|
|
|
641
640
|
hidden: hidden || !!parentHidden
|
|
642
641
|
};
|
|
643
642
|
},
|
|
644
|
-
processTextNode(node) {
|
|
643
|
+
processTextNode(node, state) {
|
|
645
644
|
const parentNode = node.parent;
|
|
646
645
|
if (!parentNode || parentNode.type !== 1) return;
|
|
647
646
|
const tailwindData = parentNode.context?.tailwind;
|
|
@@ -650,8 +649,8 @@ function tailwindPlugin() {
|
|
|
650
649
|
skip: true
|
|
651
650
|
};
|
|
652
651
|
let content = node.value;
|
|
653
|
-
const prefix = tailwindData?.prefix || "";
|
|
654
|
-
const suffix = tailwindData?.suffix || "";
|
|
652
|
+
const prefix = state.options?.format === "text" ? "" : tailwindData?.prefix || "";
|
|
653
|
+
const suffix = state.options?.format === "text" ? "" : tailwindData?.suffix || "";
|
|
655
654
|
if (prefix || suffix) {
|
|
656
655
|
content = prefix + content + suffix;
|
|
657
656
|
content = fixRedundantDelimiters(content);
|
|
@@ -1,342 +1,4 @@
|
|
|
1
|
-
import { DEFAULT_BLOCK_SPACING, NO_SPACING } from "./const.mjs";
|
|
2
|
-
import { parseHtmlStream } from "./parse.mjs";
|
|
3
1
|
import { extractionCollectorPlugin, filterPlugin, frontmatterPlugin, isolateMainPlugin, tailwindPlugin } from "./plugins.mjs";
|
|
4
|
-
function processPluginsForEvent(event, plugins, state, processEvent) {
|
|
5
|
-
if (plugins?.length) {
|
|
6
|
-
if (event.node.type === 1 && event.type === 0) {
|
|
7
|
-
const element = event.node;
|
|
8
|
-
for (const plugin of plugins) if (plugin.processAttributes) plugin.processAttributes(element, state);
|
|
9
|
-
}
|
|
10
|
-
let shouldSkip = false;
|
|
11
|
-
for (const plugin of plugins) {
|
|
12
|
-
const res = plugin.beforeNodeProcess?.(event, state);
|
|
13
|
-
if (typeof res === "object") shouldSkip = res.skip;
|
|
14
|
-
}
|
|
15
|
-
if (shouldSkip) return true;
|
|
16
|
-
if (event.node.type === 1) {
|
|
17
|
-
const element = event.node;
|
|
18
|
-
const fn = event.type === 0 ? "onNodeEnter" : "onNodeExit";
|
|
19
|
-
const pluginOutputs = [];
|
|
20
|
-
for (const plugin of plugins) if (plugin[fn]) {
|
|
21
|
-
const result = plugin[fn](element, state);
|
|
22
|
-
if (result) pluginOutputs.push(result);
|
|
23
|
-
}
|
|
24
|
-
if (pluginOutputs.length > 0) element.pluginOutput = [...element.pluginOutput || [], ...pluginOutputs];
|
|
25
|
-
} else if (event.node.type === 2 && event.type === 0) {
|
|
26
|
-
const textNode = event.node;
|
|
27
|
-
for (const plugin of plugins) if (plugin.processTextNode) {
|
|
28
|
-
const result = plugin.processTextNode(textNode, state);
|
|
29
|
-
if (result) {
|
|
30
|
-
if (result.skip) return true;
|
|
31
|
-
textNode.value = result.content;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
processEvent(event);
|
|
37
|
-
return false;
|
|
38
|
-
}
|
|
39
|
-
function updateListIndent(state, element, eventType) {
|
|
40
|
-
if (element.tagId !== 25) return;
|
|
41
|
-
if ((state.depthMap[32] || 0) > 0 || (state.depthMap[31] || 0) > 0) return;
|
|
42
|
-
if (eventType === 0) {
|
|
43
|
-
const width = element.parent?.tagId === 33 ? String(element.index + 1).length + 2 : 2;
|
|
44
|
-
state.listIndentWidths.push(width);
|
|
45
|
-
state.listIndent += " ".repeat(width);
|
|
46
|
-
} else if (eventType === 1) {
|
|
47
|
-
const width = state.listIndentWidths.pop() ?? 0;
|
|
48
|
-
state.listIndent = state.listIndent.slice(0, state.listIndent.length - width);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
function needsSpacing(lastChar, firstChar, state) {
|
|
52
|
-
if (lastChar === " " || lastChar === "\n" || lastChar === " ") return false;
|
|
53
|
-
if (firstChar === " " || firstChar === "\n" || firstChar === " ") return false;
|
|
54
|
-
const noSpaceAfter = new Set([
|
|
55
|
-
"[",
|
|
56
|
-
"(",
|
|
57
|
-
">",
|
|
58
|
-
"*",
|
|
59
|
-
"_",
|
|
60
|
-
"`"
|
|
61
|
-
]);
|
|
62
|
-
const noSpaceBefore = new Set([
|
|
63
|
-
"]",
|
|
64
|
-
")",
|
|
65
|
-
"<",
|
|
66
|
-
".",
|
|
67
|
-
",",
|
|
68
|
-
"!",
|
|
69
|
-
"?",
|
|
70
|
-
":",
|
|
71
|
-
";",
|
|
72
|
-
"*",
|
|
73
|
-
"_",
|
|
74
|
-
"`"
|
|
75
|
-
]);
|
|
76
|
-
if (lastChar === "|" && firstChar === "<" && state && (state.depthMap[28] || 0) > 0) return true;
|
|
77
|
-
if (noSpaceAfter.has(lastChar) || noSpaceBefore.has(firstChar)) return false;
|
|
78
|
-
return true;
|
|
79
|
-
}
|
|
80
|
-
function shouldAddSpacingBeforeText(lastChar, lastNode, textNode) {
|
|
81
|
-
if (!lastChar || lastChar === "\n" || lastChar === " " || lastChar === "[" || lastChar === ">") return false;
|
|
82
|
-
if (lastNode?.tagHandler?.isInline) return false;
|
|
83
|
-
const firstChar = textNode.value[0];
|
|
84
|
-
if (firstChar === " ") return false;
|
|
85
|
-
if (firstChar === "." || firstChar === "," || firstChar === "!" || firstChar === "?" || firstChar === ":" || firstChar === ";" || firstChar === "_" || firstChar === "*" || firstChar === "`" || firstChar === ")" || firstChar === "]") return false;
|
|
86
|
-
return true;
|
|
87
|
-
}
|
|
88
|
-
function canWrapHere(depthMap) {
|
|
89
|
-
if (depthMap[34] || depthMap[23] || depthMap[32] || depthMap[31]) return false;
|
|
90
|
-
for (let h = 7; h <= 12; h++) if (depthMap[h]) return false;
|
|
91
|
-
return true;
|
|
92
|
-
}
|
|
93
|
-
function currentColumn(buffer) {
|
|
94
|
-
let col = 0;
|
|
95
|
-
for (let i = buffer.length - 1; i >= 0; i--) {
|
|
96
|
-
const s = buffer[i];
|
|
97
|
-
const nl = s.lastIndexOf("\n");
|
|
98
|
-
if (nl >= 0) return col + [...s.slice(nl + 1)].length;
|
|
99
|
-
col += [...s].length;
|
|
100
|
-
}
|
|
101
|
-
return col;
|
|
102
|
-
}
|
|
103
|
-
function wrapContinuationPrefix(state, node) {
|
|
104
|
-
const chain = [];
|
|
105
|
-
let cur = node.parent;
|
|
106
|
-
while (cur) {
|
|
107
|
-
chain.push(cur);
|
|
108
|
-
cur = cur.parent;
|
|
109
|
-
}
|
|
110
|
-
let p = "";
|
|
111
|
-
let liIdx = 0;
|
|
112
|
-
for (let i = chain.length - 1; i >= 0; i--) {
|
|
113
|
-
const tagId = chain[i].tagId;
|
|
114
|
-
if (tagId === 22) p += "> ";
|
|
115
|
-
else if (tagId === 25) {
|
|
116
|
-
const w = state.listIndentWidths[liIdx] ?? 2;
|
|
117
|
-
p += " ".repeat(w);
|
|
118
|
-
liIdx++;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
return p;
|
|
122
|
-
}
|
|
123
|
-
function wrapText(value, col, width, prefix) {
|
|
124
|
-
const leading = value.charCodeAt(0) === 32;
|
|
125
|
-
const trailing = value.charCodeAt(value.length - 1) === 32;
|
|
126
|
-
const prefixLen = [...prefix].length;
|
|
127
|
-
let out = "";
|
|
128
|
-
let first = true;
|
|
129
|
-
let i = 0;
|
|
130
|
-
const len = value.length;
|
|
131
|
-
while (i < len) {
|
|
132
|
-
let next = value.indexOf(" ", i);
|
|
133
|
-
if (next === -1) next = len;
|
|
134
|
-
if (next > i) {
|
|
135
|
-
const word = value.slice(i, next);
|
|
136
|
-
const wordLen = [...word].length;
|
|
137
|
-
const needSpace = first ? leading : true;
|
|
138
|
-
if (needSpace && col > prefixLen && col + 1 + wordLen > width) {
|
|
139
|
-
out += `\n${prefix}`;
|
|
140
|
-
col = prefixLen;
|
|
141
|
-
} else if (needSpace) {
|
|
142
|
-
out += " ";
|
|
143
|
-
col += 1;
|
|
144
|
-
}
|
|
145
|
-
out += word;
|
|
146
|
-
col += wordLen;
|
|
147
|
-
first = false;
|
|
148
|
-
}
|
|
149
|
-
i = next + 1;
|
|
150
|
-
}
|
|
151
|
-
if (trailing && out !== "" && !out.endsWith(" ") && !out.endsWith("\n")) out += " ";
|
|
152
|
-
if (out === "" && (leading || trailing)) out = " ";
|
|
153
|
-
return out;
|
|
154
|
-
}
|
|
155
|
-
function calculateNewLineConfig(node) {
|
|
156
|
-
const tagId = node.tagId;
|
|
157
|
-
const depthMap = node.depthMap;
|
|
158
|
-
if (tagId !== 25 && (depthMap[25] || 0) > 0 || tagId !== 22 && (depthMap[22] || 0) > 0) return NO_SPACING;
|
|
159
|
-
const isBlockElement = tagId !== void 0 && (tagId >= 7 && tagId <= 12 || tagId === 35 || tagId === 36);
|
|
160
|
-
let currParent = node.parent;
|
|
161
|
-
while (currParent) {
|
|
162
|
-
if (currParent.tagHandler?.collapsesInnerWhiteSpace) {
|
|
163
|
-
if (isBlockElement && currParent.tagId === 37) {
|
|
164
|
-
currParent = currParent.parent;
|
|
165
|
-
continue;
|
|
166
|
-
}
|
|
167
|
-
return NO_SPACING;
|
|
168
|
-
}
|
|
169
|
-
currParent = currParent.parent;
|
|
170
|
-
}
|
|
171
|
-
if (node.tagHandler?.spacing) return node.tagHandler?.spacing;
|
|
172
|
-
if (tagId === -1) return NO_SPACING;
|
|
173
|
-
return DEFAULT_BLOCK_SPACING;
|
|
174
|
-
}
|
|
175
|
-
function hasNonWhitespace(value) {
|
|
176
|
-
for (let i = 0; i < value.length; i++) {
|
|
177
|
-
const c = value.charCodeAt(i);
|
|
178
|
-
if (c !== 32 && c !== 9 && c !== 10 && c !== 13) return true;
|
|
179
|
-
}
|
|
180
|
-
return false;
|
|
181
|
-
}
|
|
182
|
-
function flushPreFence(state) {
|
|
183
|
-
state.preFencePending = false;
|
|
184
|
-
state.preOwnFence = true;
|
|
185
|
-
const lang = state.preFenceLang || "";
|
|
186
|
-
const fence = (state.depthMap[25] || 0) > 0 ? `\n\n${state.listIndent}\`\`\`${lang}\n${state.listIndent}` : `\`\`\`${lang}\n`;
|
|
187
|
-
state.buffer.push(fence);
|
|
188
|
-
state.lastContentCache = fence;
|
|
189
|
-
}
|
|
190
|
-
function createMarkdownProcessor(options = {}, resolvedPlugins = [], tagOverrideHandlers) {
|
|
191
|
-
const state = {
|
|
192
|
-
options,
|
|
193
|
-
buffer: [],
|
|
194
|
-
depthMap: new Uint8Array(109),
|
|
195
|
-
listIndent: "",
|
|
196
|
-
listIndentWidths: []
|
|
197
|
-
};
|
|
198
|
-
let lastYieldedLength = 0;
|
|
199
|
-
function processEvent(event) {
|
|
200
|
-
const { type: eventType, node } = event;
|
|
201
|
-
const lastNode = state.lastNode;
|
|
202
|
-
state.lastNode = event.node;
|
|
203
|
-
state.depth = node.depth;
|
|
204
|
-
const buff = state.buffer;
|
|
205
|
-
if (state.preFencePending && eventType === 0) {
|
|
206
|
-
if (node.type === 1) {
|
|
207
|
-
const el = node;
|
|
208
|
-
if (el.tagId === 23 && el.parent?.tagId === 34) state.preFencePending = false;
|
|
209
|
-
else if (el.tagId !== 34) flushPreFence(state);
|
|
210
|
-
} else if (node.type === 2 && hasNonWhitespace(node.value)) flushPreFence(state);
|
|
211
|
-
}
|
|
212
|
-
const lastBuffEntry = buff.at(-1);
|
|
213
|
-
const lastChar = lastBuffEntry?.charAt(lastBuffEntry.length - 1) || "";
|
|
214
|
-
let secondLastChar;
|
|
215
|
-
if (lastBuffEntry && lastBuffEntry.length > 1) secondLastChar = lastBuffEntry.charAt(lastBuffEntry.length - 2);
|
|
216
|
-
else if (buff.length > 1) {
|
|
217
|
-
const prevBuff = buff[buff.length - 2];
|
|
218
|
-
if (prevBuff) secondLastChar = prevBuff.charAt(prevBuff.length - 1);
|
|
219
|
-
}
|
|
220
|
-
if (node.type === 2 && eventType === 0) {
|
|
221
|
-
const textNode = node;
|
|
222
|
-
if (textNode.value) {
|
|
223
|
-
if (textNode.excludedFromMarkdown) return;
|
|
224
|
-
if (state.preFencePending) return;
|
|
225
|
-
if (textNode.value === " " && lastChar === "\n") return;
|
|
226
|
-
if (shouldAddSpacingBeforeText(lastChar, lastNode, textNode)) textNode.value = ` ${textNode.value}`;
|
|
227
|
-
if ((state.depthMap[34] || 0) > 0 && (state.depthMap[25] || 0) > 0) {
|
|
228
|
-
const indent = state.listIndent;
|
|
229
|
-
let value = textNode.value.replace(/\n(?!\n|$)/g, `\n${indent}`);
|
|
230
|
-
if (lastChar === "\n" && value[0] && value[0] !== "\n") value = indent + value;
|
|
231
|
-
textNode.value = value;
|
|
232
|
-
}
|
|
233
|
-
const wrapWidth = state.options?.wrapWidth;
|
|
234
|
-
if (wrapWidth && canWrapHere(state.depthMap)) {
|
|
235
|
-
const wrapped = wrapText(textNode.value, currentColumn(state.buffer), wrapWidth, wrapContinuationPrefix(state, textNode));
|
|
236
|
-
state.buffer.push(wrapped);
|
|
237
|
-
state.lastContentCache = wrapped;
|
|
238
|
-
} else {
|
|
239
|
-
state.buffer.push(textNode.value);
|
|
240
|
-
state.lastContentCache = textNode.value;
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
state.lastTextNode = textNode;
|
|
244
|
-
return;
|
|
245
|
-
}
|
|
246
|
-
if (node.type !== 1) return;
|
|
247
|
-
const context = {
|
|
248
|
-
node,
|
|
249
|
-
state
|
|
250
|
-
};
|
|
251
|
-
const output = [];
|
|
252
|
-
const element = node;
|
|
253
|
-
if (element.pluginOutput?.length) {
|
|
254
|
-
output.push(...element.pluginOutput);
|
|
255
|
-
element.pluginOutput = [];
|
|
256
|
-
}
|
|
257
|
-
const lastFragment = state.lastContentCache;
|
|
258
|
-
let lastNewLines = 0;
|
|
259
|
-
if (lastChar === "\n") lastNewLines++;
|
|
260
|
-
if (secondLastChar === "\n") lastNewLines++;
|
|
261
|
-
const eventFn = eventType === 0 ? "enter" : "exit";
|
|
262
|
-
const handler = node.tagHandler;
|
|
263
|
-
if (!output.length && handler?.[eventFn]) {
|
|
264
|
-
const res = handler[eventFn](context);
|
|
265
|
-
if (res) output.push(res);
|
|
266
|
-
}
|
|
267
|
-
const configuredNewLines = calculateNewLineConfig(node)[eventType] || 0;
|
|
268
|
-
const newLines = Math.max(0, configuredNewLines - lastNewLines);
|
|
269
|
-
if (newLines > 0) {
|
|
270
|
-
if (!buff.length) {
|
|
271
|
-
for (const fragment of output) if (fragment) {
|
|
272
|
-
state.buffer.push(fragment);
|
|
273
|
-
state.lastContentCache = fragment;
|
|
274
|
-
}
|
|
275
|
-
updateListIndent(state, element, eventType);
|
|
276
|
-
return;
|
|
277
|
-
}
|
|
278
|
-
const newlinesStr = "\n".repeat(newLines);
|
|
279
|
-
if (lastChar === " " && buff?.length) buff[buff.length - 1] = buff.at(-1).substring(0, buff.at(-1).length - 1);
|
|
280
|
-
if (eventType === 0) output.unshift(newlinesStr);
|
|
281
|
-
else output.push(newlinesStr);
|
|
282
|
-
} else if (lastFragment && state.lastTextNode?.containsWhitespace && !!node.parent && "value" in state.lastTextNode && typeof state.lastTextNode.value === "string") {
|
|
283
|
-
if (!node.parent.depthMap[34] || node.parent.tagId === 34) {
|
|
284
|
-
const isInlineElement = node.tagHandler?.isInline;
|
|
285
|
-
const collapsesWhiteSpace = node.tagHandler?.collapsesInnerWhiteSpace;
|
|
286
|
-
const hasSpacing = node.tagHandler?.spacing && Array.isArray(node.tagHandler.spacing);
|
|
287
|
-
if ((!isInlineElement || eventType === 1) && !(!isInlineElement && !collapsesWhiteSpace && configuredNewLines > 0) && !(collapsesWhiteSpace && eventType === 0) && !(hasSpacing && eventType === 0)) {
|
|
288
|
-
const originalLength = lastFragment.length;
|
|
289
|
-
const trimmed = lastFragment.trimEnd();
|
|
290
|
-
if (originalLength - trimmed.length > 0) {
|
|
291
|
-
if (buff?.length && buff.at(-1) === lastFragment) buff[buff.length - 1] = trimmed;
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
state.lastTextNode = void 0;
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
if (output[0]?.[0] && eventType === 0 && !node.tagHandler?.literalEnter && lastChar && needsSpacing(lastChar, output[0][0], state)) {
|
|
298
|
-
state.buffer.push(" ");
|
|
299
|
-
state.lastContentCache = " ";
|
|
300
|
-
}
|
|
301
|
-
for (const fragment of output) if (fragment) {
|
|
302
|
-
state.buffer.push(fragment);
|
|
303
|
-
state.lastContentCache = fragment;
|
|
304
|
-
}
|
|
305
|
-
updateListIndent(state, element, eventType);
|
|
306
|
-
}
|
|
307
|
-
function processHtml(html) {
|
|
308
|
-
parseHtmlStream(html, {
|
|
309
|
-
depthMap: state.depthMap,
|
|
310
|
-
depth: 0,
|
|
311
|
-
resolvedPlugins,
|
|
312
|
-
tagOverrideHandlers
|
|
313
|
-
}, (event) => {
|
|
314
|
-
processPluginsForEvent(event, resolvedPlugins, state, processEvent);
|
|
315
|
-
});
|
|
316
|
-
}
|
|
317
|
-
function getMarkdown() {
|
|
318
|
-
const result = state.buffer.join("").trimStart();
|
|
319
|
-
state.buffer.length = 0;
|
|
320
|
-
return result.trimEnd();
|
|
321
|
-
}
|
|
322
|
-
function getMarkdownChunk() {
|
|
323
|
-
const currentContent = state.buffer.join("").trimStart();
|
|
324
|
-
const newContent = currentContent.slice(lastYieldedLength);
|
|
325
|
-
lastYieldedLength = currentContent.length;
|
|
326
|
-
if (state.buffer.length > 1) {
|
|
327
|
-
state.buffer.length = 0;
|
|
328
|
-
state.buffer.push(currentContent);
|
|
329
|
-
}
|
|
330
|
-
return newContent;
|
|
331
|
-
}
|
|
332
|
-
return {
|
|
333
|
-
processEvent,
|
|
334
|
-
processHtml,
|
|
335
|
-
getMarkdown,
|
|
336
|
-
getMarkdownChunk,
|
|
337
|
-
state
|
|
338
|
-
};
|
|
339
|
-
}
|
|
340
2
|
function resolveFrontmatterOpt(opt) {
|
|
341
3
|
if (typeof opt === "function") return {
|
|
342
4
|
config: {},
|
|
@@ -379,4 +41,4 @@ function resolvePlugins(options, hooks) {
|
|
|
379
41
|
frontmatterCallback
|
|
380
42
|
};
|
|
381
43
|
}
|
|
382
|
-
export {
|
|
44
|
+
export { resolvePlugins };
|
package/dist/_chunks/src.mjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import "./const.mjs";
|
|
2
|
-
import { buildTagOverrideHandlers
|
|
3
|
-
import { createMarkdownProcessor
|
|
2
|
+
import { buildTagOverrideHandlers } from "./parse.mjs";
|
|
3
|
+
import { createMarkdownProcessor } from "./markdown-processor.mjs";
|
|
4
4
|
import "./plugins.mjs";
|
|
5
|
+
import { resolvePlugins } from "./resolve-plugins.mjs";
|
|
6
|
+
import { streamHtmlToMarkdown as streamHtmlToMarkdown$1 } from "./stream.mjs";
|
|
5
7
|
import "../preset/minimal.mjs";
|
|
6
8
|
function resolveClean(clean) {
|
|
7
9
|
if (clean === true) return {
|
|
@@ -288,38 +290,6 @@ function applyClean(md, opts) {
|
|
|
288
290
|
if (opts.fragments) md = cleanFragments(md);
|
|
289
291
|
return md;
|
|
290
292
|
}
|
|
291
|
-
async function* streamHtmlToMarkdown$1(htmlStream, options = {}, resolvedPlugins = [], tagOverrideHandlers) {
|
|
292
|
-
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
293
|
-
const decoder = new TextDecoder();
|
|
294
|
-
const reader = htmlStream.getReader();
|
|
295
|
-
const processor = createMarkdownProcessor(options, resolvedPlugins, tagOverrideHandlers);
|
|
296
|
-
const parseState = {
|
|
297
|
-
depthMap: processor.state.depthMap,
|
|
298
|
-
depth: 0,
|
|
299
|
-
resolvedPlugins,
|
|
300
|
-
tagOverrideHandlers
|
|
301
|
-
};
|
|
302
|
-
let remainingHtml = "";
|
|
303
|
-
try {
|
|
304
|
-
while (true) {
|
|
305
|
-
const { done, value } = await reader.read();
|
|
306
|
-
if (done) break;
|
|
307
|
-
remainingHtml = parseHtmlStream(`${remainingHtml}${typeof value === "string" ? value : decoder.decode(value, { stream: true })}`, parseState, (event) => {
|
|
308
|
-
processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent);
|
|
309
|
-
});
|
|
310
|
-
const chunk = processor.getMarkdownChunk();
|
|
311
|
-
if (chunk) yield chunk;
|
|
312
|
-
}
|
|
313
|
-
if (remainingHtml) parseHtmlStream(remainingHtml, parseState, (event) => {
|
|
314
|
-
processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent);
|
|
315
|
-
});
|
|
316
|
-
const finalChunk = processor.getMarkdownChunk();
|
|
317
|
-
if (finalChunk) yield finalChunk;
|
|
318
|
-
} finally {
|
|
319
|
-
if (remainingHtml) decoder.decode(new Uint8Array(0), { stream: false });
|
|
320
|
-
reader.releaseLock();
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
293
|
function resolveHooks(options) {
|
|
324
294
|
return options.hooks?.length ? options.hooks : void 0;
|
|
325
295
|
}
|
|
@@ -336,7 +306,7 @@ function convert(html, options, hooks) {
|
|
|
336
306
|
}
|
|
337
307
|
function htmlToMarkdown(html, options = {}) {
|
|
338
308
|
const markdown = convert(html, options, resolveHooks(options));
|
|
339
|
-
if (options.clean) return applyClean(markdown, resolveClean(options.clean));
|
|
309
|
+
if (options.clean && options.format !== "text") return applyClean(markdown, resolveClean(options.clean));
|
|
340
310
|
return markdown;
|
|
341
311
|
}
|
|
342
312
|
function streamHtmlToMarkdown(htmlStream, options = {}) {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { finalizeParse, parseHtmlStream } from "./parse.mjs";
|
|
2
|
+
import { createMarkdownProcessor, processPluginsForEvent } from "./markdown-processor.mjs";
|
|
3
|
+
async function* streamHtmlToMarkdown(htmlStream, options = {}, resolvedPlugins = [], tagOverrideHandlers) {
|
|
4
|
+
if (!htmlStream) throw new Error("Invalid HTML stream provided");
|
|
5
|
+
const decoder = new TextDecoder();
|
|
6
|
+
const reader = htmlStream.getReader();
|
|
7
|
+
const processor = createMarkdownProcessor(options, resolvedPlugins, tagOverrideHandlers);
|
|
8
|
+
const parseState = {
|
|
9
|
+
depthMap: processor.state.depthMap,
|
|
10
|
+
depth: 0,
|
|
11
|
+
resolvedPlugins,
|
|
12
|
+
tagOverrideHandlers,
|
|
13
|
+
plainText: processor.state.plainText
|
|
14
|
+
};
|
|
15
|
+
const handleEvent = resolvedPlugins.length ? (event) => processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
|
|
16
|
+
let remainingHtml = "";
|
|
17
|
+
try {
|
|
18
|
+
while (true) {
|
|
19
|
+
const { done, value } = await reader.read();
|
|
20
|
+
if (done) break;
|
|
21
|
+
const decoded = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
|
|
22
|
+
remainingHtml = parseHtmlStream(`${remainingHtml}${decoded}`, parseState, handleEvent);
|
|
23
|
+
const chunk = processor.getMarkdownChunk();
|
|
24
|
+
if (chunk) yield chunk;
|
|
25
|
+
}
|
|
26
|
+
const decoderTail = decoder.decode();
|
|
27
|
+
const finalHtml = remainingHtml + decoderTail;
|
|
28
|
+
finalizeParse(finalHtml ? parseHtmlStream(finalHtml, parseState, handleEvent) : "", parseState, handleEvent);
|
|
29
|
+
const finalChunk = processor.getMarkdownChunk();
|
|
30
|
+
if (finalChunk) yield finalChunk;
|
|
31
|
+
} finally {
|
|
32
|
+
reader.releaseLock();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export { streamHtmlToMarkdown };
|
package/dist/_chunks/types.d.mts
CHANGED
|
@@ -144,6 +144,11 @@ interface EngineOptions {
|
|
|
144
144
|
* wrapping.
|
|
145
145
|
*/
|
|
146
146
|
wrapWidth?: number;
|
|
147
|
+
/**
|
|
148
|
+
* Output format. Defaults to `markdown`; use `text` to omit Markdown/HTML
|
|
149
|
+
* markup while preserving readable text and block spacing.
|
|
150
|
+
*/
|
|
151
|
+
format?: 'markdown' | 'text';
|
|
147
152
|
}
|
|
148
153
|
interface ElementNode extends Node {
|
|
149
154
|
/** Element tag name (for ELEMENT_NODE) */
|
|
@@ -164,8 +169,6 @@ interface TextNode extends Node {
|
|
|
164
169
|
value: string;
|
|
165
170
|
/** Custom data added by plugins */
|
|
166
171
|
context?: PluginContext;
|
|
167
|
-
/** Whether this text node should be excluded from markdown output (for script/style elements) */
|
|
168
|
-
excludedFromMarkdown?: boolean;
|
|
169
172
|
}
|
|
170
173
|
/**
|
|
171
174
|
* Base DOM node interface
|
|
@@ -185,6 +188,8 @@ interface Node {
|
|
|
185
188
|
childTextNodeIndex?: number;
|
|
186
189
|
/** Whether node contains whitespace - used for whitespace optimization */
|
|
187
190
|
containsWhitespace?: boolean;
|
|
191
|
+
/** Whether this node belongs to an inert subtree and should not render */
|
|
192
|
+
excludedFromMarkdown?: boolean;
|
|
188
193
|
/** Cached reference to tag handler for performance */
|
|
189
194
|
tagHandler?: TagHandler;
|
|
190
195
|
/** Parent node */
|
|
@@ -218,11 +223,13 @@ interface MdreamProcessingState {
|
|
|
218
223
|
isFirstTextInElement?: boolean;
|
|
219
224
|
/** Reference to the last processed text node - for context tracking */
|
|
220
225
|
lastTextNode?: Node;
|
|
221
|
-
/**
|
|
226
|
+
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
222
227
|
inSingleQuote?: boolean;
|
|
228
|
+
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
223
229
|
inDoubleQuote?: boolean;
|
|
230
|
+
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
224
231
|
inBacktick?: boolean;
|
|
225
|
-
/**
|
|
232
|
+
/** @deprecated No longer read or written. Retained for source compatibility. */
|
|
226
233
|
lastCharWasBackslash?: boolean;
|
|
227
234
|
/** Resolved plugin instances for efficient iteration */
|
|
228
235
|
resolvedPlugins?: TransformPlugin[];
|
|
@@ -264,6 +271,8 @@ interface MdreamRuntimeState extends Partial<MdreamProcessingState> {
|
|
|
264
271
|
preFencePending?: boolean;
|
|
265
272
|
preFenceLang?: string;
|
|
266
273
|
preOwnFence?: boolean;
|
|
274
|
+
/** Whether output should omit Markdown/HTML markup */
|
|
275
|
+
plainText?: boolean;
|
|
267
276
|
}
|
|
268
277
|
type NodeEventEnter = 0;
|
|
269
278
|
type NodeEventExit = 1;
|
|
@@ -308,6 +317,11 @@ interface TagHandler {
|
|
|
308
317
|
* strings so markers like `^`/`~` attach to adjacent content (issue #93).
|
|
309
318
|
*/
|
|
310
319
|
literalEnter?: boolean;
|
|
320
|
+
/**
|
|
321
|
+
* Built-in tag id used by declarative string aliases.
|
|
322
|
+
* @internal
|
|
323
|
+
*/
|
|
324
|
+
aliasTagId?: number;
|
|
311
325
|
}
|
|
312
326
|
interface TailwindContext {
|
|
313
327
|
hidden?: boolean;
|
|
@@ -356,8 +370,11 @@ interface MarkdownChunk {
|
|
|
356
370
|
content: string;
|
|
357
371
|
/** Metadata extracted during chunking */
|
|
358
372
|
metadata: {
|
|
359
|
-
/** Header hierarchy at this chunk position */
|
|
360
|
-
|
|
373
|
+
/** Header hierarchy at this chunk position */
|
|
374
|
+
headers?: Record<string, string>;
|
|
375
|
+
/** Code block language if chunk is/contains code */
|
|
376
|
+
code?: string;
|
|
377
|
+
/** Line number range in original document */
|
|
361
378
|
loc?: {
|
|
362
379
|
lines: {
|
|
363
380
|
from: number;
|
package/dist/cli.mjs
CHANGED
|
@@ -6,9 +6,16 @@ import { fileURLToPath } from "node:url";
|
|
|
6
6
|
import { cac } from "cac";
|
|
7
7
|
import { dirname, join } from "pathe";
|
|
8
8
|
async function streamingConvert(options = {}) {
|
|
9
|
+
const format = options.text ? "text" : options.format;
|
|
10
|
+
if (format && format !== "markdown" && format !== "text") {
|
|
11
|
+
process.stderr.write(`Unknown format: ${format}\n`);
|
|
12
|
+
process.exitCode = 1;
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
9
15
|
let conversionOptions = {
|
|
10
16
|
origin: options.origin,
|
|
11
|
-
wrapWidth: options.wrapWidth ? Number(options.wrapWidth) || void 0 : void 0
|
|
17
|
+
wrapWidth: options.wrapWidth ? Number(options.wrapWidth) || void 0 : void 0,
|
|
18
|
+
format
|
|
12
19
|
};
|
|
13
20
|
if (options.preset === "minimal") conversionOptions = withMinimalPreset(conversionOptions);
|
|
14
21
|
const markdownGenerator = streamHtmlToMarkdown(Readable.toWeb(process.stdin), conversionOptions);
|
|
@@ -17,7 +24,7 @@ async function streamingConvert(options = {}) {
|
|
|
17
24
|
const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
|
|
18
25
|
const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
|
|
19
26
|
const cli = cac();
|
|
20
|
-
cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").option("--wrap-width <n>", "Hard-wrap prose at <n> characters on word boundaries").action(async (_, opts) => {
|
|
27
|
+
cli.command("[options]", "Convert HTML from stdin to Markdown on stdout (JS engine)").option("--origin <url>", "Origin URL for resolving relative image paths").option("--preset <preset>", "Conversion presets: minimal").option("--wrap-width <n>", "Hard-wrap prose at <n> characters on word boundaries").option("--format <format>", "Output format: markdown, text").option("--text", "Alias for --format text").action(async (_, opts) => {
|
|
21
28
|
await streamingConvert(opts);
|
|
22
29
|
});
|
|
23
30
|
cli.help().version(packageJson.version).parse();
|
package/dist/core.d.mts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { BuiltinPlugins, MdreamOptions, TransformPlugin } from "./_chunks/types.mjs";
|
|
2
|
+
/** Imperative plugin used by the tree-shakable converter. */
|
|
3
|
+
type Plugin = TransformPlugin;
|
|
4
|
+
/**
|
|
5
|
+
* Options for the tree-shakable conversion entry. Import plugin factories
|
|
6
|
+
* explicitly and pass their results as an array, matching the pre-v1 JS API.
|
|
7
|
+
* Declarative built-in plugin config and cleanup live on the package root.
|
|
8
|
+
*/
|
|
9
|
+
type CoreOptions = Omit<MdreamOptions, 'plugins' | 'clean'> & {
|
|
10
|
+
plugins?: Plugin[] | Pick<BuiltinPlugins, 'tagOverrides'>;
|
|
11
|
+
};
|
|
12
|
+
/** Convert HTML without bundling the optional declarative built-in plugins. */
|
|
13
|
+
declare function htmlToMarkdown(html: string, options?: CoreOptions): string;
|
|
14
|
+
/** Stream HTML conversion without bundling declarative built-in plugins. */
|
|
15
|
+
declare function streamHtmlToMarkdown(htmlStream: ReadableStream<Uint8Array | string> | null, options?: CoreOptions): AsyncIterable<string>;
|
|
16
|
+
export { CoreOptions, Plugin, htmlToMarkdown, streamHtmlToMarkdown };
|