@mdream/js 2.0.3 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
1
  import "./const.mjs";
2
- import { isInsideHeading, trimOutputStart } from "./utils.mjs";
2
+ import { isInsideHeading, noteBufferRewrite, outputLeadLength, trimOutputStart } from "./url.mjs";
3
3
  import { createMarkdownProcessor, isAsciiWhitespace, lineOpensRawHtmlBlock, stripBlockquoteListIndent, tagHandlers, trimAsciiWhitespaceEnd } from "./markdown-processor.mjs";
4
4
  import { assertEngineOptions, buildTagOverrideHandlers, checkClean, endPlugins, processPluginsForEvent } from "./tag-overrides.mjs";
5
- import { finalizeParse, parseHtmlStream } from "./parse.mjs";
5
+ import { commitPlainTextPrefix, finalizeParse, parseHtmlStream } from "./parse.mjs";
6
6
  import { resolvePlugins } from "./plugin.mjs";
7
7
  function resetBufferScanCursors(scan) {
8
8
  scan[1] = 0;
@@ -63,8 +63,10 @@ function flushBlockquoteLines(state) {
63
63
  for (const fragment of tail) buffer.push(fragment);
64
64
  for (const frame of frames) {
65
65
  frame.fragment = start + 1;
66
+ frame.itemSeparatorFragments = 0;
66
67
  frame.followsWhitespace = true;
67
68
  }
69
+ noteBufferRewrite(state);
68
70
  return true;
69
71
  }
70
72
  function prepareQuoteOutput(state, bufferScan, quoteScan) {
@@ -128,18 +130,24 @@ function createMarkdownDrain(context, _hasPlugins) {
128
130
  const blankLine = content.indexOf("\n\n", Math.max(0, start - 1));
129
131
  if (blankLine !== -1 && blankLine + 2 <= end) bufferScan[0] = true;
130
132
  }
131
- function compactQuotePrefix(fragment, content, start) {
132
- if (state.blockquotes.length === 0 || start <= 0 || state.outputPositions || context.getNonQuoteHeldOutputFragment() !== Infinity) return 0;
133
- const retained = content.slice(start);
134
- state.buffer.splice(0, fragment, retained);
135
- for (const frame of state.blockquotes) frame.fragment -= fragment - 1;
133
+ function compactHeldPrefix(fragment, content, start) {
134
+ if (start <= 0) return 0;
135
+ if (state.blockquotes.length > 0) {
136
+ if (state.outputPositions || context.getNonQuoteHeldOutputFragment() !== Infinity) return 0;
137
+ state.buffer.splice(0, fragment, content.slice(start));
138
+ noteBufferRewrite(state);
139
+ for (const frame of state.blockquotes) frame.fragment -= fragment - 1;
140
+ } else if (!context.compactPrefix(fragment, content.slice(start))) return 0;
136
141
  quoteScan = void 0;
137
- bufferScan[1] = 0;
142
+ bufferScan[1] = Math.max(0, bufferScan[1] - fragment + 1);
138
143
  bufferScan[2] = 1;
139
144
  bufferScan[3] = bufferScan[5] === void 0 ? 1 : 0;
140
145
  bufferScan[4] = 0;
141
146
  return start;
142
147
  }
148
+ function noteDrained(content, end) {
149
+ if (!state.drainedContent && end > outputLeadLength(content)) state.drainedContent = true;
150
+ }
143
151
  function getMarkdownChunk(final = false) {
144
152
  if (context.holdsOutput) return final ? context.getMarkdown() : "";
145
153
  const heldFragment = prepareDrain(final);
@@ -160,9 +168,14 @@ function createMarkdownDrain(context, _hasPlugins) {
160
168
  retainMutableFragments = stableLength < currentContent.length;
161
169
  }
162
170
  } else {
171
+ let holdVerticalTabs;
163
172
  while (stableLength > 0) {
164
173
  const code = currentContent.charCodeAt(stableLength - 1);
165
- if (code !== 32 && code !== 10) break;
174
+ if (!isAsciiWhitespace(code)) {
175
+ if (code !== 11) break;
176
+ holdVerticalTabs ??= !final && !!(state.depthMap[16] || state.depthMap[17] || state.depthMap[14] || state.depthMap[15] || state.depthMap[90] || state.depthMap[91] || state.depthMap[18] || state.depthMap[110] || state.depthMap[111]);
177
+ if (!holdVerticalTabs) break;
178
+ }
166
179
  stableLength--;
167
180
  }
168
181
  retainMutableFragments = stableLength < currentContent.length;
@@ -185,11 +198,12 @@ function createMarkdownDrain(context, _hasPlugins) {
185
198
  context.markYielded();
186
199
  }
187
200
  const wrapping = !!options.wrapWidth;
188
- if (hasYieldedContent && fragmentHeld && state.blockquotes.length > 0 && !headingHeld && !inPre && (!wrapping || leadingTrimmed === 0)) {
201
+ if (hasYieldedContent && fragmentHeld && !headingHeld && !inPre && (!wrapping || leadingTrimmed === 0)) {
189
202
  observeStableOutput(content, stableLength + leadingTrimmed);
190
203
  let retainedStart = stableLength + leadingTrimmed - 17;
191
204
  if (wrapping) retainedStart = Math.min(retainedStart, content.lastIndexOf("\n", stableLength + leadingTrimmed - 1));
192
- const removed = compactQuotePrefix(heldFragment, content, Math.max(0, retainedStart));
205
+ const removed = compactHeldPrefix(heldFragment, content, Math.max(0, retainedStart));
206
+ noteDrained(content, removed);
193
207
  lastYieldedLength -= removed;
194
208
  }
195
209
  if (hasYieldedContent && !fragmentHeld && !headingHeld && (!retainMutableFragments || !inPre) && (!wrapping || leadingTrimmed === 0)) {
@@ -204,25 +218,41 @@ function createMarkdownDrain(context, _hasPlugins) {
204
218
  const fragmentStart = currentContent.length - lastFragment.length;
205
219
  const tailStart = Math.max(0, Math.min(contextStart, fragmentStart));
206
220
  const emittedTail = currentContent.slice(tailStart, fragmentStart);
221
+ const stableTailLength = Math.max(0, Math.min(stableLength, fragmentStart) - tailStart);
222
+ const stableTail = emittedTail.slice(0, stableTailLength);
223
+ const mutableTail = emittedTail.slice(stableTailLength);
224
+ noteDrained(currentContent, tailStart);
207
225
  state.buffer.length = 0;
208
226
  resetBufferScanCursors(bufferScan);
209
- if (emittedTail) state.buffer.push(emittedTail);
227
+ if (stableTail) state.buffer.push(stableTail);
228
+ bufferScan[1] = state.buffer.length;
229
+ if (mutableTail) state.buffer.push(mutableTail);
210
230
  state.buffer.push(lastFragment);
231
+ noteBufferRewrite(state);
211
232
  lastYieldedLength = stableLength - tailStart;
212
233
  } else if (!retainMutableFragments) {
213
234
  const tailStart = Math.max(0, contextStart);
214
235
  const emittedTail = currentContent.slice(tailStart, stableLength);
236
+ noteDrained(currentContent, tailStart);
215
237
  state.buffer.length = 0;
216
238
  resetBufferScanCursors(bufferScan);
217
239
  if (emittedTail) state.buffer.push(emittedTail);
240
+ noteBufferRewrite(state);
218
241
  lastYieldedLength = emittedTail.length;
219
242
  }
243
+ if (!retainMutableFragments) bufferScan[1] = state.buffer.length;
220
244
  bufferScan[2] = state.buffer.length;
221
245
  if (bufferScan[5] === void 0) bufferScan[3] = state.buffer.length;
222
246
  }
223
247
  return newContent;
224
248
  }
225
- return { getMarkdownChunk };
249
+ function drainKeepingDroppedPair(final = false) {
250
+ const pending = state.droppedPairAt === state.buffer.length;
251
+ const chunk = getMarkdownChunk(final);
252
+ state.droppedPairAt = pending ? state.buffer.length : void 0;
253
+ return chunk;
254
+ }
255
+ return { getMarkdownChunk: drainKeepingDroppedPair };
226
256
  }
227
257
  async function* streamHtmlToMarkdown$1(htmlStream, options = {}, resolvedPlugins = [], tagOverrideHandlers) {
228
258
  if (!htmlStream) throw new Error("Invalid HTML stream provided");
@@ -236,9 +266,11 @@ async function* streamHtmlToMarkdown$1(htmlStream, options = {}, resolvedPlugins
236
266
  tagHandlers,
237
267
  tagOverrideHandlers,
238
268
  plainText: false,
269
+ urlBaseTarget: processor.state,
239
270
  discardExcludedText: resolvedPlugins.length === 0
240
271
  };
241
- const handleEvent = resolvedPlugins.length ? (event) => processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent) : processor.processEvent;
272
+ const handleEvent = resolvedPlugins.length ? (event) => processPluginsForEvent(event, resolvedPlugins, processor.state, processor.processEvent, processor.skipEvent) : processor.processEvent;
273
+ const incrementalPlainText = resolvedPlugins.length === 0 && !options.wrapWidth && !tagOverrideHandlers?.size;
242
274
  let remainingHtml = "";
243
275
  try {
244
276
  while (true) {
@@ -248,6 +280,7 @@ async function* streamHtmlToMarkdown$1(htmlStream, options = {}, resolvedPlugins
248
280
  const decoded = typeof value === "string" ? decoder.decode() + value : decoder.decode(value, { stream: true });
249
281
  const htmlContent = `${remainingHtml}${decoded}`;
250
282
  remainingHtml = parseHtmlStream(htmlContent, parseState, handleEvent);
283
+ if (incrementalPlainText) remainingHtml = commitPlainTextPrefix(remainingHtml, parseState, handleEvent);
251
284
  const chunk = processor.getMarkdownChunk();
252
285
  if (chunk) yield chunk;
253
286
  }
@@ -274,9 +307,10 @@ function htmlToMarkdown(html, options = {}) {
274
307
  resolvedPlugins: plugins,
275
308
  tagHandlers,
276
309
  tagOverrideHandlers,
277
- plainText: false
310
+ plainText: false,
311
+ urlBaseTarget: processor.state
278
312
  };
279
- const handleEvent = plugins.length ? (event) => processPluginsForEvent(event, plugins, processor.state, processor.processEvent) : processor.processEvent;
313
+ const handleEvent = plugins.length ? (event) => processPluginsForEvent(event, plugins, processor.state, processor.processEvent, processor.skipEvent) : processor.processEvent;
280
314
  const leftover = parseHtmlStream(html, parseState, handleEvent);
281
315
  finalizeParse(leftover, parseState, handleEvent);
282
316
  endPlugins(plugins, processor.state);
@@ -51,15 +51,22 @@ function assertEngineOptions(options, entry = "converter") {
51
51
  const plugins = options.plugins;
52
52
  if (plugins != null && !Array.isArray(plugins)) throw new TypeError("@mdream/js takes `plugins` as an array, such as { plugins: [frontmatterPlugin()] } with frontmatterPlugin from '@mdream/js/plugins'.");
53
53
  }
54
- function processPluginsForEvent(event, plugins, state, processEvent) {
54
+ function processPluginsForEvent(event, plugins, state, processEvent, onSkipped) {
55
55
  state.depth = event.node.depth;
56
56
  if (plugins?.length) {
57
57
  if (event.node.type === 1 && event.type === 0) {
58
58
  const element = event.node;
59
+ let hasCheckedAttributes = false;
59
60
  for (let index = 0; index < plugins.length; index++) {
60
61
  const plugin = plugins[index];
61
62
  if (plugin === void 0) throw new TypeError("Plugin arrays must not contain undefined entries.");
62
- if (plugin.processAttributes) plugin.processAttributes(element, state);
63
+ if (plugin.processAttributes) {
64
+ if (!hasCheckedAttributes) {
65
+ hasCheckedAttributes = true;
66
+ if (!Object.isExtensible(element.attributes)) element.attributes = { ...element.attributes };
67
+ }
68
+ plugin.processAttributes(element, state);
69
+ }
63
70
  }
64
71
  }
65
72
  let shouldSkip = false;
@@ -69,7 +76,10 @@ function processPluginsForEvent(event, plugins, state, processEvent) {
69
76
  const res = plugin.beforeNodeProcess?.(event, state);
70
77
  if (typeof res === "object") shouldSkip = res.skip;
71
78
  }
72
- if (shouldSkip) return true;
79
+ if (shouldSkip) {
80
+ onSkipped?.(event);
81
+ return true;
82
+ }
73
83
  if (event.node.type === 1) {
74
84
  const element = event.node;
75
85
  const fn = event.type === 0 ? "onNodeEnter" : "onNodeExit";
@@ -91,7 +101,10 @@ function processPluginsForEvent(event, plugins, state, processEvent) {
91
101
  if (plugin.processTextNode) {
92
102
  const result = plugin.processTextNode(textNode, state);
93
103
  if (result) {
94
- if (result.skip) return true;
104
+ if (result.skip) {
105
+ textNode.excludedFromMarkdown = true;
106
+ return true;
107
+ }
95
108
  textNode.value = result.content;
96
109
  }
97
110
  }
@@ -124,11 +137,14 @@ function buildTagOverrideHandlers(overrides, tagHandlers) {
124
137
  }
125
138
  const baseId = TagIdMap[tagName];
126
139
  const baseHandler = baseId === void 0 ? void 0 : tagHandlers[baseId];
127
- const handler = baseHandler ? { ...baseHandler } : {};
140
+ const handler = baseHandler ? { ...baseHandler } : baseId === void 0 ? { isInline: true } : {};
128
141
  if (override.enter !== void 0) {
129
142
  const output = override.enter;
130
143
  handler.enter = () => output;
131
144
  handler.literalEnter = true;
145
+ let hashes = 0;
146
+ while (hashes < 7 && output.charCodeAt(hashes) === 35) hashes++;
147
+ handler.literalEnterHeading = hashes > 0 && hashes <= 6 && (output.charCodeAt(hashes) === 32 || output.charCodeAt(hashes) === 9);
132
148
  }
133
149
  if (override.exit !== void 0) {
134
150
  const output = override.exit;
@@ -13,21 +13,64 @@ function createIdSelector(selector) {
13
13
  toString: () => `#${id}`
14
14
  };
15
15
  }
16
+ function containsWhitespaceSeparatedToken(value, token) {
17
+ if (!token) return false;
18
+ let index = value.indexOf(token);
19
+ if (index === -1) return false;
20
+ for (let i = 0; i < token.length; i++) if (isCssWhitespace(token.charCodeAt(i))) return false;
21
+ while (index !== -1) {
22
+ const before = value.charCodeAt(index - 1);
23
+ if (index === 0 || isCssWhitespace(before)) {
24
+ const end = index + token.length;
25
+ const after = value.charCodeAt(end);
26
+ if (end === value.length || isCssWhitespace(after)) return true;
27
+ }
28
+ index = value.indexOf(token, index + 1);
29
+ }
30
+ return false;
31
+ }
16
32
  function createClassSelector(selector) {
17
33
  const className = selector.slice(1);
18
34
  return {
19
35
  matches: (element) => {
20
36
  if (!element.attributes?.class) return false;
21
- return element.attributes.class.trim().split(" ").filter(Boolean).includes(className);
37
+ return containsWhitespaceSeparatedToken(element.attributes.class, className);
22
38
  },
23
39
  toString: () => `.${className}`
24
40
  };
25
41
  }
42
+ function findAttributeEnd(selector, start) {
43
+ let quote = 0;
44
+ for (let i = start + 1; i < selector.length; i++) {
45
+ const char = selector.charCodeAt(i);
46
+ if (char === 92) {
47
+ i++;
48
+ continue;
49
+ }
50
+ if (quote) {
51
+ if (char === quote) quote = 0;
52
+ continue;
53
+ }
54
+ if (char === 34 || char === 39) quote = char;
55
+ else if (char === 93) return i;
56
+ }
57
+ return -1;
58
+ }
59
+ function isCssWhitespace(code) {
60
+ return code === 32 || code === 9 || code === 10 || code === 12 || code === 13;
61
+ }
62
+ function trimCssWhitespace(value) {
63
+ let start = 0;
64
+ let end = value.length;
65
+ while (start < end && isCssWhitespace(value.charCodeAt(start))) start++;
66
+ while (end > start && isCssWhitespace(value.charCodeAt(end - 1))) end--;
67
+ return value.slice(start, end);
68
+ }
26
69
  function parseAttributeSelector(selector) {
27
70
  if (selector.charCodeAt(0) !== 91) return null;
28
- const end = selector.indexOf("]", 1);
71
+ const end = findAttributeEnd(selector, 0);
29
72
  if (end === -1) return null;
30
- const inner = selector.slice(1, end);
73
+ const inner = trimCssWhitespace(selector.slice(1, end));
31
74
  let opIdx = -1;
32
75
  for (let i = 0; i < inner.length; i++) {
33
76
  const c = inner.charCodeAt(i);
@@ -37,12 +80,12 @@ function parseAttributeSelector(selector) {
37
80
  }
38
81
  }
39
82
  if (opIdx === -1) return { attr: inner };
40
- const attr = inner.slice(0, opIdx);
83
+ const attr = trimCssWhitespace(inner.slice(0, opIdx));
41
84
  let opEnd = opIdx + 1;
42
85
  if (opEnd < inner.length && inner.charCodeAt(opEnd) === 61) opEnd++;
43
86
  const op = inner.slice(opIdx, opEnd);
44
- let value = inner.slice(opEnd);
45
- if ((value.charCodeAt(0) === 34 || value.charCodeAt(0) === 39) && value.charCodeAt(value.length - 1) === value.charCodeAt(0)) value = value.slice(1, -1);
87
+ let value = trimCssWhitespace(inner.slice(opEnd));
88
+ if (value.length >= 2 && (value.charCodeAt(0) === 34 || value.charCodeAt(0) === 39) && value.charCodeAt(value.length - 1) === value.charCodeAt(0)) value = value.slice(1, -1);
46
89
  return {
47
90
  attr,
48
91
  op,
@@ -57,21 +100,21 @@ function createAttributeSelector(selector) {
57
100
  return {
58
101
  matches: (element) => {
59
102
  if (!(attrName in (element.attributes || {}))) return false;
60
- if (!operator || !attrValue) return true;
103
+ if (!operator || attrValue === void 0) return true;
61
104
  const value = element.attributes[attrName];
62
105
  if (value === void 0) return false;
63
106
  switch (operator) {
64
107
  case "=": return value === attrValue;
65
- case "^=": return value.startsWith(attrValue);
66
- case "$=": return value.endsWith(attrValue);
67
- case "*=": return value.includes(attrValue);
68
- case "~=": return value.trim().split(" ").filter(Boolean).includes(attrValue);
108
+ case "^=": return attrValue.length > 0 && value.startsWith(attrValue);
109
+ case "$=": return attrValue.length > 0 && value.endsWith(attrValue);
110
+ case "*=": return attrValue.length > 0 && value.includes(attrValue);
111
+ case "~=": return containsWhitespaceSeparatedToken(value, attrValue);
69
112
  case "|=": return value === attrValue || value.startsWith(`${attrValue}-`);
70
113
  default: return false;
71
114
  }
72
115
  },
73
116
  toString: () => {
74
- if (!operator || !attrValue) return `[${attrName}]`;
117
+ if (!operator || attrValue === void 0) return `[${attrName}]`;
75
118
  return `[${attrName}${operator}${attrValue}]`;
76
119
  }
77
120
  };
@@ -94,7 +137,6 @@ function createSelectorList(selectors) {
94
137
  function parseCompoundSelector(selector) {
95
138
  const selectorParts = [];
96
139
  let current = "";
97
- let inAttribute = false;
98
140
  for (let i = 0; i < selector.length; i++) {
99
141
  const char = selector[i];
100
142
  if ((char === "." || char === "#" || char === "[") && current) {
@@ -104,9 +146,12 @@ function parseCompoundSelector(selector) {
104
146
  else if (current) selectorParts.push(createTagSelector(current));
105
147
  current = char;
106
148
  } else current += char;
107
- if (char === "[") inAttribute = true;
108
- if (char === "]") inAttribute = false;
109
- if (inAttribute && char !== "[") {}
149
+ if (char === "[") {
150
+ const end = findAttributeEnd(selector, i);
151
+ if (end === -1) return createTagSelector(selector);
152
+ current += selector.slice(i + 1, end + 1);
153
+ i = end;
154
+ }
110
155
  }
111
156
  if (current) {
112
157
  if (current[0] === ".") selectorParts.push(createClassSelector(current));
@@ -122,24 +167,13 @@ function parseSelector(selector) {
122
167
  if (!selector) throw new Error("Empty selector");
123
168
  const selectors = [];
124
169
  let start = 0;
125
- let inAttribute = false;
126
- let quote = 0;
127
- let escaped = false;
128
170
  for (let i = 0; i < selector.length; i++) {
129
171
  const char = selector.charCodeAt(i);
130
- if (quote) {
131
- if (escaped) escaped = false;
132
- else if (char === 92) escaped = true;
133
- else if (char === quote) quote = 0;
134
- continue;
135
- }
136
- if (inAttribute) {
137
- if (char === 34 || char === 39) quote = char;
138
- else if (char === 93) inAttribute = false;
139
- continue;
140
- }
141
- if (char === 91) inAttribute = true;
142
- else if (char === 44) {
172
+ if (char === 91) {
173
+ const end = findAttributeEnd(selector, i);
174
+ if (end === -1) break;
175
+ i = end;
176
+ } else if (char === 44) {
143
177
  const part = selector.slice(start, i).trim();
144
178
  if (!part) return createTagSelector(selector);
145
179
  selectors.push(parseCompoundSelector(part));
@@ -561,8 +595,9 @@ function tailwindPlugin() {
561
595
  content: "",
562
596
  skip: true
563
597
  };
564
- const prefix = state.outputFormat === "markdown" ? tailwindData?.prefix || "" : "";
565
- const suffix = state.outputFormat === "markdown" ? tailwindData?.suffix || "" : "";
598
+ const formatsText = state.outputFormat === "markdown" && !state.codeText;
599
+ const prefix = formatsText ? tailwindData?.prefix || "" : "";
600
+ const suffix = formatsText ? tailwindData?.suffix || "" : "";
566
601
  if (prefix || suffix) node.generatedMarkdown = {
567
602
  prefix,
568
603
  suffix