@mdream/js 1.7.2 → 2.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  import { DEFAULT_BLOCK_SPACING, NO_SPACING, TagIdMap, isInsideRawHtmlBlock } from "./const.mjs";
2
- import { blockOpenPrefix, breakHandler, continuationPrefix, escapeHtml, figcaptionOwnsBlockSpacing, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, lastOutputChar, listMarkerLineStart, markRenderedChildContent, orderedItemNumber, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl, safeAnchorOutput } from "./parse.mjs";
2
+ import { blockOpenPrefix, breakHandler, continuationPrefix, escapeHtml, figcaptionOwnsBlockSpacing, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isDataUrl, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, lastOutputChar, listMarkerLineStart, markRenderedChildContent, orderedItemNumber, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl, safeAnchorOutput } from "./parse.mjs";
3
3
  function resolveClean(clean) {
4
4
  if (clean === true) return {
5
5
  urls: true,
@@ -1226,7 +1226,8 @@ function getPlainTextOutput(node, eventType, state) {
1226
1226
  const alt = node.attributes?.alt;
1227
1227
  const clean = state.options?.clean;
1228
1228
  if ((clean === true || clean != null && clean !== false && clean.emptyImages === true) && !(alt !== void 0 && alt.trim().length > 0)) return void 0;
1229
- const output = alt !== void 0 ? alt || void 0 : node.attributes?.title || resolveUrl(node.attributes?.src || "", state.options?.origin, state.options?.clean) || void 0;
1229
+ const src = node.attributes?.src || "";
1230
+ const output = alt !== void 0 ? alt || void 0 : node.attributes?.title || resolveUrl(isDataUrl(src) ? "" : src, state.options?.origin, state.options?.clean) || void 0;
1230
1231
  if (output && hasNonWhitespace(output)) markRenderedChildContent(node);
1231
1232
  return output;
1232
1233
  }
@@ -274,6 +274,13 @@ function schemeMatches(href, start, end, scheme) {
274
274
  }
275
275
  return false;
276
276
  }
277
+ function isDataUrl(src) {
278
+ let start = 0;
279
+ while (start < src.length && src.charCodeAt(start) <= 32) start++;
280
+ let end = src.length;
281
+ while (end > start && src.charCodeAt(end - 1) <= 32) end--;
282
+ return schemeMatches(src, start, end, "data:");
283
+ }
277
284
  function isEmptyLinkHref(href) {
278
285
  let start = 0;
279
286
  while (start < href.length && href.charCodeAt(start) <= 32) start++;
@@ -870,6 +877,7 @@ const tagHandlers = {
870
877
  exit: ({ state }) => isInsideTableCell(state) ? "</summary>" : "</summary>\n\n"
871
878
  },
872
879
  [4]: {
880
+ excludesTextNodes: true,
873
881
  collapsesInnerWhiteSpace: true,
874
882
  isNonNesting: true,
875
883
  spacing: NO_SPACING
@@ -1074,7 +1082,8 @@ const tagHandlers = {
1074
1082
  [27]: {
1075
1083
  enter: ({ node, state }) => {
1076
1084
  const alt = node.attributes?.alt || "";
1077
- const src = resolveUrl(node.attributes?.src || "", state.options?.origin, state.options?.clean);
1085
+ const rawSrc = node.attributes?.src || "";
1086
+ const src = isDataUrl(rawSrc) ? "" : resolveUrl(rawSrc, state.options?.origin, state.options?.clean);
1078
1087
  const clean = state.options?.clean;
1079
1088
  if ((clean === true || clean !== void 0 && clean !== false && clean.emptyImages === true) && !alt.trim()) {
1080
1089
  if (state.depthMap?.[106]) return void 0;
@@ -1480,6 +1489,10 @@ function buildTagOverrideHandlers(overrides) {
1480
1489
  }
1481
1490
  return result;
1482
1491
  }
1492
+ const UNKNOWN_TAG_HANDLER = {
1493
+ isInline: true,
1494
+ spacing: NO_SPACING
1495
+ };
1483
1496
  const LT_CHAR = 60;
1484
1497
  const GT_CHAR = 62;
1485
1498
  const SLASH_CHAR = 47;
@@ -2507,7 +2520,7 @@ function processCdataSection(content, state, handleEvent) {
2507
2520
  function processOpeningTag(tagName, tagId, htmlChunk, i, state, handleEvent) {
2508
2521
  tagId = effectiveTagId(tagName, tagId, state);
2509
2522
  if (state.currentNode?.tagHandler?.isNonNesting) closeNode(state.currentNode, state, handleEvent);
2510
- const tagHandler = state.tagOverrideHandlers?.get(tagName) ?? tagHandlers[tagId];
2523
+ const tagHandler = state.tagOverrideHandlers?.get(tagName) ?? (tagId === -1 ? UNKNOWN_TAG_HANDLER : tagHandlers[tagId]);
2511
2524
  const result = scanTagAttributes(htmlChunk, i, tagHandler);
2512
2525
  if (result._tag === "incomplete") return {
2513
2526
  complete: false,
@@ -2740,4 +2753,4 @@ function parseAttributes(attrStr) {
2740
2753
  if (result._tag === "incomplete") throw new Error("Isolated attribute scans always complete at EOF");
2741
2754
  return result.attributes === EMPTY_ATTRIBUTES ? {} : result.attributes;
2742
2755
  }
2743
- export { blockOpenPrefix, breakHandler, buildTagOverrideHandlers, continuationPrefix, escapeHtml, figcaptionOwnsBlockSpacing, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, lastOutputChar, listMarkerLineStart, markRenderedChildContent, orderedItemNumber, parseAttributes, parseHtml, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl, safeAnchorOutput };
2756
+ export { blockOpenPrefix, breakHandler, buildTagOverrideHandlers, continuationPrefix, escapeHtml, figcaptionOwnsBlockSpacing, finalizeParse, getLanguageFromClass, isCharacterReferenceTail, isDataUrl, isEmptyLinkHref, isInsideHeading, isInsideTableCell, isSafeHtmlUrl, lastOutputChar, listMarkerLineStart, markRenderedChildContent, orderedItemNumber, parseAttributes, parseHtml, parseHtmlStream, parseUnsignedInteger, renderBreak, resolveUrl, safeAnchorOutput };
@@ -76,8 +76,12 @@ export interface FrontmatterConfig {
76
76
  * Works with both the JavaScript and Rust engines.
77
77
  */
78
78
  export interface BuiltinPlugins {
79
- /** Filter elements by CSS selectors, tag names, or TAG_* constants */
80
- filter?: {
79
+ /**
80
+ * Filter elements by CSS selectors, tag names, or TAG_* constants. With
81
+ * `withMinimalPreset()`, a filter adds to the preset's excludes; `false`
82
+ * turns filtering off.
83
+ */
84
+ filter?: false | {
81
85
  include?: (string | number)[];
82
86
  exclude?: (string | number)[];
83
87
  processChildren?: boolean;
package/dist/index.mjs CHANGED
@@ -5,6 +5,18 @@ import { createPlugin } from "./_chunks/plugins.mjs";
5
5
  import { resolvePlugins } from "./_chunks/resolve-plugins.mjs";
6
6
  import { streamHtmlToMarkdown as streamHtmlToMarkdown$1 } from "./_chunks/stream.mjs";
7
7
  import { withMinimalPreset } from "./preset/minimal.mjs";
8
+ const TOP_LEVEL_PLUGIN_KEYS = [
9
+ "frontmatter",
10
+ "isolateMain",
11
+ "tailwind",
12
+ "filter",
13
+ "extraction",
14
+ "tagOverrides"
15
+ ];
16
+ function assertEngineOptions(options) {
17
+ if ("minimal" in options) throw new TypeError("@mdream/js has no `minimal` option. Use withMinimalPreset() from '@mdream/js/preset/minimal': htmlToMarkdown(html, withMinimalPreset({ origin })).");
18
+ for (const key of TOP_LEVEL_PLUGIN_KEYS) if (key in options) throw new TypeError(`@mdream/js reads \`${key}\` from \`plugins\`. Pass { plugins: { ${key} } } instead of { ${key} }.`);
19
+ }
8
20
  function resolveHooks(options) {
9
21
  return options.hooks?.length ? options.hooks : void 0;
10
22
  }
@@ -21,14 +33,26 @@ function convert(html, options, hooks) {
21
33
  return processor.getMarkdown();
22
34
  }
23
35
  function htmlToMarkdown(html, options = {}) {
36
+ assertEngineOptions(options);
24
37
  const markdown = convert(html, options, resolveHooks(options));
25
38
  if (options.clean && (options.format === void 0 || options.format === "markdown")) return applyClean(markdown, resolveClean(options.clean));
26
39
  return markdown;
27
40
  }
28
41
  function streamHtmlToMarkdown(htmlStream, options = {}) {
42
+ assertEngineOptions(options);
29
43
  const hooks = resolveHooks(options);
30
- const { plugins } = resolvePlugins(options, hooks);
44
+ const { plugins, callExtractionHandlers, getFrontmatter, frontmatterCallback } = resolvePlugins(options, hooks);
31
45
  const tagOverrideHandlers = options.plugins?.tagOverrides ? buildTagOverrideHandlers(options.plugins.tagOverrides) : void 0;
32
- return streamHtmlToMarkdown$1(htmlStream, options, plugins, tagOverrideHandlers);
46
+ const stream = streamHtmlToMarkdown$1(htmlStream, options, plugins, tagOverrideHandlers);
47
+ if (!callExtractionHandlers && !frontmatterCallback) return stream;
48
+ return withCallbacks(stream, () => {
49
+ const fm = getFrontmatter?.();
50
+ if (fm && frontmatterCallback) frontmatterCallback(fm);
51
+ callExtractionHandlers?.();
52
+ });
53
+ }
54
+ async function* withCallbacks(stream, onEnd) {
55
+ yield* stream;
56
+ onEnd();
33
57
  }
34
58
  export { ELEMENT_NODE, NodeEventEnter, NodeEventExit, TAG_H1, TAG_H2, TAG_H3, TAG_H4, TAG_H5, TAG_H6, TEXT_NODE, createPlugin, htmlToMarkdown, streamHtmlToMarkdown, withMinimalPreset };
@@ -1,5 +1,20 @@
1
1
  import "../_chunks/const.mjs";
2
+ const MINIMAL_EXCLUDE = [
3
+ 40,
4
+ 68,
5
+ 103,
6
+ 58,
7
+ 47,
8
+ 88,
9
+ 73,
10
+ 59,
11
+ 66,
12
+ 65,
13
+ 43,
14
+ 41
15
+ ];
2
16
  function withMinimalPreset(options = {}) {
17
+ const filter = options.plugins?.filter;
3
18
  return {
4
19
  clean: options.clean !== void 0 ? options.clean : true,
5
20
  ...options,
@@ -7,21 +22,11 @@ function withMinimalPreset(options = {}) {
7
22
  frontmatter: true,
8
23
  isolateMain: true,
9
24
  tailwind: true,
10
- filter: { exclude: [
11
- 40,
12
- 68,
13
- 103,
14
- 58,
15
- 47,
16
- 88,
17
- 73,
18
- 59,
19
- 66,
20
- 65,
21
- 43,
22
- 41
23
- ] },
24
- ...options.plugins
25
+ ...options.plugins,
26
+ filter: filter === false ? false : {
27
+ ...filter,
28
+ exclude: [...MINIMAL_EXCLUDE, ...filter?.exclude ?? []]
29
+ }
25
30
  }
26
31
  };
27
32
  }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/js",
3
3
  "type": "module",
4
- "version": "1.7.2",
4
+ "version": "2.0.0-beta.1",
5
5
  "description": "JavaScript HTML-to-Markdown engine for mdream. Escape hatch for hooks and edge runtimes.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",