mdream 0.15.3 → 0.17.0

package/README.md

@@ -201,7 +201,6 @@ Mdream includes several built-in plugins that can be used individually or combin
 - **[`frontmatterPlugin`](./src/plugins/frontmatter.ts)**: Generate YAML frontmatter from HTML head elements (title, meta tags)
 - **[`isolateMainPlugin`](./src/plugins/isolate-main.ts)**: Isolate main content using `<main>` elements or header-to-footer boundaries
 - **[`tailwindPlugin`](./src/plugins/tailwind.ts)**: Convert Tailwind CSS classes to Markdown formatting (bold, italic, etc.)
-- **[`readabilityPlugin`](./src/plugins/readability.ts)**: Content scoring and extraction (experimental)
 
 ```ts
 import { filterPlugin, frontmatterPlugin, isolateMainPlugin } from 'mdream/plugins'
@@ -215,6 +214,26 @@ const markdown = htmlToMarkdown(html, {
 })
 ```
 
+### Content Extraction with Readability
+
+For advanced content extraction (article detection, boilerplate removal), use [@mozilla/readability](https://github.com/mozilla/readability) before mdream:
+
+```ts
+import { Readability } from '@mozilla/readability'
+import { JSDOM } from 'jsdom'
+import { htmlToMarkdown } from 'mdream'
+
+const dom = new JSDOM(html, { url: 'https://example.com' })
+const article = new Readability(dom.window.document).parse()
+
+if (article) {
+  const markdown = htmlToMarkdown(article.content)
+  // article.title, article.excerpt, article.byline also available
+}
+```
+
+This pipeline gives you battle-tested content extraction + fast markdown conversion.
+
 ### Plugin Hooks
 
 - `beforeNodeProcess`: Called before any node processing, can skip nodes
@@ -526,12 +545,3 @@ Custom notes
 
 Licensed under the [MIT license](https://github.com/harlan-zw/mdream/blob/main/LICENSE.md).
 
-<!-- Badges -->
-[npm-version-src]: https://img.shields.io/npm/v/mdream/latest.svg?style=flat&colorA=18181B&colorB=4C9BE0
-[npm-version-href]: https://npmjs.com/package/mdream
-
-[npm-downloads-src]: https://img.shields.io/npm/dm/mdream.svg?style=flat&colorA=18181B&colorB=4C9BE0
-[npm-downloads-href]: https://npmjs.com/package/mdream
-
-[license-src]: https://img.shields.io/github/license/harlan-zw/mdream.svg?style=flat&colorA=18181B&colorB=4C9BE0
-[license-href]: https://github.com/harlan-zw/mdream/blob/main/LICENSE.md

package/dist/_chunks/const.mjs

@@ -1,151 +1,3 @@
-//#region src/buffer-region.ts
-/**
-* Creates a new buffer region
-* Returns null if node already has a region assigned
-*/
-function createBufferRegion(node, state, include) {
-	if (node.regionId) return null;
-	const id = state.regionToggles.size + 1;
-	node.regionId = id;
-	state.regionToggles.set(id, include);
-	state.regionContentBuffers.set(id, []);
-	return id;
-}
-/**
-* Collects content for a node into appropriate buffer (optimized)
-*/
-function collectNodeContent(node, content, state) {
-	if (!content) return;
-	const regionId = node.regionId || 0;
-	const targetBuffer = state.regionContentBuffers.get(regionId);
-	if (targetBuffer) {
-		targetBuffer.push(content);
-		state.lastContentCache = content;
-	}
-}
-/**
-* Assembles final content from buffer regions and clears them after use
-* Ensures frontmatter (regionId -1) appears first, followed by other included regions
-*/
-function assembleBufferedContent(state) {
-	const fragments = [];
-	for (const [regionId, content] of Array.from(state.regionContentBuffers.entries())) if (state.regionToggles.get(regionId)) fragments.push(...content);
-	state.regionToggles.clear();
-	state.regionContentBuffers.clear();
-	return fragments.join("").trimStart();
-}
-
-//#endregion
-//#region src/const.ts
-const TAG_HTML = 0;
-const TAG_HEAD = 1;
-const TAG_DETAILS = 2;
-const TAG_SUMMARY = 3;
-const TAG_TITLE = 4;
-const TAG_META = 5;
-const TAG_BR = 6;
-const TAG_H1 = 7;
-const TAG_H2 = 8;
-const TAG_H3 = 9;
-const TAG_H4 = 10;
-const TAG_H5 = 11;
-const TAG_H6 = 12;
-const TAG_HR = 13;
-const TAG_STRONG = 14;
-const TAG_B = 15;
-const TAG_EM = 16;
-const TAG_I = 17;
-const TAG_DEL = 18;
-const TAG_SUB = 19;
-const TAG_SUP = 20;
-const TAG_INS = 21;
-const TAG_BLOCKQUOTE = 22;
-const TAG_CODE = 23;
-const TAG_UL = 24;
-const TAG_LI = 25;
-const TAG_A = 26;
-const TAG_IMG = 27;
-const TAG_TABLE = 28;
-const TAG_THEAD = 29;
-const TAG_TR = 30;
-const TAG_TH = 31;
-const TAG_TD = 32;
-const TAG_OL = 33;
-const TAG_PRE = 34;
-const TAG_P = 35;
-const TAG_DIV = 36;
-const TAG_SPAN = 37;
-const TAG_TBODY = 38;
-const TAG_TFOOT = 39;
-const TAG_FORM = 40;
-const TAG_NAV = 41;
-const TAG_LABEL = 42;
-const TAG_BUTTON = 43;
-const TAG_BODY = 44;
-const TAG_CENTER = 45;
-const TAG_KBD = 46;
-const TAG_FOOTER = 47;
-const TAG_PATH = 48;
-const TAG_SVG = 49;
-const TAG_ARTICLE = 50;
-const TAG_SECTION = 51;
-const TAG_SCRIPT = 52;
-const TAG_STYLE = 53;
-const TAG_LINK = 54;
-const TAG_AREA = 55;
-const TAG_BASE = 56;
-const TAG_COL = 57;
-const TAG_EMBED = 58;
-const TAG_INPUT = 59;
-const TAG_KEYGEN = 60;
-const TAG_PARAM = 61;
-const TAG_SOURCE = 62;
-const TAG_TRACK = 63;
-const TAG_WBR = 64;
-const TAG_SELECT = 65;
-const TAG_TEXTAREA = 66;
-const TAG_OPTION = 67;
-const TAG_FIELDSET = 68;
-const TAG_LEGEND = 69;
-const TAG_AUDIO = 70;
-const TAG_VIDEO = 71;
-const TAG_CANVAS = 72;
-const TAG_IFRAME = 73;
-const TAG_MAP = 74;
-const TAG_DIALOG = 75;
-const TAG_METER = 76;
-const TAG_PROGRESS = 77;
-const TAG_TEMPLATE = 78;
-const TAG_ABBR = 79;
-const TAG_MARK = 80;
-const TAG_Q = 81;
-const TAG_SAMP = 82;
-const TAG_SMALL = 83;
-const TAG_NOSCRIPT = 84;
-const TAG_NOFRAMES = 85;
-const TAG_XMP = 86;
-const TAG_PLAINTEXT = 87;
-const TAG_ASIDE = 88;
-const TAG_U = 89;
-const TAG_CITE = 90;
-const TAG_DFN = 91;
-const TAG_VAR = 92;
-const TAG_TIME = 93;
-const TAG_BDO = 94;
-const TAG_RUBY = 95;
-const TAG_RT = 96;
-const TAG_RP = 97;
-const TAG_DD = 98;
-const TAG_DT = 99;
-const TAG_ADDRESS = 100;
-const TAG_DL = 101;
-const TAG_FIGURE = 102;
-const TAG_OBJECT = 103;
-const TAG_MAIN = 104;
-const TAG_HEADER = 105;
-const TAG_FIGCAPTION = 106;
-const TAG_CAPTION = 107;
-const MAX_TAG_ID = 108;
 const HTML_ENTITIES = {
 	"&": "&",
 	"&lt;": "<",
@@ -155,131 +7,120 @@ const HTML_ENTITIES = {
 	"&apos;": "'",
 	"&nbsp;": " "
 };
-const ELEMENT_NODE = 1;
-const TEXT_NODE = 2;
-const NodeEventEnter = 0;
-const NodeEventExit = 1;
 const TagIdMap = {
-	html: TAG_HTML,
-	head: TAG_HEAD,
-	details: TAG_DETAILS,
-	summary: TAG_SUMMARY,
-	title: TAG_TITLE,
-	meta: TAG_META,
-	br: TAG_BR,
-	h1: TAG_H1,
-	h2: TAG_H2,
-	h3: TAG_H3,
-	h4: TAG_H4,
-	h5: TAG_H5,
-	h6: TAG_H6,
-	hr: TAG_HR,
-	strong: TAG_STRONG,
-	b: TAG_B,
-	em: TAG_EM,
-	i: TAG_I,
-	del: TAG_DEL,
-	sub: TAG_SUB,
-	sup: TAG_SUP,
-	ins: TAG_INS,
-	blockquote: TAG_BLOCKQUOTE,
-	code: TAG_CODE,
-	ul: TAG_UL,
-	li: TAG_LI,
-	a: TAG_A,
-	img: TAG_IMG,
-	table: TAG_TABLE,
-	thead: TAG_THEAD,
-	tr: TAG_TR,
-	th: TAG_TH,
-	td: TAG_TD,
-	ol: TAG_OL,
-	pre: TAG_PRE,
-	p: TAG_P,
-	div: TAG_DIV,
-	span: TAG_SPAN,
-	tbody: TAG_TBODY,
-	tfoot: TAG_TFOOT,
-	form: TAG_FORM,
-	nav: TAG_NAV,
-	label: TAG_LABEL,
-	button: TAG_BUTTON,
-	body: TAG_BODY,
-	center: TAG_CENTER,
-	kbd: TAG_KBD,
-	footer: TAG_FOOTER,
-	path: TAG_PATH,
-	svg: TAG_SVG,
-	article: TAG_ARTICLE,
-	section: TAG_SECTION,
-	script: TAG_SCRIPT,
-	style: TAG_STYLE,
-	link: TAG_LINK,
-	area: TAG_AREA,
-	base: TAG_BASE,
-	col: TAG_COL,
-	embed: TAG_EMBED,
-	input: TAG_INPUT,
-	keygen: TAG_KEYGEN,
-	param: TAG_PARAM,
-	source: TAG_SOURCE,
-	track: TAG_TRACK,
-	wbr: TAG_WBR,
-	select: TAG_SELECT,
-	textarea: TAG_TEXTAREA,
-	option: TAG_OPTION,
-	fieldset: TAG_FIELDSET,
-	legend: TAG_LEGEND,
-	audio: TAG_AUDIO,
-	video: TAG_VIDEO,
-	canvas: TAG_CANVAS,
-	iframe: TAG_IFRAME,
-	map: TAG_MAP,
-	dialog: TAG_DIALOG,
-	meter: TAG_METER,
-	progress: TAG_PROGRESS,
-	template: TAG_TEMPLATE,
-	abbr: TAG_ABBR,
-	mark: TAG_MARK,
-	q: TAG_Q,
-	samp: TAG_SAMP,
-	small: TAG_SMALL,
-	noscript: TAG_NOSCRIPT,
-	noframes: TAG_NOFRAMES,
-	xmp: TAG_XMP,
-	plaintext: TAG_PLAINTEXT,
-	aside: TAG_ASIDE,
-	u: TAG_U,
-	cite: TAG_CITE,
-	dfn: TAG_DFN,
-	var: TAG_VAR,
-	time: TAG_TIME,
-	bdo: TAG_BDO,
-	ruby: TAG_RUBY,
-	rt: TAG_RT,
-	rp: TAG_RP,
-	dd: TAG_DD,
-	dt: TAG_DT,
-	dl: TAG_DL,
-	address: TAG_ADDRESS,
-	figure: TAG_FIGURE,
-	object: TAG_OBJECT,
-	main: TAG_MAIN,
-	header: TAG_HEADER,
-	figcaption: TAG_FIGCAPTION,
-	caption: TAG_CAPTION
+	html: 0,
+	head: 1,
+	details: 2,
+	summary: 3,
+	title: 4,
+	meta: 5,
+	br: 6,
+	h1: 7,
+	h2: 8,
+	h3: 9,
+	h4: 10,
+	h5: 11,
+	h6: 12,
+	hr: 13,
+	strong: 14,
+	b: 15,
+	em: 16,
+	i: 17,
+	del: 18,
+	sub: 19,
+	sup: 20,
+	ins: 21,
+	blockquote: 22,
+	code: 23,
+	ul: 24,
+	li: 25,
+	a: 26,
+	img: 27,
+	table: 28,
+	thead: 29,
+	tr: 30,
+	th: 31,
+	td: 32,
+	ol: 33,
+	pre: 34,
+	p: 35,
+	div: 36,
+	span: 37,
+	tbody: 38,
+	tfoot: 39,
+	form: 40,
+	nav: 41,
+	label: 42,
+	button: 43,
+	body: 44,
+	center: 45,
+	kbd: 46,
+	footer: 47,
+	path: 48,
+	svg: 49,
+	article: 50,
+	section: 51,
+	script: 52,
+	style: 53,
+	link: 54,
+	area: 55,
+	base: 56,
+	col: 57,
+	embed: 58,
+	input: 59,
+	keygen: 60,
+	param: 61,
+	source: 62,
+	track: 63,
+	wbr: 64,
+	select: 65,
+	textarea: 66,
+	option: 67,
+	fieldset: 68,
+	legend: 69,
+	audio: 70,
+	video: 71,
+	canvas: 72,
+	iframe: 73,
+	map: 74,
+	dialog: 75,
+	meter: 76,
+	progress: 77,
+	template: 78,
+	abbr: 79,
+	mark: 80,
+	q: 81,
+	samp: 82,
+	small: 83,
+	noscript: 84,
+	noframes: 85,
+	xmp: 86,
+	plaintext: 87,
+	aside: 88,
+	u: 89,
+	cite: 90,
+	dfn: 91,
+	var: 92,
+	time: 93,
+	bdo: 94,
+	ruby: 95,
+	rt: 96,
+	rp: 97,
+	dd: 98,
+	dt: 99,
+	dl: 101,
+	address: 100,
+	figure: 102,
+	object: 103,
+	main: 104,
+	header: 105,
+	figcaption: 106,
+	caption: 107
 };
-const MARKDOWN_STRONG = "**";
-const MARKDOWN_EMPHASIS = "_";
-const MARKDOWN_STRIKETHROUGH = "~~";
-const MARKDOWN_CODE_BLOCK = "```";
-const MARKDOWN_INLINE_CODE = "`";
-const MARKDOWN_HORIZONTAL_RULE = "---";
 const NO_SPACING = [0, 0];
 const DEFAULT_BLOCK_SPACING = [2, 2];
 const BLOCKQUOTE_SPACING = [1, 1];
 const LIST_ITEM_SPACING = [1, 0];
 const TABLE_ROW_SPACING = [0, 1];
-
 //#endregion
-export { TAG_H2 as $, TAG_TBODY as $t, TAG_BUTTON as A, TAG_P as At, TAG_DFN as B, TAG_SCRIPT as Bt, TAG_AUDIO as C, TAG_METER as Ct, TAG_BLOCKQUOTE as D, TAG_OBJECT as Dt, TAG_BDO as E, TAG_NOSCRIPT as Et, TAG_CODE as F, TAG_Q as Ft, TAG_EM as G, TAG_SPAN as Gt, TAG_DIV as H, TAG_SELECT as Ht, TAG_COL as I, TAG_RP as It, TAG_FIGCAPTION as J, TAG_SUB as Jt, TAG_EMBED as K, TAG_STRONG as Kt, TAG_DD as L, TAG_RT as Lt, TAG_CAPTION as M, TAG_PLAINTEXT as Mt, TAG_CENTER as N, TAG_PRE as Nt, TAG_BODY as O, TAG_OL as Ot, TAG_CITE as P, TAG_PROGRESS as Pt, TAG_H1 as Q, TAG_TABLE as Qt, TAG_DEL as R, TAG_RUBY as Rt, TAG_ASIDE as S, TAG_META as St, TAG_BASE as T, TAG_NOFRAMES as Tt, TAG_DL as U, TAG_SMALL as Ut, TAG_DIALOG as V, TAG_SECTION as Vt, TAG_DT as W, TAG_SOURCE as Wt, TAG_FOOTER as X, TAG_SUP as Xt, TAG_FIGURE as Y, TAG_SUMMARY as Yt, TAG_FORM as Z, TAG_SVG as Zt, TAG_A as _, TagIdMap as _n, TAG_LI as _t, LIST_ITEM_SPACING as a, TAG_THEAD as an, TAG_HEADER as at, TAG_AREA as b, createBufferRegion as bn, TAG_MAP as bt, MARKDOWN_HORIZONTAL_RULE as c, TAG_TR as cn, TAG_I as ct, MARKDOWN_STRONG as d, TAG_UL as dn, TAG_INPUT as dt, TAG_TD as en, TAG_H3 as et, MAX_TAG_ID as f, TAG_VAR as fn, TAG_INS as ft, TABLE_ROW_SPACING as g, TEXT_NODE as gn, TAG_LEGEND as gt, NodeEventExit as h, TAG_XMP as hn, TAG_LABEL as ht, HTML_ENTITIES as i, TAG_TH as in, TAG_HEAD as it, TAG_CANVAS as j, TAG_PARAM as jt, TAG_BR as k, TAG_OPTION as kt, MARKDOWN_INLINE_CODE as l, TAG_TRACK as ln, TAG_IFRAME as lt, NodeEventEnter as m, TAG_WBR as mn, TAG_KEYGEN as mt, DEFAULT_BLOCK_SPACING as n, TAG_TEXTAREA as nn, TAG_H5 as nt, MARKDOWN_CODE_BLOCK as o, TAG_TIME as on, TAG_HR as ot, NO_SPACING as p, TAG_VIDEO as pn, TAG_KBD as pt, TAG_FIELDSET as q, TAG_STYLE as qt, ELEMENT_NODE as r, TAG_TFOOT as rn, TAG_H6 as rt, MARKDOWN_EMPHASIS as s, TAG_TITLE as sn, TAG_HTML as st, BLOCKQUOTE_SPACING as t, TAG_TEMPLATE as tn, TAG_H4 as tt, MARKDOWN_STRIKETHROUGH as u, TAG_U as un, TAG_IMG as ut, TAG_ABBR as v, assembleBufferedContent as vn, TAG_LINK as vt, TAG_B as w, TAG_NAV as wt, TAG_ARTICLE as x, TAG_MARK as xt, TAG_ADDRESS as y, collectNodeContent as yn, TAG_MAIN as yt, TAG_DETAILS as z, TAG_SAMP as zt };
+export { NO_SPACING as a, LIST_ITEM_SPACING as i, DEFAULT_BLOCK_SPACING as n, TABLE_ROW_SPACING as o, HTML_ENTITIES as r, TagIdMap as s, BLOCKQUOTE_SPACING as t };

package/dist/_chunks/extraction.mjs

@@ -1,5 +1,4 @@
 import { t as createPlugin } from "./plugin.mjs";
-
 //#region src/libs/query-selector.ts
 /**
 * Creates a tag selector matcher (e.g., 'div', 'p', 'h1')
@@ -33,11 +32,12 @@ function createClassSelector(selector) {
 		toString: () => `.${className}`
 	};
 }
+const ATTR_SELECTOR_RE = /\[([^\]=~|^$*]+)(?:([=~|^$*]+)["']?([^"'\]]+)["']?)?\]/;
 /**
 * Creates an attribute selector matcher (e.g., '[data-id]', '[href="https://example.com"]')
 */
 function createAttributeSelector(selector) {
-	const match = selector.match(/\[([^\]=~|^$*]+)(?:([=~|^$*]+)["']?([^"'\]]+)["']?)?\]/);
+	const match = selector.match(ATTR_SELECTOR_RE);
 	const attrName = match ? match[1] : selector.slice(1, -1);
 	const operator = match?.[2];
 	const attrValue = match?.[3];
@@ -100,7 +100,6 @@ function parseSelector(selector) {
 	if (selectorParts.length === 1) return selectorParts[0];
 	return createCompoundSelector(selectorParts);
 }
-
 //#endregion
 //#region src/plugins/extraction.ts
 function extractionPlugin(selectors) {
@@ -139,6 +138,5 @@ function extractionPlugin(selectors) {
 		}
 	});
 }
-
 //#endregion
-export { parseSelector as n, extractionPlugin as t };
+export { parseSelector as n, extractionPlugin as t };