mdream 0.1.1 → 0.2.1

package/README.md

@@ -99,16 +99,16 @@ pnpm add mdream
 ### Usage
 
 Mdream provides two utils for working with HTML, both will process content as a stream.
-- `syncHtmlToMarkdown`: Useful if you already have the entire HTML payload you want to convert.
+- `htmlToMarkdown`: Useful if you already have the entire HTML payload you want to convert.
 - `streamHtmlToMarkdown`: Best practice if you are fetching or reading from a local file.
 
 **Convert existing HTML**
 
 ```ts
-import { syncHtmlToMarkdown } from 'mdream'
+import { htmlToMarkdown } from 'mdream'
 
 // Simple conversion
-const markdown = syncHtmlToMarkdown('<h1>Hello World</h1>')
+const markdown = htmlToMarkdown('<h1>Hello World</h1>')
 console.log(markdown) // # Hello World
 ````
 
@@ -138,7 +138,7 @@ for await (const chunk of markdownGenerator) {
 Mdream now features a powerful plugin system that allows you to customize and extend the HTML-to-Markdown conversion process.
 
 ```ts
-import { createPlugin, filterUnsupportedTags, syncHtmlToMarkdown, withTailwind } from 'mdream'
+import { createPlugin, filterUnsupportedTags, htmlToMarkdown, withTailwind } from 'mdream'
 
 // Create a custom plugin
 const myPlugin = createPlugin({
@@ -153,7 +153,7 @@ const myPlugin = createPlugin({
 
 // Use multiple plugins together
 const html = '<div role="alert" class="font-bold">Important message</div>'
-const markdown = syncHtmlToMarkdown(html, {
+const markdown = htmlToMarkdown(html, {
   plugins: [
     withTailwind(), // Apply Tailwind class processing
     filterUnsupportedTags(), // Filter out unsupported tags
@@ -164,6 +164,37 @@ const markdown = syncHtmlToMarkdown(html, {
 console.log(markdown) // "⚠️ **Important message** ⚠️"
 ```
 
+#### Extraction Plugin
+
+Extract specific elements and their content during HTML processing for data analysis or content discovery:
+
+```ts
+import { extractionPlugin, htmlToMarkdown } from 'mdream'
+
+const html = `
+  <article>
+    <h2>Getting Started</h2>
+    <p>This is a tutorial about web scraping.</p>
+    <img src="/hero.jpg" alt="Hero image" />
+  </article>
+`
+
+// Extract elements using CSS selectors
+const plugin = extractionPlugin({
+  'h2': (element) => {
+    console.log('Heading:', element.textContent) // "Getting Started"
+  },
+  'img[alt]': (element) => {
+    console.log('Image:', element.attributes.src, element.attributes.alt)
+    // "Image: /hero.jpg Hero image"
+  }
+})
+
+htmlToMarkdown(html, { plugins: [plugin] })
+```
+
+The extraction plugin provides memory-efficient element extraction with full text content and attributes, perfect for SEO analysis, content discovery, and data mining.
+
 For more details, see the [plugin documentation](./docs/plugins.md).
 
 ## Credits

package/dist/index.d.mts

@@ -10,6 +10,6 @@ import { ReadableStream } from 'node:stream/web';
  */
 declare function streamHtmlToMarkdown(htmlStream: ReadableStream | null, options?: HTMLToMarkdownOptions): AsyncIterable<string>;
 
-declare function syncHtmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
+declare function htmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
 
-export { HTMLToMarkdownOptions, streamHtmlToMarkdown, syncHtmlToMarkdown };
+export { HTMLToMarkdownOptions, htmlToMarkdown, streamHtmlToMarkdown };

package/dist/index.d.ts

@@ -10,6 +10,6 @@ import { ReadableStream } from 'node:stream/web';
  */
 declare function streamHtmlToMarkdown(htmlStream: ReadableStream | null, options?: HTMLToMarkdownOptions): AsyncIterable<string>;
 
-declare function syncHtmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
+declare function htmlToMarkdown(html: string, options?: HTMLToMarkdownOptions): string;
 
-export { HTMLToMarkdownOptions, streamHtmlToMarkdown, syncHtmlToMarkdown };
+export { HTMLToMarkdownOptions, htmlToMarkdown, streamHtmlToMarkdown };

package/dist/index.mjs

@@ -2,7 +2,7 @@ import { p as processPartialHTMLToMarkdown } from './shared/mdream.DUeWbUFG.mjs'
 export { s as streamHtmlToMarkdown } from './shared/mdream.DUeWbUFG.mjs';
 import './shared/mdream.-hdaPj9a.mjs';
 
-function syncHtmlToMarkdown(html, options = {}) {
+function htmlToMarkdown(html, options = {}) {
   const state = {
     options
   };
@@ -10,4 +10,4 @@ function syncHtmlToMarkdown(html, options = {}) {
   return result.trimEnd();
 }
 
-export { syncHtmlToMarkdown };
+export { htmlToMarkdown };

package/dist/plugins.mjs

@@ -1,4 +1,4 @@
 export { c as createPlugin, f as frontmatterPlugin } from './shared/mdream.cpEmpxyh.mjs';
-export { f as filterPlugin, i as isolateMainPlugin, t as tailwindPlugin } from './shared/mdream.5zaIXVJz.mjs';
+export { f as filterPlugin, i as isolateMainPlugin, t as tailwindPlugin } from './shared/mdream.DEM9pag4.mjs';
 export { r as readabilityPlugin } from './shared/mdream.C8ruysN5.mjs';
 import './shared/mdream.-hdaPj9a.mjs';

package/dist/preset/minimal.mjs

@@ -1,5 +1,5 @@
 import { y as TAG_FORM, t as TAG_FIELDSET, q as TAG_OBJECT, r as TAG_EMBED, a0 as TAG_FIGURE, B as TAG_FOOTER, z as TAG_ASIDE, s as TAG_IFRAME, w as TAG_INPUT, v as TAG_TEXTAREA, u as TAG_SELECT, x as TAG_BUTTON, A as TAG_NAV } from '../shared/mdream.-hdaPj9a.mjs';
-import { i as isolateMainPlugin, t as tailwindPlugin, f as filterPlugin } from '../shared/mdream.5zaIXVJz.mjs';
+import { i as isolateMainPlugin, t as tailwindPlugin, f as filterPlugin } from '../shared/mdream.DEM9pag4.mjs';
 import { f as frontmatterPlugin } from '../shared/mdream.cpEmpxyh.mjs';
 
 function withMinimalPreset(options = {}) {

package/dist/shared/{mdream.5zaIXVJz.mjs → mdream.DEM9pag4.mjs}

@@ -103,7 +103,6 @@ function parseSelector(selector) {
   }
   const selectorParts = [];
   let current = "";
-  let inAttribute = false;
   for (let i = 0; i < selector.length; i++) {
     const char = selector[i];
     if ((char === "." || char === "#" || char === "[") && current) {
@@ -120,13 +119,6 @@ function parseSelector(selector) {
     } else {
       current += char;
     }
-    if (char === "[")
-      inAttribute = true;
-    if (char === "]")
-      inAttribute = false;
-    if (inAttribute && char !== "[") {
-      continue;
-    }
   }
   if (current) {
     if (current[0] === ".") {
@@ -144,6 +136,7 @@ function parseSelector(selector) {
   }
   return new CompoundSelector(selectorParts);
 }
+
 function filterPlugin(options = {}) {
   const includeSelectors = options.include?.map((selector) => {
     if (typeof selector === "string") {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mdream",
   "type": "module",
-  "version": "0.1.1",
+  "version": "0.2.1",
   "description": "Ultra-performant JavaScript HTML to Markdown converter optimized for LLMs.",
   "author": {
     "name": "Harlan Wilton",