html2md4llm 1.0.0

package/README.md

@@ -0,0 +1,74 @@
+# html2md4llm
+
+Convert HTML to clean Markdown or JSON format, optimized for LLM processing.
+
+## Features
+
+- Convert HTML to Markdown or JSON
+- Intelligent content extraction (list/article modes)
+- Automatic HTML cleaning (removes scripts, styles, iframes)
+- Preserves metadata (title, description, keywords)
+- Zero dependencies - uses only Node.js built-in modules
+
+## Installation
+
+### NPM Package
+
+```bash
+npm install html2md4llm
+```
+
+### Standalone Script
+
+Download `dist/html2md4llm.min.js` and include it directly:
+
+```html
+<script src="html2md4llm.min.js"></script>
+<script>
+  const result = html2md4llm('<h1>Hello</h1>');
+</script>
+```
+
+### Dify Plugin
+
+Install from the plugin marketplace or import from this repository's `plugin/` directory.
+
+## Usage
+
+```javascript
+import { main } from 'html2md4llm';
+
+// Basic conversion to Markdown
+const markdown = main('<h1>Hello</h1><p>World</p>');
+
+// Convert to JSON
+const json = main('<h1>Hello</h1>', { outputFormat: 'json' });
+
+// Extract largest list
+const list = main(html, { strategy: 'list' });
+
+// Extract article content
+const article = main(html, { strategy: 'article' });
+```
+
+## API
+
+### `main(htmlInput, options)`
+
+**Parameters:**
+- `htmlInput` (string): HTML text to convert
+- `options` (object, optional):
+  - `outputFormat` (string): `'markdown'` (default) or `'json'`
+  - `strategy` (string): `'list'`, `'article'`, or undefined
+  - `removeAttributes` (boolean): Remove HTML attributes during parsing
+
+**Returns:** String (Markdown or JSON)
+
+## Strategies
+
+- **list**: Extracts the largest `<ul>` or `<ol>` element
+- **article**: Filters out empty containers, keeping main content
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "html2md4llm",
+  "version": "1.0.0",
+  "description": "Convert HTML to clean Markdown or JSON format, optimized for LLM processing",
+  "type": "module",
+  "main": "src/main.js",
+  "exports": {
+    ".": "./src/main.js"
+  },
+  "scripts": {
+    "test": "node tests/run-tests.js",
+    "build": "node build.js"
+  },
+  "keywords": [
+    "html",
+    "markdown",
+    "json",
+    "converter",
+    "llm",
+    "parser",
+    "html-to-markdown"
+  ],
+  "author": "kaiye",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/kaiye/html2md4llm.git"
+  },
+  "files": [
+    "src/**/*",
+    "dist/**/*",
+    "plugin/**/*",
+    "README.md",
+    "LICENSE"
+  ],
+  "devDependencies": {
+    "esbuild": "^0.27.0"
+  }
+}

package/plugin/main.js

@@ -0,0 +1,41 @@
+import { main } from '../src/main.js';
+
+export default class Html2Md4LlmTool {
+  async invoke(parameters) {
+    const { html, outputFormat = 'markdown', strategy } = parameters;
+
+    if (!html) {
+      throw new Error('html parameter is required');
+    }
+
+    const options = { outputFormat };
+    if (strategy) {
+      options.strategy = strategy;
+    }
+
+    const result = main(html, options);
+
+    return {
+      result,
+      format: outputFormat
+    };
+  }
+
+  async validate(parameters) {
+    if (!parameters.html || typeof parameters.html !== 'string') {
+      return { valid: false, error: 'html must be a non-empty string' };
+    }
+
+    const { outputFormat, strategy } = parameters;
+
+    if (outputFormat && !['markdown', 'json'].includes(outputFormat)) {
+      return { valid: false, error: 'outputFormat must be "markdown" or "json"' };
+    }
+
+    if (strategy && !['list', 'article'].includes(strategy)) {
+      return { valid: false, error: 'strategy must be "list" or "article"' };
+    }
+
+    return { valid: true };
+  }
+}

package/plugin/manifest.json

@@ -0,0 +1,16 @@
+{
+  "version": "0.0.1",
+  "type": "tool",
+  "author": "kaiye",
+  "name": "html2md4llm",
+  "label": {
+    "en_US": "HTML to Markdown/JSON",
+    "zh_Hans": "HTML 转 Markdown/JSON"
+  },
+  "description": {
+    "en_US": "Convert HTML to clean Markdown or JSON format, optimized for LLM processing",
+    "zh_Hans": "将 HTML 转换为干净的 Markdown 或 JSON 格式，针对 LLM 处理优化"
+  },
+  "icon": "icon.svg",
+  "tags": ["html", "markdown", "json", "converter", "parser"]
+}

package/src/generators/markdown.js

@@ -0,0 +1,132 @@
+const inlineElements = ['span', 'a', 'strong', 'em', 'code', 'b', 'i'];
+const blockElements = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'ul', 'ol', 'pre', 'br', 'div', 'section'];
+
+function isInline(node) {
+  if (node.type === 'element' && node.tag === 'br') return false;
+  if (node.type === 'element' && inlineElements.includes(node.tag)) return true;
+  if (node.type === 'text' && node.flattenedTags) {
+    return node.flattenedTags.every(tag => inlineElements.includes(tag));
+  }
+  return false;
+}
+
+function isBlock(node) {
+  if (node.type === 'element' && node.tag === 'br') return false;
+  if (node.flattenedTags && node.flattenedTags.length > 0) {
+    return node.flattenedTags.some(tag => blockElements.includes(tag));
+  }
+  if (node.type === 'element') {
+    return blockElements.includes(node.tag);
+  }
+  return false;
+}
+
+export function generate(node, indent = 0) {
+  if (node.type === 'text') {
+    return node.text;
+  }
+
+  if (node.type !== 'element') {
+    return '';
+  }
+
+  const tag = node.tag;
+  const children = node.children || [];
+
+  // If only one child and no special handling for this tag, pass through transparently
+  const hasSpecialHandling = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'ul', 'ol', 'pre', 'br', 'strong', 'b', 'em', 'i', 'code', 'a'].includes(tag);
+  if (children.length === 1 && !hasSpecialHandling) {
+    return generate(children[0], indent);
+  }
+
+  // Generate children with proper spacing
+  const parts = [];
+  for (let i = 0; i < children.length; i++) {
+    const child = children[i];
+    const childText = generate(child, indent);
+    if (childText) parts.push(childText);
+
+    // Add separator between children
+    if (i < children.length - 1 && childText) {
+      const nextChild = children[i + 1];
+      const nextText = generate(nextChild, indent);
+      if (nextText) {
+        const childIsInline = isInline(child);
+        const childIsBlock = isBlock(child);
+        const nextIsInline = isInline(nextChild);
+        const nextIsBlock = isBlock(nextChild);
+
+        if (childIsInline && nextIsInline) {
+          // inline + inline → space
+          parts.push(' ');
+        } else if (childIsBlock && nextIsBlock) {
+          // block + block → double newline
+          parts.push('\n\n');
+        } else if (childIsBlock || nextIsBlock) {
+          // block + anything or anything + block → single newline
+          parts.push('\n');
+        }
+        // container + container or inline + container → no separator (transparent)
+      }
+    }
+  }
+  const childText = parts.join('');
+
+  // Headings
+  if (tag === 'h1') return `# ${childText}`;
+  if (tag === 'h2') return `## ${childText}`;
+  if (tag === 'h3') return `### ${childText}`;
+  if (tag === 'h4') return `#### ${childText}`;
+  if (tag === 'h5') return `##### ${childText}`;
+  if (tag === 'h6') return `###### ${childText}`;
+
+  // Paragraph
+  if (tag === 'p') return childText;
+
+  // Inline formatting
+  if (tag === 'strong' || tag === 'b') return `**${childText}**`;
+  if (tag === 'em' || tag === 'i') return `*${childText}*`;
+  if (tag === 'code') return `\`${childText}\``;
+  if (tag === 'a') {
+    // Extract all text nodes and join with comma
+    const texts = [];
+    function collectText(n) {
+      if (n.type === 'text') {
+        const t = n.text.trim();
+        if (t) texts.push(t);
+      } else if (n.children) {
+        n.children.forEach(collectText);
+      }
+    }
+    children.forEach(collectText);
+    const linkText = texts.join(', ');
+    return `[${linkText}](${node.attributes.href || ''})`;
+  }
+  if (tag === 'img') {
+    const alt = node.attributes.alt || '';
+    const src = node.attributes.src || '';
+    return `![${alt}](${src})`;
+  }
+
+  // Lists
+  if (tag === 'ul' || tag === 'ol') {
+    const listContent = children.map((c, i) => {
+      if (c.type === 'element' && c.tag === 'li') {
+        const marker = tag === 'ul' ? '-' : `${i + 1}.`;
+        const content = c.children.map(ch => generate(ch, indent + 2)).join('');
+        return `${' '.repeat(indent)}${marker} ${content}\n`;
+      }
+      return '';
+    }).join('');
+    return listContent.trimEnd();
+  }
+
+  // Code block
+  if (tag === 'pre') return `\`\`\`\n${childText}\n\`\`\``;
+
+  // Line break
+  if (tag === 'br') return '\n';
+
+  // Default: just return children
+  return childText;
+}

package/src/main.js

@@ -0,0 +1,82 @@
+import { parse } from './parser.js';
+import { generate as generateMarkdown } from './generators/markdown.js';
+
+export function main(htmlInput, options = {}) {
+  // Validate input
+  if (typeof htmlInput !== 'string') {
+    throw new TypeError('htmlInput must be a string');
+  }
+
+  // Validate options
+  const outputFormat = options.outputFormat || 'markdown';
+  if (outputFormat !== 'markdown' && outputFormat !== 'json') {
+    throw new Error('options.outputFormat must be \'markdown\' or \'json\'');
+  }
+
+  const strategy = options.strategy;
+  if (strategy && strategy !== 'list' && strategy !== 'article') {
+    throw new Error('options.strategy must be \'list\', \'article\', or undefined');
+  }
+
+  // Parse HTML to virtual DOM
+  let tree = parse(htmlInput, options.removeAttributes);
+
+  // Apply extraction strategy
+  if (strategy === 'list') {
+    tree = extractLargestList(tree);
+  } else if (strategy === 'article') {
+    tree = extractArticle(tree);
+  }
+
+  // Generate output
+  if (outputFormat === 'markdown') {
+    return generateMarkdown(tree);
+  }
+
+  // JSON output - remove parent refs to avoid circular structure
+  return JSON.stringify(tree, (key, value) => key === 'parent' ? undefined : value, 2);
+}
+
+function extractLargestList(node) {
+  let largest = null;
+  let maxCount = 0;
+
+  function traverse(n) {
+    if (n.type === 'element' && (n.tag === 'ul' || n.tag === 'ol')) {
+      const count = n.children.filter(c => c.type === 'element' && c.tag === 'li').length;
+      if (count > maxCount) {
+        maxCount = count;
+        largest = n;
+      }
+    }
+    if (n.children) {
+      n.children.forEach(traverse);
+    }
+  }
+
+  traverse(node);
+  return largest || node;
+}
+
+function extractArticle(node) {
+  // Filter out sibling divs with no text content
+  function hasText(n) {
+    if (n.type === 'text' && n.text.trim()) return true;
+    if (n.children) return n.children.some(hasText);
+    return false;
+  }
+
+  function filterNode(n) {
+    if (n.type !== 'element') return n;
+
+    const filtered = { ...n, children: [] };
+    for (const child of n.children || []) {
+      if (child.type === 'text' || hasText(child)) {
+        filtered.children.push(filterNode(child));
+      }
+    }
+    return filtered;
+  }
+
+  return filterNode(node);
+}

package/src/parser.js

@@ -0,0 +1,208 @@
+import { createElement, createText, decodeEntities } from './utils.js';
+
+function cleanText(text) {
+  return text.replace(/[\u200E\u200F\u202A-\u202E]/g, '');
+}
+
+export function parse(html, removeAttributes = []) {
+  // Default blacklist (style temporarily preserved for display:none filtering)
+  const defaultBlacklist = ['loading', 'decoding', 'fetchpriority'];
+  const blacklist = [...defaultBlacklist, ...removeAttributes];
+
+  function shouldRemove(attrName) {
+    if (attrName === 'style') return false; // Preserve for filtering
+    if (attrName.startsWith('data-')) return true;
+    return blacklist.some(pattern => {
+      if (pattern.endsWith('-*')) {
+        return attrName.startsWith(pattern.slice(0, -1));
+      }
+      return attrName === pattern;
+    });
+  }
+
+  // Pre-clean: remove DOCTYPE, script, style, iframe, svg, link, source, input, comments
+  html = html.replace(/<!DOCTYPE[^>]*>/gi, '');
+  html = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
+  html = html.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
+  html = html.replace(/<iframe[^>]*>[\s\S]*?<\/iframe>/gi, '');
+  html = html.replace(/<svg[^>]*>[\s\S]*?<\/svg>/gi, '');
+  html = html.replace(/<link[^>]*>/gi, '');
+  html = html.replace(/<source[^>]*>/gi, '');
+  html = html.replace(/<input[^>]*>/gi, '');
+  html = html.replace(/<!--[\s\S]*?-->/g, '');
+
+  const stack = [];
+  const root = createElement('root', {}, []);
+  stack.push(root);
+
+  const tagRegex = /<\/?([a-z][a-z0-9]*)[^>]*>/gi;
+  let lastIndex = 0;
+  let match;
+
+  while ((match = tagRegex.exec(html)) !== null) {
+    // Add text before tag
+    if (match.index > lastIndex) {
+      const text = html.slice(lastIndex, match.index).trim();
+      if (text) {
+        const textNode = createText(decodeEntities(text));
+        const parent = stack[stack.length - 1];
+        textNode.parent = parent;
+        parent.children.push(textNode);
+      }
+    }
+
+    const fullTag = match[0];
+    const tagName = match[1].toLowerCase();
+
+    if (fullTag.startsWith('</')) {
+      // Closing tag
+      if (stack.length > 1 && stack[stack.length - 1].tag === tagName) {
+        stack.pop();
+      }
+    } else {
+      // Opening tag
+      const attrs = {};
+      const attrRegex = /([a-z][a-z0-9-]*)="([^"]*)"/gi;
+      let attrMatch;
+      let dataSrc = null;
+      while ((attrMatch = attrRegex.exec(fullTag)) !== null) {
+        const attrName = attrMatch[1];
+        if (attrName === 'data-src') dataSrc = attrMatch[2];
+        if (shouldRemove(attrName)) continue;
+        attrs[attrName] = attrMatch[2];
+      }
+
+      // For img tags, use data-src as src if src is missing
+      if (tagName === 'img' && !attrs.src && dataSrc) {
+        attrs.src = dataSrc;
+      }
+
+      const node = createElement(tagName, attrs, []);
+      const parent = stack[stack.length - 1];
+      node.parent = parent;
+      parent.children.push(node);
+
+      // Self-closing or void elements
+      const voidElements = ['br', 'hr', 'img', 'input', 'meta', 'link', 'source', 'area', 'base', 'col', 'embed', 'param', 'track', 'wbr'];
+      if (fullTag.endsWith('/>') || voidElements.includes(tagName)) {
+        // Don't push to stack
+      } else {
+        stack.push(node);
+      }
+    }
+
+    lastIndex = tagRegex.lastIndex;
+  }
+
+  // Add remaining text
+  if (lastIndex < html.length) {
+    const text = html.slice(lastIndex).trim();
+    if (text) {
+      const textNode = createText(decodeEntities(text));
+      const parent = stack[stack.length - 1];
+      textNode.parent = parent;
+      parent.children.push(textNode);
+    }
+  }
+
+  // Post-processing: flatten pre/code, flatten containers, remove unwanted nodes
+  const voidElements = ['br', 'hr', 'img'];
+  const flattenableTags = ['div', 'span', 'section', 'p'];
+
+  function flattenPreCode(node) {
+    if (node.type === 'element' && (node.tag === 'pre' || node.tag === 'code')) {
+      const texts = [];
+      function collectText(n) {
+        if (n.type === 'text') texts.push(n.text);
+        else if (n.children) n.children.forEach(collectText);
+      }
+      node.children.forEach(collectText);
+      node.children = [createText(texts.join(''))];
+    } else if (node.children) {
+      node.children.forEach(flattenPreCode);
+    }
+  }
+
+  function removeUnwantedNodes(node) {
+    if (!node.children) return;
+
+    node.children = node.children.filter(child => {
+      if (child.type === 'text') return true;
+      if (child.type === 'element') {
+        // Filter elements with display:none
+        const style = child.attributes.style;
+        if (style && /display\s*:\s*none/i.test(style)) return false;
+
+        // Filter ARIA hidden elements
+        if (child.attributes['aria-hidden'] === 'true') return false;
+        if (child.attributes.tabindex === '-1') return false;
+        if (child.attributes.hidden !== undefined) return false;
+        const role = child.attributes.role;
+        if (role === 'presentation' || role === 'none') return false;
+
+        // Filter img without src
+        if (child.tag === 'img' && !child.attributes.src) return false;
+
+        // Filter a with javascript: href
+        if (child.tag === 'a' && child.attributes.href?.startsWith('javascript:')) return false;
+
+        // Keep description and keywords meta tags, remove others
+        if (child.tag === 'meta') {
+          const name = child.attributes.name;
+          if (name === 'description' || name === 'keywords') {
+            return true;
+          }
+          return false;
+        }
+        // Keep void elements
+        if (voidElements.includes(child.tag)) return true;
+        // Recursively process children
+        removeUnwantedNodes(child);
+        // Remove style attribute after filtering
+        delete child.attributes.style;
+        // Remove empty nodes
+        if (child.children && child.children.length === 0) return false;
+      }
+      return true;
+    });
+  }
+
+  function flattenContainers(node) {
+    if (!node.children) return node;
+
+    node.children = node.children.map(child => {
+      // Collect flattened tags and classes
+      const tags = [];
+      const classes = [];
+      let current = child;
+
+      // Walk down single-child flattenable containers
+      while (current.type === 'element' &&
+             flattenableTags.includes(current.tag) &&
+             current.children?.length === 1) {
+        tags.push(current.tag);
+        if (current.attributes.class) {
+          classes.push(current.attributes.class);
+        }
+        current = current.children[0];
+      }
+
+      // If we collected any tags, attach them to the final node
+      if (tags.length > 0) {
+        current.flattenedTags = tags;
+        current.flattenedClasses = classes;
+      }
+
+      // Recursively process the final node
+      return flattenContainers(current);
+    });
+
+    return node;
+  }
+
+  flattenPreCode(root);
+  removeUnwantedNodes(root);
+  flattenContainers(root);
+
+  return root.children.length === 1 ? root.children[0] : root;
+}

package/src/utils.js

@@ -0,0 +1,35 @@
+const htmlEntities = {
+  ' ': ' ',
+  '&lt;': '<',
+  '&gt;': '>',
+  '&amp;': '&',
+  '&quot;': '"',
+  '&#39;': "'",
+  '&apos;': "'"
+};
+
+export function decodeEntity(entity) {
+  if (entity.startsWith('&#x')) {
+    return String.fromCharCode(parseInt(entity.slice(3, -1), 16));
+  }
+  if (entity.startsWith('&#')) {
+    return String.fromCharCode(parseInt(entity.slice(2, -1), 10));
+  }
+  return htmlEntities[entity] || entity;
+}
+
+export function decodeEntities(text) {
+  return text.replace(/&[#\w]+;/g, decodeEntity);
+}
+
+export function createElement(tag, attributes = {}, children = []) {
+  return { type: 'element', tag, attributes, children, parent: null };
+}
+
+function cleanText(text) {
+  return text.replace(/[\u200E\u200F\u202A-\u202E]/g, '');
+}
+
+export function createText(text) {
+  return { type: 'text', text: cleanText(text), parent: null };
+}