defuddle-js 0.1.0

package/src/removals/content-patterns.js

@@ -0,0 +1,174 @@
+/**
+ * Remove content patterns: breadcrumbs, bylines, hero headers, boilerplate.
+ * Ported from defuddle-php/src/Removals/ContentPatternRemover.php
+ */
+import { countWords } from '../utils.js';
+
+const DATE_PATTERN = /\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}/i;
+const READ_TIME_PATTERN = /\d+\s*min(?:ute)?s?\s+read\b/i;
+const STARTS_WITH_BY = /^by\s+\S/i;
+const BOILERPLATE_PATTERNS = [
+  /^This (?:article|story|piece) (?:appeared|was published|originally appeared) in\b/i,
+  /^A version of this (?:article|story) (?:appeared|was published) in\b/i,
+  /^Originally (?:published|appeared) (?:in|on|at)\b/i,
+  /^Any re-?use permitted\b/i,
+];
+
+/**
+ * Remove content patterns from the isolated content element.
+ * @param {Element} root
+ * @param {boolean} debug
+ * @param {string} url
+ * @returns {Array}
+ */
+export function removeByContentPattern(root, debug = false, url = '') {
+  const removed = [];
+  removeHeroHeader(root, debug, removed);
+  removeBreadcrumbs(root, debug, removed);
+  removeAuthorBylines(root, debug, removed);
+  removeReadTimeBlocks(root, debug, removed);
+  removeBoilerplateSentences(root, debug, removed);
+  removeTrailingThinSections(root, debug, removed);
+  return removed;
+}
+
+function removeHeroHeader(root, debug, removed) {
+  const timeNodes = Array.from(root.querySelectorAll('time'));
+  for (const time of timeNodes) {
+    const textBefore = getTextBefore(root, time);
+    if (textBefore.length > 400) continue;
+
+    let container = time.parentElement;
+    let depth = 0;
+    while (container && container !== root && depth < 4) {
+      const headings = container.querySelectorAll('h1,h2,h3').length;
+      if (headings > 0) {
+        const proseWords = countProseWords(container);
+        if (proseWords < 30) {
+          safeRemove(container, root, debug, removed, 'removeHeroHeader');
+          break;
+        }
+      }
+      container = container.parentElement;
+      depth++;
+    }
+  }
+}
+
+function removeBreadcrumbs(root, debug, removed) {
+  for (const tag of ['ul', 'ol', 'nav']) {
+    for (const list of Array.from(root.querySelectorAll(tag))) {
+      const textBefore = getTextBefore(root, list);
+      if (textBefore.length > 600) continue;
+      if (isBreadcrumbList(list)) {
+        safeRemove(list, root, debug, removed, 'removeBreadcrumbs');
+      }
+    }
+  }
+
+  for (const el of Array.from(root.querySelectorAll('[aria-label*="breadcrumb"],[class*="breadcrumb"]'))) {
+    safeRemove(el, root, debug, removed, 'removeBreadcrumbs');
+  }
+}
+
+function isBreadcrumbList(list) {
+  const children = Array.from(list.children).filter(
+    el => ['li', 'a'].includes(el.tagName.toLowerCase())
+  );
+  if (children.length < 2 || children.length > 8) return false;
+
+  const linkOnly = children.filter(item => {
+    const text = (item.textContent || '').trim();
+    const words = countWords(text);
+    if (words > 4) return false;
+    return item.querySelector('a') !== null || item.tagName.toLowerCase() === 'a';
+  });
+
+  return linkOnly.length >= children.length * 0.5;
+}
+
+function removeAuthorBylines(root, debug, removed) {
+  const candidates = Array.from(root.querySelectorAll('*:not(p *)'));
+  for (const el of candidates) {
+    const textBefore = getTextBefore(root, el);
+    if (textBefore.length > 600) continue;
+
+    const text = (el.textContent || '').trim();
+    const words = countWords(text);
+    if (words < 2 || words > 15) continue;
+
+    if (STARTS_WITH_BY.test(text)) {
+      safeRemove(el, root, debug, removed, 'removeAuthorBylines');
+    }
+  }
+}
+
+function removeReadTimeBlocks(root, debug, removed) {
+  // Leaf elements near top
+  for (const el of Array.from(root.querySelectorAll('*'))) {
+    if (el.children.length > 0) continue; // leaf elements only
+    const textBefore = getTextBefore(root, el);
+    if (textBefore.length > 400) continue;
+
+    const text = (el.textContent || '').trim();
+    if (countWords(text) > 10) continue;
+
+    if (READ_TIME_PATTERN.test(text)) {
+      safeRemove(el, root, debug, removed, 'removeReadTimeBlocks');
+    }
+  }
+}
+
+function removeBoilerplateSentences(root, debug, removed) {
+  for (const p of Array.from(root.querySelectorAll('p'))) {
+    const text = (p.textContent || '').trim();
+    for (const pattern of BOILERPLATE_PATTERNS) {
+      if (pattern.test(text)) {
+        safeRemove(p, root, debug, removed, 'removeBoilerplateSentences');
+        break;
+      }
+    }
+  }
+}
+
+function removeTrailingThinSections(root, debug, removed) {
+  const totalWords = countWords(root.textContent || '');
+  if (totalWords < 100) return;
+
+  const threshold = totalWords * 0.15;
+  const children = Array.from(root.children);
+  const lastChildren = children.slice(-3);
+
+  for (const child of lastChildren.reverse()) {
+    const tag = child.tagName.toLowerCase();
+    if (!['div', 'section', 'aside'].includes(tag)) continue;
+
+    const words = countWords(child.textContent || '');
+    if (words < threshold && child.querySelectorAll('h2,h3,h4').length > 0) {
+      safeRemove(child, root, debug, removed, 'removeTrailingThinSections');
+    }
+  }
+}
+
+function safeRemove(el, root, debug, removed, step) {
+  if (el === root || !el.parentNode) return;
+  if (debug) removed.push({ step, tag: el.tagName, text: (el.textContent || '').slice(0, 80) });
+  el.parentNode.removeChild(el);
+}
+
+function getTextBefore(root, target) {
+  const rootText = root.textContent || '';
+  const targetText = (target.textContent || '').trim().slice(0, 30);
+  const pos = rootText.indexOf(targetText);
+  return pos === -1 ? '' : rootText.slice(0, pos);
+}
+
+function countProseWords(el) {
+  let text = '';
+  for (const tag of ['p', 'li', 'dd']) {
+    for (const node of el.getElementsByTagName(tag)) {
+      text += ' ' + (node.textContent || '');
+    }
+  }
+  return countWords(text);
+}

package/src/removals/hidden.js

@@ -0,0 +1,51 @@
+/**
+ * Remove hidden elements from a document.
+ */
+import { removeAll } from '../utils.js';
+
+/**
+ * Remove elements hidden via display:none, aria-hidden, [hidden], or CSS classes.
+ * Preserves math/katex content even if marked hidden.
+ * @param {Document} doc
+ * @param {boolean} debug
+ * @returns {Array}
+ */
+export function removeHiddenElements(doc, debug = false) {
+  const removed = [];
+  const all = Array.from(doc.querySelectorAll('*'));
+  const toRemove = [];
+
+  for (const el of all) {
+    if (isHidden(el) && !containsMath(el)) {
+      toRemove.push(el);
+      if (debug) removed.push({ step: 'removeHidden', tag: el.tagName, class: el.className });
+    }
+  }
+
+  removeAll(toRemove);
+  return removed;
+}
+
+function isHidden(el) {
+  if (el.hasAttribute('hidden')) return true;
+  if (el.getAttribute('aria-hidden') === 'true') return true;
+
+  const style = el.getAttribute('style') || '';
+  if (style && /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0)\s*(?:;|$)/i.test(style)) {
+    return true;
+  }
+
+  const cls = el.getAttribute('class') || '';
+  if (cls) {
+    const tokens = cls.trim().split(/\s+/);
+    if (tokens.includes('hidden') || tokens.includes('invisible')) return true;
+  }
+
+  return false;
+}
+
+function containsMath(el) {
+  const cls = (el.getAttribute('class') || '').toLowerCase();
+  if (cls.includes('katex-mathml') || cls.includes('math')) return true;
+  return el.querySelector('math') !== null;
+}

package/src/removals/selector-remover.js

@@ -0,0 +1,137 @@
+/**
+ * Remove elements by exact CSS selectors and partial class/id patterns.
+ */
+import {
+  EXACT_SELECTORS,
+  PARTIAL_SELECTORS_REGEX,
+  PARTIAL_MATCH_ATTRIBUTES,
+  FOOTNOTE_SELECTORS,
+  VIDEO_EMBED_PATTERNS,
+} from '../constants.js';
+import { isAncestorOrSelf, removeAll } from '../utils.js';
+
+/**
+ * Remove elements matching exact CSS selectors.
+ * @param {Document} doc
+ * @param {Element|null} mainContent
+ * @param {boolean} debug
+ * @returns {Array}
+ */
+export function removeExact(doc, mainContent = null, debug = false) {
+  const removed = [];
+  const toRemove = new Set();
+
+  for (const selector of EXACT_SELECTORS) {
+    try {
+      for (const el of doc.querySelectorAll(selector)) {
+        if (mainContent && isAncestorOrSelf(el, mainContent)) continue;
+        if (isInsideCodeBlock(el)) continue;
+        if (isFootnoteContainer(doc, el)) continue;
+        toRemove.add(el);
+      }
+    } catch (e) {
+      // Skip unsupported selectors
+    }
+  }
+
+  // Special: scripts (not math)
+  for (const el of doc.querySelectorAll('script')) {
+    const type = (el.getAttribute('type') || '').toLowerCase();
+    if (type.startsWith('math/')) continue;
+    if (mainContent && isAncestorOrSelf(el, mainContent)) continue;
+    toRemove.add(el);
+  }
+
+  // Special: iframes — keep video embeds
+  for (const el of doc.querySelectorAll('iframe')) {
+    const src = (el.getAttribute('src') || '').toLowerCase();
+    if (VIDEO_EMBED_PATTERNS.some(p => src.includes(p))) continue;
+    if (mainContent && isAncestorOrSelf(el, mainContent)) continue;
+    toRemove.add(el);
+  }
+
+  // Special: aside (keep callouts)
+  for (const el of doc.querySelectorAll('aside')) {
+    const cls = (el.getAttribute('class') || '').toLowerCase();
+    if (cls.includes('callout')) continue;
+    if (mainContent && isAncestorOrSelf(el, mainContent)) continue;
+    toRemove.add(el);
+  }
+
+  for (const el of toRemove) {
+    if (el.parentNode) {
+      if (debug) removed.push({ step: 'removeExact', tag: el.tagName, class: el.className });
+      el.parentNode.removeChild(el);
+    }
+  }
+
+  return removed;
+}
+
+/**
+ * Remove elements whose class/id matches partial selector patterns.
+ * @param {Document} doc
+ * @param {Element|null} mainContent
+ * @param {boolean} debug
+ * @returns {Array}
+ */
+export function removePartial(doc, mainContent = null, debug = false) {
+  const removed = [];
+  const attrSelector = PARTIAL_MATCH_ATTRIBUTES.map(a => `[${a}]`).join(',');
+
+  let elements;
+  try {
+    elements = Array.from(doc.querySelectorAll(attrSelector));
+  } catch (e) {
+    return removed;
+  }
+
+  const toRemove = [];
+
+  for (const el of elements) {
+    const tag = el.tagName.toLowerCase();
+    if (tag === 'code' || tag === 'pre') continue;
+    if (isInsideCodeBlock(el)) continue;
+
+    const combined = PARTIAL_MATCH_ATTRIBUTES
+      .map(a => el.getAttribute(a) || '')
+      .join(' ');
+
+    if (!PARTIAL_SELECTORS_REGEX.test(combined)) continue;
+    if (mainContent && isAncestorOrSelf(el, mainContent)) continue;
+    if (isFootnoteContainer(doc, el)) continue;
+
+    toRemove.push(el);
+  }
+
+  for (const el of toRemove) {
+    if (el.parentNode) {
+      if (debug) removed.push({ step: 'removePartial', tag: el.tagName, class: el.className });
+      el.parentNode.removeChild(el);
+    }
+  }
+
+  return removed;
+}
+
+function isInsideCodeBlock(el) {
+  let parent = el.parentElement;
+  while (parent) {
+    const tag = parent.tagName.toLowerCase();
+    if (tag === 'code' || tag === 'pre') return true;
+    parent = parent.parentElement;
+  }
+  return false;
+}
+
+function isFootnoteContainer(doc, el) {
+  for (const sel of FOOTNOTE_SELECTORS) {
+    try {
+      const matches = doc.querySelectorAll(sel);
+      for (const m of matches) {
+        if (m === el) return true;
+      }
+    } catch (e) {}
+  }
+  return false;
+}

package/src/removals/small-images.js

@@ -0,0 +1,45 @@
+/**
+ * Remove images smaller than MIN_IMAGE_DIMENSION.
+ */
+import { MIN_IMAGE_DIMENSION } from '../constants.js';
+
+/**
+ * @param {Document} doc
+ * @param {boolean} debug
+ * @returns {number} count removed
+ */
+export function removeSmallImages(doc, debug = false) {
+  const toRemove = [];
+
+  for (const img of doc.querySelectorAll('img, svg')) {
+    const [w, h] = getDimensions(img);
+    if ((w !== null && w < MIN_IMAGE_DIMENSION) || (h !== null && h < MIN_IMAGE_DIMENSION)) {
+      toRemove.push(img);
+    }
+  }
+
+  for (const img of toRemove) {
+    if (img.parentNode) img.parentNode.removeChild(img);
+  }
+
+  return toRemove.length;
+}
+
+function getDimensions(el) {
+  let w = null, h = null;
+
+  const wAttr = el.getAttribute('width');
+  const hAttr = el.getAttribute('height');
+  if (wAttr && /^\d+$/.test(wAttr)) w = parseInt(wAttr);
+  if (hAttr && /^\d+$/.test(hAttr)) h = parseInt(hAttr);
+
+  const style = el.getAttribute('style') || '';
+  if (style) {
+    const wm = style.match(/\bwidth\s*:\s*(\d+)px/i);
+    const hm = style.match(/\bheight\s*:\s*(\d+)px/i);
+    if (wm) w = parseInt(wm[1]);
+    if (hm) h = parseInt(hm[1]);
+  }
+
+  return [w, h];
+}

package/src/schema-org.js

@@ -0,0 +1,102 @@
+/**
+ * Schema.org JSON-LD extractor for defuddle-js.
+ */
+
+/**
+ * Extract Schema.org JSON-LD data from a document.
+ * @param {Document} doc
+ * @returns {Array|null}
+ */
+export function extractSchemaOrg(doc) {
+  const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
+  if (!scripts.length) return null;
+
+  const items = [];
+
+  for (const script of scripts) {
+    const content = script.textContent || '';
+    if (!content.trim()) continue;
+
+    const parsed = parseJsonLd(content);
+    if (!parsed) continue;
+
+    if (parsed['@graph'] && Array.isArray(parsed['@graph'])) {
+      parsed['@graph'].forEach(item => items.push(decodeStrings(item)));
+    } else if (Array.isArray(parsed)) {
+      parsed.forEach(item => items.push(decodeStrings(item)));
+    } else {
+      items.push(decodeStrings(parsed));
+    }
+  }
+
+  return items.length > 0 ? items : null;
+}
+
+/**
+ * Parse a JSON-LD string, stripping comments and CDATA wrappers.
+ * @param {string} content
+ * @returns {object|null}
+ */
+function parseJsonLd(content) {
+  // Strip block comments /* ... */
+  content = content.replace(/\/\*[\s\S]*?\*\//g, '');
+  // Strip line comments // ...
+  content = content.replace(/^\s*\/\/.*$/gm, '');
+  // Strip CDATA wrappers
+  content = content.replace(/^\s*\/\/<!\[CDATA\[|\]\]>\/\/\s*$/gm, '');
+  content = content.replace(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/, '$1');
+  content = content.trim();
+
+  if (!content) return null;
+
+  try {
+    return JSON.parse(content);
+  } catch (e) {
+    return null;
+  }
+}
+
+/**
+ * Recursively decode HTML entities in all string values.
+ * @param {any} item
+ * @returns {any}
+ */
+function decodeStrings(item) {
+  if (typeof item === 'string') {
+    return item
+      .replace(/&amp;/g, '&')
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&quot;/g, '"')
+      .replace(/&#39;/g, "'")
+      .replace(/&apos;/g, "'");
+  }
+  if (Array.isArray(item)) return item.map(decodeStrings);
+  if (item && typeof item === 'object') {
+    const result = {};
+    for (const [key, val] of Object.entries(item)) {
+      result[key] = decodeStrings(val);
+    }
+    return result;
+  }
+  return item;
+}
+
+/**
+ * Get a property value from schema.org data using dot-notation path.
+ * @param {Array|null} items
+ * @param {string[]} types  @type values to match
+ * @param {string} property
+ * @returns {any}
+ */
+export function getSchemaProperty(items, types, property) {
+  if (!items) return null;
+  for (const item of items) {
+    if (!item || typeof item !== 'object') continue;
+    const itemType = item['@type'] || '';
+    if (types.length > 0 && !types.includes(itemType) && itemType !== '') continue;
+    const val = item[property];
+    if (val !== undefined && val !== null && val !== '') return val;
+  }
+  return null;
+}

package/src/standardizer.js

@@ -0,0 +1,116 @@
+/**
+ * HTML standardization for defuddle-js.
+ */
+import { ALLOWED_ATTRIBUTES, ALLOWED_EMPTY_TAGS } from './constants.js';
+
+/**
+ * Standardize HTML in a content element.
+ * @param {Element} root
+ */
+export function standardize(root) {
+  removeComments(root);
+  normalizeNbsp(root);
+  convertH1ToH2(root);
+  stripAttributes(root);
+  removeEmptyElements(root);
+}
+
+function removeComments(root) {
+  const walker = document.createTreeWalker
+    ? createTreeWalker(root)
+    : null;
+
+  // Use XPath-style traversal since createTreeWalker may not exist in all environments
+  const comments = [];
+  const iter = (node) => {
+    for (const child of Array.from(node.childNodes)) {
+      if (child.nodeType === 8 /* COMMENT_NODE */) {
+        comments.push(child);
+      } else if (child.childNodes && child.childNodes.length) {
+        iter(child);
+      }
+    }
+  };
+  iter(root);
+  for (const c of comments) {
+    if (c.parentNode) c.parentNode.removeChild(c);
+  }
+}
+
+function normalizeNbsp(root) {
+  const iter = (node) => {
+    for (const child of Array.from(node.childNodes)) {
+      if (child.nodeType === 3 /* TEXT_NODE */) {
+        if (child.nodeValue && child.nodeValue.includes('\u00A0')) {
+          child.nodeValue = child.nodeValue.replace(/\u00A0/g, ' ');
+        }
+      } else if (child.childNodes && child.childNodes.length) {
+        iter(child);
+      }
+    }
+  };
+  iter(root);
+}
+
+function convertH1ToH2(root) {
+  const h1s = Array.from(root.querySelectorAll('h1'));
+  for (const h1 of h1s) {
+    const h2 = root.ownerDocument
+      ? root.ownerDocument.createElement('h2')
+      : h1.cloneNode(false);
+    h2.innerHTML = h1.innerHTML;
+    // Copy attributes
+    for (const attr of Array.from(h1.attributes)) {
+      h2.setAttribute(attr.name, attr.value);
+    }
+    if (h1.parentNode) h1.parentNode.replaceChild(h2, h1);
+  }
+}
+
+function stripAttributes(root) {
+  for (const el of root.querySelectorAll('*')) {
+    const toRemove = [];
+    for (const attr of Array.from(el.attributes)) {
+      const name = attr.name.toLowerCase();
+      if (name.startsWith('aria-')) continue;
+      if (ALLOWED_ATTRIBUTES.has(name)) continue;
+      // Keep footnote-related id attributes
+      if (name === 'id') {
+        const id = (attr.value || '').toLowerCase();
+        if (id.startsWith('fn') || id.startsWith('ref') || id.startsWith('footnote')) continue;
+      }
+      toRemove.push(attr.name);
+    }
+    for (const name of toRemove) el.removeAttribute(name);
+  }
+}
+
+function removeEmptyElements(root) {
+  const emptyTags = ['p', 'div', 'span', 'section', 'aside', 'li'];
+  let changed = true;
+  while (changed) {
+    changed = false;
+    for (const tag of emptyTags) {
+      for (const el of Array.from(root.getElementsByTagName(tag))) {
+        if (el === root) continue;
+        if (ALLOWED_EMPTY_TAGS.has(tag)) continue;
+        if (isEffectivelyEmpty(el)) {
+          if (el.parentNode) {
+            el.parentNode.removeChild(el);
+            changed = true;
+          }
+        }
+      }
+    }
+  }
+}
+
+function isEffectivelyEmpty(el) {
+  if ((el.textContent || '').trim()) return false;
+  const MEDIA = ['img', 'video', 'audio', 'iframe', 'svg', 'picture', 'canvas'];
+  return !MEDIA.some(tag => el.getElementsByTagName(tag).length > 0);
+}
+
+function createTreeWalker(root) {
+  return null; // placeholder — we use our own traversal above
+}