@agenticmail/enterprise 0.5.327 → 0.5.329

package/src/engine/knowledge-import/processors/clean.ts

@@ -1,149 +0,0 @@
-/**
- * Layer 2: CLEAN — Deep Content Cleaning & Normalization
- *
- * Source-agnostic. Takes extracted text and applies multiple cleaning passes:
- *   Pass 1: Structural — Remove boilerplate patterns, navigation, UI artifacts
- *   Pass 2: Content   — Normalize formatting, fix encoding, deduplicate content
- *   Pass 3: Quality   — Remove low-value content, trim noise, normalize whitespace
- *
- * Every pass reports what it removed for auditability.
- */
-
-import type { LayerReport } from './types.js';
-
-export interface CleanResult {
-  content: string;
-  report: LayerReport;
-}
-
-export function cleanContent(input: string): CleanResult {
-  const start = Date.now();
-  const operations: string[] = [];
-  const inputSize = input.length;
-
-  let text = input;
-
-  // ─── Pass 1: Structural Cleaning ──────────────────
-
-  const beforePass1 = text.length;
-
-  // Remove anchor references (#section-id)
-  text = text.replace(/\(#[a-zA-Z0-9_-]+\)/g, '');
-  // Remove empty markdown links
-  text = text.replace(/\[([^\]]*)\]\(\s*\)/g, '$1');
-  // Remove empty anchor tags leftovers
-  text = text.replace(/\[\s*\]\([^)]*\)/g, '');
-  // Remove "Skip to main content", "Skip to navigation", etc.
-  text = text.replace(/^.*skip\s+to\s+(?:main\s+)?(?:content|navigation|search).*$/gim, '');
-  // Remove breadcrumb trails (Home > Section > Page)
-  text = text.replace(/^(?:Home|Main)\s*(?:>|›|»|→)\s*(?:\w[\w\s]*(?:>|›|»|→)\s*)+\w[\w\s]*$/gm, '');
-  // Remove breadcrumbs with < style (< "Page" < "Section")
-  text = text.replace(/(?:<\s*"[^"]*"\s*)+/g, '');
-  // Remove "Table of contents" headers + their content (list of # links)
-  text = text.replace(/#{1,3}\s*(?:Table of Contents|Contents|In this (?:article|page|guide))\s*\n(?:\s*[-*]\s*\[.*?\].*\n?)*/gi, '');
-  text = text.replace(/^(?:Table of [Cc]ontents|On this page|In this article)\s*$/gm, '');
-  // Remove standalone "- - - -" separators
-  text = text.replace(/^[\s\-–—*=]{4,}$/gm, '');
-  // Remove page numbers
-  text = text.replace(/^(?:Page\s+)?\d+\s*(?:of\s+\d+)?\s*$/gm, '');
-
-  if (text.length < beforePass1) operations.push(`pass1:structural removed ${beforePass1 - text.length} chars`);
-
-  // ─── Pass 2: Content Normalization ────────────────
-
-  const beforePass2 = text.length;
-
-  // Fix broken UTF-8 / mojibake
-  text = text.replace(/’/g, "'").replace(/“/g, '"').replace(/â€\u009D/g, '"');
-  text = text.replace(/â€"/g, '—').replace(/â€"/g, '–').replace(/ /g, ' ');
-  text = text.replace(/é/g, 'é').replace(/è/g, 'è').replace(/ü/g, 'ü').replace(/ö/g, 'ö');
-
-  // Normalize quotes and dashes
-  text = text.replace(/[\u2018\u2019\u201A\u201B]/g, "'");
-  text = text.replace(/[\u201C\u201D\u201E\u201F]/g, '"');
-  text = text.replace(/[\u2013\u2014]/g, '-');
-  text = text.replace(/\u2026/g, '...');
-
-  // Normalize bullet points
-  text = text.replace(/^[\s]*[•●○◦▪▸►▹‣⁃]\s*/gm, '- ');
-
-  // Remove zero-width characters
-  text = text.replace(/[\u200B\u200C\u200D\uFEFF\u00AD]/g, '');
-
-  // Remove excessive markdown emphasis (****text****)
-  text = text.replace(/\*{3,}([^*]+)\*{3,}/g, '**$1**');
-
-  // Normalize markdown headings (ensure space after #)
-  text = text.replace(/^(#{1,6})([^ #\n])/gm, '$1 $2');
-
-  // Remove duplicate headings (same heading appearing twice in a row)
-  text = text.replace(/(^#{1,6}\s+.+$)\n+\1/gm, '$1');
-
-  // Remove repeated content blocks (paragraphs appearing multiple times)
-  const paragraphs = text.split(/\n{2,}/);
-  const seen = new Set<string>();
-  const deduped: string[] = [];
-  for (const p of paragraphs) {
-    const normalized = p.trim().toLowerCase().replace(/\s+/g, ' ');
-    if (normalized.length < 10 || !seen.has(normalized)) {
-      deduped.push(p);
-      if (normalized.length >= 10) seen.add(normalized);
-    }
-  }
-  text = deduped.join('\n\n');
-
-  if (text.length < beforePass2) operations.push(`pass2:normalize removed ${beforePass2 - text.length} chars, deduped ${paragraphs.length - deduped.length} blocks`);
-
-  // ─── Pass 3: Quality Trim ────────────────────────
-
-  const beforePass3 = text.length;
-
-  // Remove "Written by..." bylines
-  text = text.replace(/(?:written|authored|published|posted)\s+by\s+[\w\s.]+(?:updated|modified|edited)?\s*[\w\s]*(?:ago)?/gi, '');
-  // Remove "Did this answer your question?" type feedback
-  text = text.replace(/(?:did\s+this\s+(?:answer\s+your\s+question|help)|was\s+this\s+(?:article\s+)?helpful|rate\s+this\s+article|thumbs\s+up|thumbs\s+down|yes\s+no)[^.]*[.?]?\s*[😞😐😃🙁😊👍👎☹️🙂😀\s]*/gi, '');
-  // Remove "Share this article" / social sharing prompts
-  text = text.replace(/(?:share\s+(?:this\s+)?(?:article|page|post)|tweet\s+this|share\s+on\s+(?:facebook|twitter|linkedin|x))[^.]*[.?]?/gi, '');
-  // Remove "Subscribe to newsletter" prompts
-  text = text.replace(/(?:subscribe\s+to\s+(?:our|the)?\s*(?:newsletter|updates|blog)|sign\s+up\s+for\s+(?:updates|our))[^.]*[.?]?/gi, '');
-  // Remove cookie/privacy notices
-  text = text.replace(/(?:we\s+use\s+cookies|cookie\s+(?:policy|preferences|settings)|accept\s+(?:all\s+)?cookies|by\s+continuing\s+to\s+(?:use|browse))[^.]*\.?/gi, '');
-  // Remove "Copyright ©" lines
-  text = text.replace(/^.*(?:copyright|©|\(c\))\s*\d{4}.*$/gim, '');
-  // Remove "All rights reserved"
-  text = text.replace(/all\s+rights?\s+reserved\.?/gi, '');
-  // Remove "Last updated/modified on..."
-  text = text.replace(/(?:last\s+)?(?:updated|modified|edited|reviewed)\s+(?:on\s+)?(?:\d{1,2}[-/]\d{1,2}[-/]\d{2,4}|\w+\s+\d{1,2},?\s+\d{4}|\d+\s+\w+\s+ago)/gi, '');
-  // Remove "Print this page" / "Download PDF"
-  text = text.replace(/(?:print\s+this\s+(?:page|article)|download\s+(?:as\s+)?pdf|export\s+to\s+pdf)/gi, '');
-  // Remove standalone URLs on their own line
-  text = text.replace(/^\s*https?:\/\/[^\s]+\s*$/gm, '');
-  // Remove emoji-only lines
-  text = text.replace(/^[\s]*(?:[\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F1E0}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]+[\s]*)+$/gmu, '');
-  // Remove navigation prompts ("Back to top", "Next article", "Previous page")
-  text = text.replace(/^.*(?:back\s+to\s+top|next\s+(?:article|page|step)|previous\s+(?:article|page|step)|go\s+(?:back|to\s+top)).*$/gim, '');
-
-  // Final whitespace normalization
-  text = text
-    .split('\n')
-    .map(line => line.trimEnd())    // trim trailing spaces per line
-    .join('\n')
-    .replace(/\n{4,}/g, '\n\n\n')  // max 2 blank lines
-    .replace(/^\n+/, '')            // no leading blank lines
-    .replace(/\n+$/, '')            // no trailing blank lines
-    .trim();
-
-  if (text.length < beforePass3) operations.push(`pass3:quality removed ${beforePass3 - text.length} chars`);
-
-  return {
-    content: text,
-    report: {
-      name: 'clean',
-      inputSize,
-      outputSize: text.length,
-      removedBytes: inputSize - text.length,
-      operations,
-      durationMs: Date.now() - start,
-    },
-  };
-}

package/src/engine/knowledge-import/processors/extract-gdrive.ts

@@ -1,102 +0,0 @@
-/**
- * Layer 1: EXTRACT — Google Drive / Google Sites Content Extractor
- *
- * Handles: Google Docs (HTML export), Google Sites pages, Google Slides (text export).
- * Strips: Google-specific wrapper divs, suggested edits, comment markers,
- *         revision metadata, Google Sites navigation/chrome.
- */
-
-import type { ContentExtractor, ExtractResult } from './types.js';
-
-export class GoogleDriveContentExtractor implements ContentExtractor {
-  extract(raw: string, sourceUrl?: string): ExtractResult {
-    const isGoogleSites = sourceUrl?.includes('sites.google.com') || raw.includes('tyJCtd');
-    const isGoogleDoc = raw.includes('docs-internal-guid') || raw.includes('kix-');
-
-    let content = raw;
-    let title = 'Untitled';
-
-    if (isGoogleSites) {
-      ({ content, title } = this.extractGoogleSites(raw));
-    } else if (isGoogleDoc) {
-      ({ content, title } = this.extractGoogleDoc(raw));
-    } else {
-      // Generic: might be a plain text/markdown file from Drive
-      title = this.extractTitle(raw);
-    }
-
-    // Convert any remaining HTML to text
-    if (content.includes('<') && content.includes('>')) {
-      content = this.htmlToText(content);
-    }
-
-    return { title, content: content.trim(), contentType: 'text' };
-  }
-
-  private extractGoogleSites(html: string): { content: string; title: string } {
-    // Google Sites wraps content in divs with specific classes
-    const title = (html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || 'Untitled')
-      .replace(/<[^>]+>/g, '').replace(/\s*-\s*Google Sites\s*$/i, '').trim();
-
-    // Remove Google Sites navigation, header, footer
-    let content = html
-      .replace(/<[^>]*class="[^"]*(?:navigation|yp5lHe|MnRGSb|VsJjTc|QxVDse|dR43Bb)[^"]*"[^>]*>[\s\S]*?<\/[^>]+>/gi, '')
-      // Remove edit buttons, share buttons
-      .replace(/<[^>]*class="[^"]*(?:edit-button|share-button|fab-container)[^"]*"[^>]*>[\s\S]*?<\/[^>]+>/gi, '');
-
-    // Try to extract main content area
-    const mainContent = this.extractByClass(content, 'tyJCtd') || this.extractByClass(content, 'IFjolb');
-    if (mainContent) content = mainContent;
-
-    return { content, title };
-  }
-
-  private extractGoogleDoc(html: string): { content: string; title: string } {
-    const title = (html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] || 'Untitled')
-      .replace(/<[^>]+>/g, '').trim();
-
-    let content = html
-      // Remove Google Docs internal GUIDs
-      .replace(/\s*id="docs-internal-guid-[^"]*"/gi, '')
-      // Remove kix-* markers
-      .replace(/<[^>]*class="[^"]*kix-[^"]*"[^>]*>/gi, '')
-      // Remove suggested edits
-      .replace(/<[^>]*class="[^"]*(?:docos-suggestion|docos-rewrite)[^"]*"[^>]*>[\s\S]*?<\/[^>]+>/gi, '')
-      // Remove comment anchors
-      .replace(/<[^>]*class="[^"]*docos-comment[^"]*"[^>]*>[\s\S]*?<\/[^>]+>/gi, '')
-      // Remove Google font imports
-      .replace(/<link[^>]*fonts\.googleapis[^>]*>/gi, '')
-      // Clean empty styled spans
-      .replace(/<span\s+style="[^"]*(?:font-weight:\s*400|font-style:\s*normal)[^"]*">([\s\S]*?)<\/span>/gi, '$1');
-
-    return { content, title };
-  }
-
-  private extractTitle(raw: string): string {
-    const h1 = raw.match(/^#\s+(.+)/m);
-    if (h1) return h1[1].trim();
-    const title = raw.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-    if (title) return title[1].replace(/<[^>]+>/g, '').trim();
-    return 'Untitled';
-  }
-
-  private extractByClass(html: string, className: string): string | null {
-    const regex = new RegExp(`<([a-z][a-z0-9]*)\\s[^>]*class="[^"]*${className}[^"]*"[^>]*>([\\s\\S]*?)<\\/\\1>`, 'i');
-    const match = regex.exec(html);
-    return match ? match[2] : null;
-  }
-
-  private htmlToText(html: string): string {
-    return html
-      .replace(/<(script|style|noscript)[^>]*>[\s\S]*?<\/\1>/gi, '')
-      .replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, l, t) => '\n\n' + '#'.repeat(parseInt(l)) + ' ' + t.replace(/<[^>]+>/g, '').trim() + '\n\n')
-      .replace(/<\/p>/gi, '\n\n').replace(/<p[^>]*>/gi, '')
-      .replace(/<br\s*\/?>/gi, '\n')
-      .replace(/<li[^>]*>/gi, '- ').replace(/<\/li>/gi, '\n')
-      .replace(/<[^>]+>/g, '')
-      .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
-      .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ')
-      .replace(/\n{3,}/g, '\n\n')
-      .trim();
-  }
-}

package/src/engine/knowledge-import/processors/extract-github.ts

@@ -1,74 +0,0 @@
-/**
- * Layer 1: EXTRACT — GitHub Content Extractor
- *
- * Handles: README.md, docs/, wiki, markdown files, RST, AsciiDoc.
- * Strips GitHub-specific artifacts: badges, CI status, contribution guides,
- * auto-generated TOCs, sponsor sections, PR templates.
- */
-
-import type { ContentExtractor, ExtractResult } from './types.js';
-
-export class GitHubContentExtractor implements ContentExtractor {
-  extract(raw: string, sourceUrl?: string): ExtractResult {
-    let content = raw;
-    const _isReadme = sourceUrl?.toLowerCase().includes('readme') || false;
-
-    // Remove GitHub badge images at top of README
-    content = content.replace(/^\s*(\[!\[[^\]]*\]\([^)]*\)\]\([^)]*\)\s*)+/gm, '');
-    // Remove inline badge images
-    content = content.replace(/!\[(?:build|ci|test|coverage|license|npm|version|downloads|stars|forks|issues|badge)[^\]]*\]\([^)]*\)/gi, '');
-
-    // Remove auto-generated TOC markers
-    content = content.replace(/<!--\s*(?:TOC|toc|table-of-contents)\s*-->/gi, '');
-    content = content.replace(/<!--\s*(?:START|END)\s+(?:TOC|toc|doctoc)[^-]*-->/gi, '');
-    // Remove doctoc-generated TOC blocks
-    content = content.replace(/<!--\s*START doctoc[\s\S]*?END doctoc\s*-->/gi, '');
-
-    // Remove HTML comments entirely
-    content = content.replace(/<!--[\s\S]*?-->/g, '');
-
-    // Remove sponsor/funding sections
-    content = content.replace(/#{1,3}\s*(?:Sponsors?|Funding|Support(?:ers)?|Backers?|Donate|Patron)\s*\n[\s\S]*?(?=\n#{1,3}\s|\n*$)/gi, '');
-
-    // Remove contributing section if it's just a link
-    content = content.replace(/#{1,3}\s*Contributing\s*\n+(?:(?:Please\s+)?(?:see|read|check)\s+.*CONTRIBUTING.*\n?)+/gi, '');
-
-    // Remove license section if short
-    content = content.replace(/#{1,3}\s*License\s*\n+(?:(?:MIT|Apache|BSD|ISC|GPL)[^\n]*\n?){1,3}/gi, '');
-
-    // Remove "Table of Contents" sections with only links
-    content = content.replace(/#{1,3}\s*(?:Table of Contents|Contents|TOC)\s*\n(?:\s*[-*]\s*\[.*?\]\(#.*?\)\s*\n?)+/gi, '');
-
-    // Remove GitHub-specific action prompts
-    content = content.replace(/^\s*> \[!(?:NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]\s*$/gm, '');
-
-    // Extract title from first heading
-    const titleMatch = content.match(/^#\s+(.+)/m);
-    const title = titleMatch ? titleMatch[1].trim() : this.titleFromPath(sourceUrl || '');
-
-    // Clean up excessive blank lines
-    content = content.replace(/\n{4,}/g, '\n\n\n');
-
-    return {
-      title,
-      content: content.trim(),
-      contentType: 'markdown',
-      sections: this.extractSections(content),
-    };
-  }
-
-  private titleFromPath(path: string): string {
-    const filename = path.split('/').pop() || path;
-    return filename.replace(/\.[^.]+$/, '').replace(/[-_]/g, ' ');
-  }
-
-  private extractSections(content: string): string[] {
-    const sections: string[] = [];
-    const headingRegex = /^#{1,3}\s+(.+)/gm;
-    let match;
-    while ((match = headingRegex.exec(content)) !== null) {
-      sections.push(match[1].trim());
-    }
-    return sections;
-  }
-}

package/src/engine/knowledge-import/processors/extract-sharepoint.ts

@@ -1,69 +0,0 @@
-/**
- * Layer 1: EXTRACT — SharePoint/OneDrive Content Extractor
- *
- * Handles: SharePoint pages, Word docs (HTML export), OneDrive files.
- * Strips: SharePoint chrome, metadata panels, version history, ribbon UI,
- *         Microsoft-specific markup (mso-*, o:p, v:shape).
- */
-
-import type { ContentExtractor, ExtractResult } from './types.js';
-
-export class SharePointContentExtractor implements ContentExtractor {
-  extract(raw: string, _sourceUrl?: string): ExtractResult {
-    let content = raw;
-
-    // If HTML (SharePoint page or Word export)
-    if (content.includes('<html') || content.includes('<div') || content.includes('mso-')) {
-      content = this.cleanSharePointHtml(content);
-    }
-
-    // Extract title
-    const titleMatch = content.match(/<title[^>]*>([\s\S]*?)<\/title>/i)
-      || content.match(/^#\s+(.+)/m);
-    const title = titleMatch
-      ? titleMatch[1].replace(/<[^>]+>/g, '').replace(/\s*[-–|]\s*SharePoint.*$/i, '').trim()
-      : 'Untitled';
-
-    // Convert remaining HTML to text
-    content = this.htmlToText(content);
-
-    return { title, content: content.trim(), contentType: 'text' };
-  }
-
-  private cleanSharePointHtml(html: string): string {
-    return html
-      // Remove Microsoft Office markup
-      .replace(/<o:p>[\s\S]*?<\/o:p>/gi, '')
-      .replace(/<v:[^>]*>[\s\S]*?<\/v:[^>]+>/gi, '')
-      .replace(/<w:[^>]*>[\s\S]*?<\/w:[^>]+>/gi, '')
-      // Remove mso-* styles (Word HTML export junk)
-      .replace(/\s*mso-[^;:"']+:[^;:"']+;?/gi, '')
-      .replace(/\s*style="[\s;]*"/gi, '')
-      // Remove SharePoint UI elements
-      .replace(/<[^>]*(?:class|id)="[^"]*(?:ms-rte|ms-webpart|ms-core|ms-srch|ms-nav|ms-menu|ExternalClass|MsoNormal)[^"]*"[^>]*>[\s\S]*?<\/[^>]+>/gi, '')
-      // Remove empty spans from Word export
-      .replace(/<span[^>]*>\s*<\/span>/gi, '')
-      // Remove conditional comments (IE-specific)
-      .replace(/<!--\[if[\s\S]*?<!\[endif\]-->/gi, '')
-      // Remove SharePoint metadata tables
-      .replace(/<table[^>]*(?:class|id)="[^"]*(?:propertysheet|metadata|versionTable)[^"]*"[^>]*>[\s\S]*?<\/table>/gi, '')
-      // Remove ribbon/command bar
-      .replace(/<[^>]*(?:class|id)="[^"]*(?:ribbon|commandBar|suiteBar|globalNav)[^"]*"[^>]*>[\s\S]*?<\/[^>]+>/gi, '');
-  }
-
-  private htmlToText(html: string): string {
-    return html
-      .replace(/<(script|style|noscript)[^>]*>[\s\S]*?<\/\1>/gi, '')
-      .replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, l, t) => '\n\n' + '#'.repeat(parseInt(l)) + ' ' + t.replace(/<[^>]+>/g, '').trim() + '\n\n')
-      .replace(/<\/p>/gi, '\n\n').replace(/<p[^>]*>/gi, '')
-      .replace(/<br\s*\/?>/gi, '\n')
-      .replace(/<li[^>]*>/gi, '- ').replace(/<\/li>/gi, '\n')
-      .replace(/<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, '**$1**')
-      .replace(/<(?:em|i)[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, '*$1*')
-      .replace(/<[^>]+>/g, '')
-      .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
-      .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ')
-      .replace(/\n{3,}/g, '\n\n')
-      .trim();
-  }
-}

package/src/engine/knowledge-import/processors/extract-web.ts

@@ -1,275 +0,0 @@
-/**
- * Layer 1: EXTRACT — Web/URL Content Extractor
- *
- * Handles: Intercom, Zendesk, Freshdesk, GitBook, ReadTheDocs, Docusaurus,
- *          MkDocs, generic help centers, blogs, docs sites.
- *
- * Strategy: Detect the platform → use platform-specific selectors → fallback to generic.
- */
-
-import type { ContentExtractor, ExtractResult } from './types.js';
-
-interface PlatformRule {
-  detect: (html: string, url?: string) => boolean;
-  name: string;
-  // CSS-style selectors to find main content (checked in order)
-  contentSelectors: string[];
-  // Elements to remove before extraction
-  removeSelectors: string[];
-  // Title extraction
-  titleSelector?: string;
-}
-
-const PLATFORM_RULES: PlatformRule[] = [
-  {
-    name: 'intercom',
-    detect: (html, url) => !!(url?.includes('intercom.help') || html.includes('intercom-container') || html.includes('intercom')),
-    contentSelectors: ['article__body', 'article-body', 'c__2GA', 'article__content'],
-    removeSelectors: ['intercom-reaction', 'article__actions', 'article-footer', 'article__footer', 'related-articles', 'feedback'],
-    titleSelector: 'article__title',
-  },
-  {
-    name: 'zendesk',
-    detect: (html, url) => !!(url?.includes('zendesk.com') || html.includes('zd-') || html.includes('zendesk')),
-    contentSelectors: ['article-body', 'article_body', 'zd-article-body'],
-    removeSelectors: ['article-votes', 'article-footer', 'article-sidebar', 'share-', 'follow-article'],
-  },
-  {
-    name: 'freshdesk',
-    detect: (html, url) => !!(url?.includes('freshdesk.com') || html.includes('freshdesk')),
-    contentSelectors: ['article-body', 'solution-article-body', 'fr-element-'],
-    removeSelectors: ['article-feedback', 'article-footer', 'article-tools'],
-  },
-  {
-    name: 'gitbook',
-    detect: (html, url) => !!(url?.includes('gitbook.io') || html.includes('gitbook') || html.includes('BookBody')),
-    contentSelectors: ['page-inner', 'page-body', 'markdown-section', 'page-content-wrapper'],
-    removeSelectors: ['page-footer', 'navigation', 'header', 'search-'],
-  },
-  {
-    name: 'docusaurus',
-    detect: (html) => html.includes('docusaurus') || html.includes('docs-doc-id'),
-    contentSelectors: ['markdown', 'docMainContainer', 'docs-doc-page'],
-    removeSelectors: ['pagination-nav', 'table-of-contents', 'theme-doc-sidebar', 'navbar', 'footer'],
-  },
-  {
-    name: 'readthedocs',
-    detect: (html, url) => !!(url?.includes('readthedocs') || html.includes('rst-content') || html.includes('wy-')),
-    contentSelectors: ['rst-content', 'document', 'body-content'],
-    removeSelectors: ['wy-nav-side', 'wy-nav-top', 'rst-footer', 'footer'],
-  },
-  {
-    name: 'mkdocs',
-    detect: (html) => html.includes('mkdocs') || html.includes('md-content'),
-    contentSelectors: ['md-content', 'md-main', 'content'],
-    removeSelectors: ['md-sidebar', 'md-header', 'md-footer', 'md-tabs'],
-  },
-  {
-    name: 'notion',
-    detect: (html, url) => !!(url?.includes('notion.so') || url?.includes('notion.site') || html.includes('notion-')),
-    contentSelectors: ['notion-page-content', 'layout-content', 'notion-frame'],
-    removeSelectors: ['notion-topbar', 'notion-sidebar', 'notion-overlay-container'],
-  },
-  {
-    name: 'confluence',
-    detect: (html, url) => !!(url?.includes('atlassian.net/wiki') || html.includes('confluence') || html.includes('wiki-content')),
-    contentSelectors: ['wiki-content', 'confluence-information-macro', 'page-content-body'],
-    removeSelectors: ['page-metadata', 'likes-section', 'content-navigation', 'page-restrictions'],
-  },
-  {
-    name: 'medium',
-    detect: (html, url) => !!(url?.includes('medium.com') || html.includes('Medium') || html.includes('graf--')),
-    contentSelectors: ['postArticle-content', 'section-content', 'story-body'],
-    removeSelectors: ['metabar', 'postActions', 'js-postShareWidget', 'response'],
-  },
-];
-
-export class WebContentExtractor implements ContentExtractor {
-  extract(raw: string, sourceUrl?: string): ExtractResult {
-    // Detect platform
-    const platform = PLATFORM_RULES.find(r => r.detect(raw, sourceUrl));
-
-    // Extract title
-    const title = this.extractTitle(raw, platform);
-
-    // Extract main content
-    let content = platform
-      ? this.extractWithPlatformRules(raw, platform)
-      : this.extractGeneric(raw);
-
-    // Convert HTML to markdown-like text
-    content = this.htmlToCleanText(content);
-
-    // Extract author
-    const author = this.extractAuthor(raw);
-
-    return { title, content, contentType: 'text', author };
-  }
-
-  private extractTitle(html: string, platform?: PlatformRule): string {
-    // Try platform-specific title
-    if (platform?.titleSelector) {
-      const match = this.extractByClassOrId(html, platform.titleSelector);
-      if (match) return this.stripTags(match).trim();
-    }
-
-    // Try og:title
-    const ogTitle = html.match(/<meta\s+property="og:title"\s+content="([^"]*?)"/i);
-    if (ogTitle) return this.decodeEntities(ogTitle[1]);
-
-    // Try <h1>
-    const h1 = html.match(/<h1[^>]*>([\s\S]*?)<\/h1>/i);
-    if (h1) return this.stripTags(h1[1]).trim();
-
-    // Try <title>
-    const title = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
-    if (title) {
-      return this.decodeEntities(title[1])
-        .replace(/\s*[|–—-]\s*.+$/, '')  // remove site name suffix
-        .trim();
-    }
-
-    return 'Untitled';
-  }
-
-  private extractWithPlatformRules(html: string, platform: PlatformRule): string {
-    // Remove unwanted elements first
-    let cleaned = html;
-    for (const sel of platform.removeSelectors) {
-      // Remove by class or id containing the selector
-      const regex = new RegExp(`<[^>]*(?:class|id)="[^"]*${this.escRegex(sel)}[^"]*"[^>]*>[\\s\\S]*?<\\/[^>]+>`, 'gi');
-      cleaned = cleaned.replace(regex, '');
-    }
-
-    // Try each content selector
-    for (const sel of platform.contentSelectors) {
-      const extracted = this.extractByClassOrId(cleaned, sel);
-      if (extracted && extracted.length > 100) return extracted;
-    }
-
-    // Fallback to generic
-    return this.extractGeneric(cleaned);
-  }
-
-  private extractGeneric(html: string): string {
-    let content = html;
-
-    // Remove definitely-not-content elements
-    const removePatterns = [
-      /<(script|style|noscript|iframe|svg|object|embed|applet|video|audio|canvas|map|form)[^>]*>[\s\S]*?<\/\1>/gi,
-      /<(nav|header|footer|aside)[^>]*>[\s\S]*?<\/\1>/gi,
-      /<[^>]*(?:class|id)="[^"]*(?:nav|menu|sidebar|footer|header|cookie|banner|modal|popup|overlay|breadcrumb|pagination|social|share|comment|ad-|ads-|advert|promo|signup|newsletter|subscribe)[^"]*"[^>]*>[\s\S]*?<\/[^>]+>/gi,
-      /<[^>]*role="(?:navigation|banner|complementary|contentinfo|search)"[^>]*>[\s\S]*?<\/[^>]+>/gi,
-      /<[^>]*aria-hidden="true"[^>]*>[\s\S]*?<\/[^>]+>/gi,
-    ];
-
-    for (const pattern of removePatterns) {
-      content = content.replace(pattern, '');
-    }
-
-    // Try to find <main>, <article>, or role="main"
-    const mainMatch = content.match(/<(?:main|article)[^>]*>([\s\S]*?)<\/(?:main|article)>/i)
-      || content.match(/<[^>]*role="main"[^>]*>([\s\S]*?)<\/[^>]+>/i);
-
-    if (mainMatch) return mainMatch[1];
-
-    // Try common content class names
-    for (const cls of ['content', 'post-content', 'entry-content', 'article-content', 'page-content', 'main-content', 'doc-content', 'body-content']) {
-      const extracted = this.extractByClassOrId(content, cls);
-      if (extracted && extracted.length > 200) return extracted;
-    }
-
-    // Last resort: extract body
-    const bodyMatch = content.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
-    return bodyMatch ? bodyMatch[1] : content;
-  }
-
-  private htmlToCleanText(html: string): string {
-    return html
-      // Headings → markdown
-      .replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/gi, (_, l, t) => '\n\n' + '#'.repeat(parseInt(l)) + ' ' + this.stripTags(t).trim() + '\n\n')
-      // Paragraphs
-      .replace(/<\/p>/gi, '\n\n').replace(/<p[^>]*>/gi, '')
-      // Line breaks
-      .replace(/<br\s*\/?>/gi, '\n')
-      // Lists
-      .replace(/<\/li>/gi, '\n').replace(/<li[^>]*>/gi, '- ')
-      .replace(/<\/?[ou]l[^>]*>/gi, '\n')
-      // Blockquotes
-      .replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, t) => '\n> ' + this.stripTags(t).trim().replace(/\n/g, '\n> ') + '\n')
-      // Code
-      .replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, '\n```\n$1\n```\n')
-      .replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, '`$1`')
-      // Bold/italic
-      .replace(/<(?:strong|b)[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, '**$1**')
-      .replace(/<(?:em|i)[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, '*$1*')
-      // Tables → simple text
-      .replace(/<tr[^>]*>/gi, '\n').replace(/<td[^>]*>/gi, ' | ').replace(/<th[^>]*>/gi, ' | ')
-      .replace(/<\/?table[^>]*>/gi, '\n').replace(/<\/?t(?:head|body|foot|r|d|h)[^>]*>/gi, '')
-      // Images → alt text
-      .replace(/<img[^>]*alt="([^"]*)"[^>]*>/gi, '[$1]')
-      .replace(/<img[^>]*>/gi, '')
-      // Anchors → text only (drop href)
-      .replace(/<a[^>]*>([\s\S]*?)<\/a>/gi, '$1')
-      // Strip all remaining tags
-      .replace(/<[^>]+>/g, '')
-      // Decode entities
-      .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"')
-      .replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ').replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n)))
-      .replace(/&[a-z]+;/gi, ' ')
-      .trim();
-  }
-
-  private extractByClassOrId(html: string, selector: string): string | null {
-    // Match elements where class or id contains the selector
-    const regex = new RegExp(`<([a-z][a-z0-9]*)\\s[^>]*(?:class|id)="[^"]*${this.escRegex(selector)}[^"]*"[^>]*>`, 'i');
-    const match = regex.exec(html);
-    if (!match) return null;
-
-    const tag = match[1];
-    const startIdx = match.index + match[0].length;
-    // Find the matching closing tag (handle nesting)
-    let depth = 1;
-    let idx = startIdx;
-    const openTag = new RegExp(`<${tag}[\\s>]`, 'gi');
-    const closeTag = new RegExp(`</${tag}>`, 'gi');
-
-    while (depth > 0 && idx < html.length) {
-      openTag.lastIndex = idx;
-      closeTag.lastIndex = idx;
-      const nextOpen = openTag.exec(html);
-      const nextClose = closeTag.exec(html);
-
-      if (!nextClose) break;
-
-      if (nextOpen && nextOpen.index < nextClose.index) {
-        depth++;
-        idx = nextOpen.index + nextOpen[0].length;
-      } else {
-        depth--;
-        if (depth === 0) return html.slice(startIdx, nextClose.index);
-        idx = nextClose.index + nextClose[0].length;
-      }
-    }
-    return null;
-  }
-
-  private extractAuthor(html: string): string | undefined {
-    const meta = html.match(/<meta\s+(?:name="author"|property="article:author")\s+content="([^"]*?)"/i);
-    if (meta) return meta[1];
-    const byline = html.match(/(?:by|author|written\s+by)\s*[:\s]*([A-Z][a-z]+\s+[A-Z][a-z]+)/i);
-    return byline ? byline[1] : undefined;
-  }
-
-  private stripTags(html: string): string {
-    return html.replace(/<[^>]+>/g, '');
-  }
-
-  private decodeEntities(text: string): string {
-    return text.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;/g, "'");
-  }
-
-  private escRegex(str: string): string {
-    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-  }
-}