arcfetch 3.0.0

package/src/core/playwright/manager.ts

@@ -0,0 +1,89 @@
+import type { PlaywrightConfig } from '../../config/schema.js';
+import type { BrowserManager, FetchWithBrowserResult } from './types.js';
+import { LocalBrowserManager } from './local.js';
+import { DockerBrowserManager, isDockerAvailable } from './docker.js';
+
+let currentManager: BrowserManager | null = null;
+
+export async function getBrowserManager(config: PlaywrightConfig): Promise<BrowserManager> {
+  if (currentManager) {
+    return currentManager;
+  }
+  
+  const mode = config.mode;
+  
+  if (mode === 'local') {
+    currentManager = new LocalBrowserManager(config);
+    return currentManager;
+  }
+  
+  if (mode === 'docker') {
+    const dockerAvailable = await isDockerAvailable();
+    if (!dockerAvailable) {
+      throw new Error('Docker mode requested but Docker is not available');
+    }
+    currentManager = new DockerBrowserManager(config);
+    return currentManager;
+  }
+  
+  // Auto mode: prefer Docker if available, fall back to local
+  if (mode === 'auto') {
+    const dockerAvailable = await isDockerAvailable();
+    if (dockerAvailable) {
+      console.error('Using Docker for Playwright');
+      currentManager = new DockerBrowserManager(config);
+    } else {
+      console.error('Docker not available, using local Playwright');
+      currentManager = new LocalBrowserManager(config);
+    }
+    return currentManager;
+  }
+  
+  throw new Error(`Unknown Playwright mode: ${mode}`);
+}
+
+export async function fetchWithBrowser(
+  url: string,
+  config: PlaywrightConfig,
+  verbose = false
+): Promise<FetchWithBrowserResult> {
+  const manager = await getBrowserManager(config);
+  const browser = await manager.getBrowser();
+  
+  try {
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    });
+    const page = await context.newPage();
+    
+    if (verbose) {
+      console.error(`🎭 Playwright: Navigating to ${url} (${manager.isDocker() ? 'Docker' : 'local'})`);
+    }
+    
+    await page.goto(url, {
+      waitUntil: config.waitStrategy,
+      timeout: config.timeout,
+    });
+    
+    const html = await page.content();
+    
+    if (verbose) {
+      console.error(`🎭 Playwright: Got ${html.length} chars of HTML`);
+    }
+    
+    await page.close();
+    await context.close();
+
+    return { html };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { html: '', error: message };
+  }
+}
+
+export async function closeBrowser(): Promise<void> {
+  if (currentManager) {
+    await currentManager.closeBrowser();
+    currentManager = null;
+  }
+}

package/src/core/playwright/types.ts

@@ -0,0 +1,12 @@
+import type { Browser } from 'playwright';
+
+export interface BrowserManager {
+  getBrowser(): Promise<Browser>;
+  closeBrowser(): Promise<void>;
+  isDocker(): boolean;
+}
+
+export interface FetchWithBrowserResult {
+  html: string;
+  error?: string;
+}

package/src/types/turndown-plugin-gfm.d.ts

@@ -0,0 +1,8 @@
+declare module 'turndown-plugin-gfm' {
+  import TurndownService from 'turndown';
+
+  export function gfm(turndownService: TurndownService): void;
+  export function strikethrough(turndownService: TurndownService): void;
+  export function tables(turndownService: TurndownService): void;
+  export function taskListItems(turndownService: TurndownService): void;
+}

package/src/utils/markdown-cleaner.ts

@@ -0,0 +1,79 @@
+/**
+ * Markdown Cleaning Utilities
+ *
+ * Post-processing functions to clean and optimize markdown for LLM context efficiency
+ */
+
+export function cleanMarkdown(markdown: string): string {
+  if (!markdown.trim()) {
+    return markdown;
+  }
+
+  let cleaned = markdown
+
+  cleaned = cleaned.replace(/\r\n/g, '\n')
+  cleaned = cleaned.replace(/\n{3,}/g, '\n\n')
+  cleaned = cleaned.replace(/[ \t]+$/gm, '')
+  cleaned = cleaned.trim()
+  cleaned = cleaned.replace(/([^\n])\n(#{1,6} )/g, '$1\n\n$2')
+  cleaned = cleaned.replace(/(#{1,6} .+)\n([^#\n])/g, '$1\n\n$2')
+  cleaned = cleaned.replace(/([^\n])\n([-*+] |\d+\. )/g, '$1\n\n$2')
+  cleaned = cleaned.replace(/(\*|_) +/g, '$1')
+  cleaned = cleaned.replace(/ +(\*|_)/g, '$1')
+  cleaned = cleaned.replace(/([^\n])\n```/g, '$1\n\n```')
+  cleaned = cleaned.replace(/```\n([^`])/g, '```\n\n$1')
+  cleaned = cleaned.replace(/<!--[\s\S]*?-->/g, ' ')
+  cleaned = cleaned.replace(/ {2,}/g, ' ')
+
+  return cleaned
+}
+
+export function advancedClean(markdown: string): string {
+  let cleaned = markdown
+
+  cleaned = cleaned.replace(/\[([^\]]+)\]\(\)/g, '$1')
+  cleaned = cleaned.replace(/<[^>]+>/g, '')
+  cleaned = cleaned.replace(/\*\*\*\*/g, '')
+  cleaned = cleaned.replace(/(?<!\*)\*\*(?!\*)/g, '')
+  cleaned = cleaned.replace(/__/g, '')
+  cleaned = cleaned.replace(/!\[\]\(([^)]+)\)/g, '![]($1)')
+  cleaned = cleaned.replace(/[\u200B-\u200D\uFEFF]/g, '')
+  cleaned = cleaned.replace(/[\u201C\u201D]/g, '"')
+  cleaned = cleaned.replace(/[\u2018\u2019]/g, "'")
+  cleaned = cleaned.replace(/[\u2013\u2014]/g, '-')
+
+  cleaned = cleaned.replace(/^(?!```)[^\n]*$/gm, (line) => {
+    return line.replace(/ {2,}/g, ' ')
+  })
+
+  return cleaned
+}
+
+export function finalCleanup(markdown: string): string {
+  if (!markdown.trim()) {
+    return markdown;
+  }
+
+  let cleaned = markdown
+
+  cleaned = cleaned.replace(/^(\s*)[*+] /gm, '$1- ')
+  cleaned = cleaned.replace(/_([^_]+)_/g, '*$1*')
+  cleaned = cleaned.replace(/^~~~(\w*)\n/gm, '```$1\n')
+  cleaned = cleaned.replace(/^~~~$/gm, '```')
+  cleaned = cleaned.replace(/\n{3,}/g, '\n\n')
+  cleaned = `${cleaned.trim()}\n`
+
+  return cleaned
+}
+
+export function cleanMarkdownComplete(markdown: string): string {
+  if (!markdown.trim()) {
+    return markdown;
+  }
+
+  let cleaned = cleanMarkdown(markdown)
+  cleaned = advancedClean(cleaned)
+  cleaned = finalCleanup(cleaned)
+
+  return cleaned
+}

package/src/utils/markdown-validator.ts

@@ -0,0 +1,136 @@
+/**
+ * Markdown Quality Validator
+ *
+ * Validates that extracted markdown is clean and usable.
+ * Returns quality score and issues.
+ */
+
+export interface ValidationResult {
+  isValid: boolean;
+  score: number; // 0-100
+  issues: string[];
+  warnings: string[];
+}
+
+export function validateMarkdown(markdown: string): ValidationResult {
+  const issues: string[] = [];
+  const warnings: string[] = [];
+  let score = 100;
+
+  // Check for excessive HTML tags (indicates poor conversion)
+  const htmlTagMatches = markdown.match(/<[^>]+>/g);
+  const htmlTagCount = htmlTagMatches ? htmlTagMatches.length : 0;
+
+  if (htmlTagCount > 100) {
+    score -= 40;
+    issues.push(`${htmlTagCount} leftover HTML tags found (likely forum/discussion thread)`);
+  } else if (htmlTagCount > 50) {
+    score -= 20;
+    warnings.push(`${htmlTagCount} HTML tags present`);
+  } else if (htmlTagCount > 10) {
+    score -= 5;
+    warnings.push(`${htmlTagCount} minor HTML tags present`);
+  }
+
+  // Check for table structure (tr/td) not converted
+  const tableTagMatches = markdown.match(/<t[rd][\s>]/gi);
+  const tableTagCount = tableTagMatches ? tableTagMatches.length : 0;
+
+  if (tableTagCount > 50) {
+    score -= 30;
+    issues.push(`${tableTagCount} unconverted table tags (complex layout not suitable)`);
+  }
+
+  // Check markdown to HTML ratio (too much HTML means poor extraction)
+  const htmlCharCount = markdown.match(/<[^>]*>/g)?.join('').length || 0;
+  const htmlRatio = htmlCharCount / markdown.length;
+
+  if (htmlRatio > 0.3) {
+    score -= 25;
+    issues.push(`${(htmlRatio * 100).toFixed(1)}% of content is HTML tags`);
+  } else if (htmlRatio > 0.15) {
+    score -= 10;
+    warnings.push(`${(htmlRatio * 100).toFixed(1)}% HTML tag ratio`);
+  }
+
+  // Check for script tags (should never be present)
+  const scriptMatches = markdown.match(/<script/gi);
+  if (scriptMatches && scriptMatches.length > 0) {
+    score -= 15;
+    warnings.push(`${scriptMatches.length} script tags present`);
+  }
+
+  // Check for style tags
+  const styleMatches = markdown.match(/<style/gi);
+  if (styleMatches && styleMatches.length > 0) {
+    score -= 10;
+    warnings.push(`${styleMatches.length} style tags present`);
+  }
+
+  // Check for minimal/blank content
+  const contentLength = markdown.replace(/<[^>]*>/g, '').replace(/[#*\-_`[\]()]/g, '').trim().length;
+
+  if (contentLength === 0) {
+    score = 0;
+    issues.push("Blank content - no text extracted");
+  } else if (contentLength < 50) {
+    score -= 50;
+    issues.push(`Extremely short content (${contentLength} chars) - likely extraction failure`);
+  } else if (contentLength < 200 && (htmlTagCount > 50 || tableTagCount > 20)) {
+    score -= 30;
+    issues.push(`Only ${contentLength} chars of actual content with excessive HTML (extraction likely failed)`);
+  } else if (contentLength < 300) {
+    score -= 15;
+    warnings.push(`Short content (${contentLength} chars) - may not be a full article`);
+  }
+
+  // Check for excessive newlines (indicates poor formatting)
+  const excessiveNewlines = markdown.match(/\n{5,}/g);
+  if (excessiveNewlines && excessiveNewlines.length > 10) {
+    score -= 5;
+    warnings.push(`${excessiveNewlines.length} sections with excessive newlines`);
+  }
+
+  // Quality thresholds
+  const isValid = score >= 60; // Below 60 is unusable
+
+  return {
+    isValid,
+    score: Math.max(0, score),
+    issues,
+    warnings,
+  };
+}
+
+/**
+ * Generate human-readable quality report
+ */
+export function formatValidationReport(result: ValidationResult): string {
+  let report = `**Quality Score**: ${result.score}/100`;
+
+  if (result.score >= 90) {
+    report += " ✅ Excellent";
+  } else if (result.score >= 75) {
+    report += " ✅ Good";
+  } else if (result.score >= 60) {
+    report += " ⚠️ Acceptable";
+  } else {
+    report += " ❌ Poor";
+  }
+
+  if (result.issues.length > 0) {
+    report += "\n\n**Issues**:\n";
+    result.issues.forEach(issue => {
+      report += `- ${issue}\n`;
+    });
+  }
+
+  if (result.warnings.length > 0) {
+    report += "\n**Warnings**:\n";
+    result.warnings.forEach(warning => {
+      report += `- ${warning}\n`;
+    });
+  }
+
+  return report;
+}