arcfetch 3.0.0

package/src/core/cache.ts

@@ -0,0 +1,260 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { writeFile } from 'node:fs/promises';
+import { join, basename } from 'node:path';
+import type { FetchiConfig } from '../config/schema.js';
+
+export interface CachedReference {
+  refId: string;
+  title: string;
+  url: string;
+  filepath: string;
+  fetchedDate: string;
+  size: number;
+  query?: string;
+}
+
+export interface SaveResult {
+  refId: string;
+  filepath: string;
+  error?: string;
+}
+
+export interface ListResult {
+  references: CachedReference[];
+  error?: string;
+}
+
+export interface PromoteResult {
+  success: boolean;
+  fromPath: string;
+  toPath: string;
+  error?: string;
+}
+
+export interface DeleteResult {
+  success: boolean;
+  filepath: string;
+  error?: string;
+}
+
+/**
+ * Get the next available reference ID
+ */
+export function getNextRefId(dir: string): string {
+  if (!existsSync(dir)) {
+    return 'REF-001';
+  }
+
+  try {
+    const files = readdirSync(dir);
+    let maxId = 0;
+
+    for (const file of files) {
+      const match = file.match(/REF-(\d+)/);
+      if (match) {
+        const id = parseInt(match[1], 10);
+        if (id > maxId) maxId = id;
+      }
+    }
+
+    return `REF-${String(maxId + 1).padStart(3, '0')}`;
+  } catch {
+    return 'REF-001';
+  }
+}
+
+/**
+ * Generate a slug from title
+ */
+function slugify(title: string): string {
+  return title
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 60);
+}
+
+/**
+ * Save content to temp directory
+ */
+export async function saveToTemp(
+  config: FetchiConfig,
+  title: string,
+  url: string,
+  content: string,
+  query?: string
+): Promise<SaveResult> {
+  try {
+    const tempDir = config.paths.tempDir;
+
+    mkdirSync(tempDir, { recursive: true });
+
+    const refId = getNextRefId(tempDir);
+    const slug = slugify(title);
+    const filename = `${refId}-${slug}.md`;
+    const filepath = join(tempDir, filename);
+
+    const today = new Date().toISOString().split('T')[0];
+    let fileContent = `---\n`;
+    fileContent += `id: ${refId}\n`;
+    fileContent += `title: "${title.replace(/"/g, '\\"')}"\n`;
+    fileContent += `source_url: ${url}\n`;
+    fileContent += `fetched_date: ${today}\n`;
+    fileContent += `type: web\n`;
+    fileContent += `status: temporary\n`;
+    if (query) {
+      fileContent += `query: "${query.replace(/"/g, '\\"')}"\n`;
+    }
+    fileContent += `---\n\n`;
+    fileContent += content;
+
+    await writeFile(filepath, fileContent, 'utf-8');
+
+    return { refId, filepath };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { refId: '', filepath: '', error: message };
+  }
+}
+
+/**
+ * List all cached references
+ */
+export function listCached(config: FetchiConfig): ListResult {
+  try {
+    const tempDir = config.paths.tempDir;
+
+    if (!existsSync(tempDir)) {
+      return { references: [] };
+    }
+
+    const files = readdirSync(tempDir).filter(f => f.endsWith('.md') && f.startsWith('REF-'));
+    const references: CachedReference[] = [];
+
+    for (const file of files) {
+      const filepath = join(tempDir, file);
+      const content = readFileSync(filepath, 'utf-8');
+
+      const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/);
+      if (!frontmatterMatch) continue;
+
+      const frontmatter = frontmatterMatch[1];
+
+      const getId = (key: string): string => {
+        const match = frontmatter.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
+        return match ? match[1].trim().replace(/^["']|["']$/g, '').trim() : '';
+      };
+
+      const ref = {
+        refId: getId('id'),
+        title: getId('title'),
+        url: getId('source_url'),
+        filepath,
+        fetchedDate: getId('fetched_date'),
+        size: content.length,
+        query: getId('query') || undefined,
+      };
+      references.push(ref);
+    }
+
+    references.sort((a, b) => b.refId.localeCompare(a.refId));
+
+    return { references };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { references: [], error: message };
+  }
+}
+
+/**
+ * Find a cached reference by ID
+ */
+export function findCached(config: FetchiConfig, refId: string): CachedReference | null {
+  const { references } = listCached(config);
+  return references.find(r => r.refId === refId) || null;
+}
+
+/**
+ * Promote a reference from temp to docs folder
+ */
+export function promoteReference(config: FetchiConfig, refId: string): PromoteResult {
+  try {
+    const cached = findCached(config, refId);
+
+    if (!cached) {
+      return {
+        success: false,
+        fromPath: '',
+        toPath: '',
+        error: `Reference ${refId} not found in ${config.paths.tempDir}`,
+      };
+    }
+
+    const docsDir = config.paths.docsDir;
+
+    mkdirSync(docsDir, { recursive: true });
+
+    let content = readFileSync(cached.filepath, 'utf-8');
+
+    content = content.replace(/^status:\s*temporary$/m, 'status: permanent');
+
+    const filename = basename(cached.filepath);
+    const toPath = join(docsDir, filename);
+
+    writeFileSync(toPath, content, 'utf-8');
+
+    unlinkSync(cached.filepath);
+
+    return {
+      success: true,
+      fromPath: cached.filepath,
+      toPath,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      success: false,
+      fromPath: '',
+      toPath: '',
+      error: message,
+    };
+  }
+}
+
+/**
+ * Delete a cached reference
+ */
+export function deleteCached(config: FetchiConfig, refId: string): DeleteResult {
+  try {
+    const cached = findCached(config, refId);
+
+    if (!cached) {
+      return {
+        success: false,
+        filepath: '',
+        error: `Reference ${refId} not found in ${config.paths.tempDir}`,
+      };
+    }
+
+    unlinkSync(cached.filepath);
+
+    return {
+      success: true,
+      filepath: cached.filepath,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      success: false,
+      filepath: '',
+      error: message,
+    };
+  }
+}
+
+/**
+ * Get cache root (for backwards compatibility)
+ */
+export function findCacheRoot(): string {
+  return process.cwd();
+}

package/src/core/extractor.ts

@@ -0,0 +1,87 @@
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import { cleanMarkdownComplete } from '../utils/markdown-cleaner.js';
+
+export interface ExtractionResult {
+  markdown?: string;
+  title?: string;
+  byline?: string;
+  excerpt?: string;
+  siteName?: string;
+  error?: string;
+}
+
+const turndown = new TurndownService({
+  headingStyle: 'atx',
+  codeBlockStyle: 'fenced',
+  bulletListMarker: '-',
+  emDelimiter: '*',
+  strongDelimiter: '**',
+  hr: '---',
+});
+
+turndown.use(gfm);
+
+turndown.addRule('removeComments', {
+  filter: (node) => (node as unknown as { nodeType: number }).nodeType === 8,
+  replacement: () => '',
+});
+
+export async function processHtmlToMarkdown(
+  html: string,
+  url: string,
+  verbose = false
+): Promise<ExtractionResult> {
+  try {
+    if (verbose) {
+      console.error(`📝 Processing HTML (${html.length} chars)`);
+    }
+
+    const { document } = parseHTML(html, { url });
+
+    const reader = new Readability(document, {
+      debug: false,
+      maxElemsToParse: 0,
+      nbTopCandidates: 5,
+      charThreshold: 500,
+      keepClasses: false,
+    });
+
+    const article = reader.parse();
+
+    if (!article) {
+      return {
+        error: 'Could not extract article content. Page may not contain article-like content.',
+      };
+    }
+
+    const content = article.content ?? '';
+
+    if (verbose) {
+      console.error(`📝 Extracted: "${article.title}" (${content.length} chars)`);
+    }
+
+    let markdown = turndown.turndown(content);
+
+    markdown = cleanMarkdownComplete(markdown);
+
+    let header = `# ${article.title}\n\n`;
+    if (article.byline) header += `**By:** ${article.byline}\n\n`;
+    if (article.siteName) header += `**Source:** ${article.siteName}\n\n`;
+    if (article.excerpt) header += `**Summary:** ${article.excerpt}\n\n`;
+    header += `**URL:** ${url}\n\n---\n\n`;
+
+    return {
+      markdown: header + markdown,
+      title: article.title ?? undefined,
+      byline: article.byline ?? undefined,
+      excerpt: article.excerpt ?? undefined,
+      siteName: article.siteName ?? undefined,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { error: message };
+  }
+}

package/src/core/index.ts

@@ -0,0 +1,4 @@
+export * from './pipeline.js';
+export * from './extractor.js';
+export * from './cache.js';
+export * from './playwright/index.js';

package/src/core/pipeline.ts

@@ -0,0 +1,181 @@
+import type { FetchiConfig } from '../config/schema.js';
+import { validateMarkdown, type ValidationResult } from '../utils/markdown-validator.js';
+import { fetchWithBrowser, closeBrowser } from './playwright/manager.js';
+import { processHtmlToMarkdown } from './extractor.js';
+
+export interface FetchResult {
+  success: boolean;
+  markdown?: string;
+  title?: string;
+  byline?: string;
+  excerpt?: string;
+  siteName?: string;
+  quality?: ValidationResult;
+  error?: string;
+  suggestion?: string;
+  usedPlaywright?: boolean;
+  playwrightReason?: string;
+}
+
+interface SimpleFetchResult {
+  html: string;
+  error?: string;
+}
+
+async function simpleFetch(url: string, verbose = false): Promise<SimpleFetchResult> {
+  try {
+    if (verbose) {
+      console.error(`📡 Simple fetch: ${url}`);
+    }
+
+    const response = await fetch(url, {
+      redirect: 'follow',
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+      },
+    });
+
+    if (!response.ok) {
+      return { html: '', error: `HTTP ${response.status}: ${response.statusText}` };
+    }
+
+    const html = await response.text();
+
+    if (verbose) {
+      console.error(`📡 Simple fetch: Got ${html.length} chars`);
+    }
+
+    return { html };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { html: '', error: message };
+  }
+}
+
+async function tryPlaywright(
+  url: string,
+  config: FetchiConfig,
+  reason: string,
+  verbose = false
+): Promise<FetchResult> {
+  if (verbose) {
+    console.error(`🎭 Trying Playwright (reason: ${reason})`);
+  }
+
+  const browserResult = await fetchWithBrowser(url, config.playwright, verbose);
+
+  if (browserResult.error) {
+    return {
+      success: false,
+      error: `Playwright fetch failed: ${browserResult.error}`,
+    };
+  }
+
+  const extracted = await processHtmlToMarkdown(browserResult.html, url, verbose);
+
+  if (extracted.error) {
+    return {
+      success: false,
+      error: extracted.error,
+    };
+  }
+
+  const quality = validateMarkdown(extracted.markdown!);
+
+  if (quality.score < config.quality.minScore) {
+    return {
+      success: false,
+      error: `Content quality too low (${quality.score}/100) even with JavaScript rendering`,
+      quality,
+      suggestion: 'This page may be a login wall, forum, or complex web app not suitable for article extraction',
+      usedPlaywright: true,
+      playwrightReason: reason,
+    };
+  }
+
+  return {
+    success: true,
+    markdown: extracted.markdown,
+    title: extracted.title,
+    byline: extracted.byline,
+    excerpt: extracted.excerpt,
+    siteName: extracted.siteName,
+    quality,
+    usedPlaywright: true,
+    playwrightReason: reason,
+  };
+}
+
+export async function fetchUrl(
+  url: string,
+  config: FetchiConfig,
+  verbose = false
+): Promise<FetchResult> {
+  const simpleResult = await simpleFetch(url, verbose);
+
+  if (simpleResult.error) {
+    if (verbose) {
+      console.error(`📡 Simple fetch failed: ${simpleResult.error}`);
+    }
+    return tryPlaywright(url, config, 'network_error', verbose);
+  }
+
+  const extracted = await processHtmlToMarkdown(simpleResult.html, url, verbose);
+
+  if (extracted.error) {
+    if (verbose) {
+      console.error(`📝 Extraction failed: ${extracted.error}`);
+    }
+    return tryPlaywright(url, config, 'extraction_failed', verbose);
+  }
+
+  const quality = validateMarkdown(extracted.markdown!);
+
+  if (verbose) {
+    console.error(`📊 Quality score: ${quality.score}/100`);
+  }
+
+  if (quality.score >= config.quality.jsRetryThreshold) {
+    return {
+      success: true,
+      markdown: extracted.markdown,
+      title: extracted.title,
+      byline: extracted.byline,
+      excerpt: extracted.excerpt,
+      siteName: extracted.siteName,
+      quality,
+    };
+  }
+
+  if (quality.score >= config.quality.minScore) {
+    if (verbose) {
+      console.error(`📊 Quality marginal (${quality.score}), trying Playwright...`);
+    }
+
+    const playwrightResult = await tryPlaywright(url, config, 'quality_marginal', verbose);
+
+    if (playwrightResult.success && playwrightResult.quality!.score > quality.score) {
+      return playwrightResult;
+    }
+
+    return {
+      success: true,
+      markdown: extracted.markdown,
+      title: extracted.title,
+      byline: extracted.byline,
+      excerpt: extracted.excerpt,
+      siteName: extracted.siteName,
+      quality,
+    };
+  }
+
+  if (verbose) {
+    console.error(`📊 Quality too low (${quality.score}), trying Playwright...`);
+  }
+
+  return tryPlaywright(url, config, 'quality_too_low', verbose);
+}
+
+export { closeBrowser };

package/src/core/playwright/docker.ts

@@ -0,0 +1,138 @@
+import { chromium, type Browser } from 'playwright';
+import { exec } from 'node:child_process';
+import { promisify } from 'node:util';
+import type { PlaywrightConfig } from '../../config/schema.js';
+import type { BrowserManager } from './types.js';
+
+const execAsync = promisify(exec);
+
+let containerId: string | null = null;
+let browserInstance: Browser | null = null;
+
+export async function isDockerAvailable(): Promise<boolean> {
+  try {
+    await execAsync('docker info', { timeout: 5000 });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function pullDockerImage(image: string): Promise<void> {
+  try {
+    // Check if image exists locally first
+    await execAsync(`docker image inspect ${image}`, { timeout: 10000 });
+  } catch {
+    // Image doesn't exist, pull it
+    console.error(`Pulling Docker image: ${image}...`);
+    await execAsync(`docker pull ${image}`, { timeout: 300000 }); // 5 min timeout for pull
+  }
+}
+
+export class DockerBrowserManager implements BrowserManager {
+  private config: PlaywrightConfig;
+  private wsEndpoint: string | null = null;
+  
+  constructor(config: PlaywrightConfig) {
+    this.config = config;
+  }
+  
+  async getBrowser(): Promise<Browser> {
+    if (browserInstance) {
+      return browserInstance;
+    }
+    
+    // Pull image if needed
+    await pullDockerImage(this.config.dockerImage);
+    
+    // Find available port
+    const port = await this.findAvailablePort();
+    
+    // Start container with Playwright server
+    const { stdout } = await execAsync(`
+      docker run -d --rm \
+        -p ${port}:3000 \
+        --name arcfetch-playwright-${port} \
+        ${this.config.dockerImage} \
+        npx -y playwright run-server --port 3000
+    `.trim());
+    
+    containerId = stdout.trim();
+    this.wsEndpoint = `ws://localhost:${port}`;
+    
+    // Wait for server to be ready
+    await this.waitForServer(port);
+    
+    // Connect to browser
+    browserInstance = await chromium.connect(this.wsEndpoint);
+    
+    return browserInstance;
+  }
+  
+  async closeBrowser(): Promise<void> {
+    if (browserInstance) {
+      await browserInstance.close();
+      browserInstance = null;
+    }
+    
+    if (containerId) {
+      try {
+        await execAsync(`docker stop ${containerId}`, { timeout: 10000 });
+      } catch {
+        // Container may have already stopped
+      }
+      containerId = null;
+    }
+  }
+  
+  isDocker(): boolean {
+    return true;
+  }
+  
+  private async findAvailablePort(): Promise<number> {
+    // Start from 3001 and find an available port
+    for (let port = 3001; port < 3100; port++) {
+      try {
+        const server = Bun.serve({
+          port,
+          fetch() { return new Response(''); }
+        });
+        server.stop();
+        return port;
+      } catch {
+      }
+    }
+    throw new Error('No available ports found');
+  }
+  
+  private async waitForServer(port: number, maxAttempts = 30): Promise<void> {
+    for (let i = 0; i < maxAttempts; i++) {
+      try {
+        const response = await fetch(`http://localhost:${port}/json`);
+        if (response.ok) return;
+      } catch {
+        // Server not ready yet
+      }
+      await new Promise(resolve => setTimeout(resolve, 1000));
+    }
+    throw new Error('Playwright server failed to start');
+  }
+}
+
+// Cleanup on process exit
+process.on('exit', async () => {
+  if (containerId) {
+    try {
+      await execAsync(`docker stop ${containerId}`);
+    } catch {}
+  }
+});
+
+process.on('SIGINT', async () => {
+  if (containerId) {
+    try {
+      await execAsync(`docker stop ${containerId}`);
+    } catch {}
+  }
+  process.exit();
+});

package/src/core/playwright/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js';
+export * from './manager.js';
+export { isDockerAvailable } from './docker.js';

package/src/core/playwright/local.ts

@@ -0,0 +1,38 @@
+import { chromium } from 'playwright-extra';
+import type { Browser } from 'playwright';
+import stealth from 'puppeteer-extra-plugin-stealth';
+import type { PlaywrightConfig } from '../../config/schema.js';
+import type { BrowserManager } from './types.js';
+
+chromium.use(stealth());
+
+let browserInstance: Browser | null = null;
+
+export class LocalBrowserManager implements BrowserManager {
+  private config: PlaywrightConfig;
+  
+  constructor(config: PlaywrightConfig) {
+    this.config = config;
+  }
+  
+  async getBrowser(): Promise<Browser> {
+    if (!browserInstance) {
+      browserInstance = await chromium.launch({ 
+        headless: true,
+        timeout: this.config.timeout,
+      });
+    }
+    return browserInstance;
+  }
+  
+  async closeBrowser(): Promise<void> {
+    if (browserInstance) {
+      await browserInstance.close();
+      browserInstance = null;
+    }
+  }
+  
+  isDocker(): boolean {
+    return false;
+  }
+}