arcfetch 0.0.1

package/src/core/cache.ts

@@ -0,0 +1,260 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { writeFile } from 'node:fs/promises';
+import { join, basename } from 'node:path';
+import type { FetchiConfig } from '../config/schema.js';
+
+export interface CachedReference {
+  refId: string;
+  title: string;
+  url: string;
+  filepath: string;
+  fetchedDate: string;
+  size: number;
+  query?: string;
+}
+
+export interface SaveResult {
+  refId: string;
+  filepath: string;
+  error?: string;
+}
+
+export interface ListResult {
+  references: CachedReference[];
+  error?: string;
+}
+
+export interface PromoteResult {
+  success: boolean;
+  fromPath: string;
+  toPath: string;
+  error?: string;
+}
+
+export interface DeleteResult {
+  success: boolean;
+  filepath: string;
+  error?: string;
+}
+
+/**
+ * Get the next available reference ID
+ */
+export function getNextRefId(dir: string): string {
+  if (!existsSync(dir)) {
+    return 'REF-001';
+  }
+
+  try {
+    const files = readdirSync(dir);
+    let maxId = 0;
+
+    for (const file of files) {
+      const match = file.match(/REF-(\d+)/);
+      if (match) {
+        const id = parseInt(match[1], 10);
+        if (id > maxId) maxId = id;
+      }
+    }
+
+    return `REF-${String(maxId + 1).padStart(3, '0')}`;
+  } catch {
+    return 'REF-001';
+  }
+}
+
+/**
+ * Generate a slug from title
+ */
+function slugify(title: string): string {
+  return title
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 60);
+}
+
+/**
+ * Save content to temp directory
+ */
+export async function saveToTemp(
+  config: FetchiConfig,
+  title: string,
+  url: string,
+  content: string,
+  query?: string
+): Promise<SaveResult> {
+  try {
+    const tempDir = config.paths.tempDir;
+
+    mkdirSync(tempDir, { recursive: true });
+
+    const refId = getNextRefId(tempDir);
+    const slug = slugify(title);
+    const filename = `${refId}-${slug}.md`;
+    const filepath = join(tempDir, filename);
+
+    const today = new Date().toISOString().split('T')[0];
+    let fileContent = `---\n`;
+    fileContent += `id: ${refId}\n`;
+    fileContent += `title: "${title.replace(/"/g, '\\"')}"\n`;
+    fileContent += `source_url: ${url}\n`;
+    fileContent += `fetched_date: ${today}\n`;
+    fileContent += `type: web\n`;
+    fileContent += `status: temporary\n`;
+    if (query) {
+      fileContent += `query: "${query.replace(/"/g, '\\"')}"\n`;
+    }
+    fileContent += `---\n\n`;
+    fileContent += content;
+
+    await writeFile(filepath, fileContent, 'utf-8');
+
+    return { refId, filepath };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { refId: '', filepath: '', error: message };
+  }
+}
+
+/**
+ * List all cached references
+ */
+export function listCached(config: FetchiConfig): ListResult {
+  try {
+    const tempDir = config.paths.tempDir;
+
+    if (!existsSync(tempDir)) {
+      return { references: [] };
+    }
+
+    const files = readdirSync(tempDir).filter(f => f.endsWith('.md') && f.startsWith('REF-'));
+    const references: CachedReference[] = [];
+
+    for (const file of files) {
+      const filepath = join(tempDir, file);
+      const content = readFileSync(filepath, 'utf-8');
+
+      const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/);
+      if (!frontmatterMatch) continue;
+
+      const frontmatter = frontmatterMatch[1];
+
+      const getId = (key: string): string => {
+        const match = frontmatter.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
+        return match ? match[1].trim().replace(/^["']|["']$/g, '').trim() : '';
+      };
+
+      const ref = {
+        refId: getId('id'),
+        title: getId('title'),
+        url: getId('source_url'),
+        filepath,
+        fetchedDate: getId('fetched_date'),
+        size: content.length,
+        query: getId('query') || undefined,
+      };
+      references.push(ref);
+    }
+
+    references.sort((a, b) => b.refId.localeCompare(a.refId));
+
+    return { references };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { references: [], error: message };
+  }
+}
+
+/**
+ * Find a cached reference by ID
+ */
+export function findCached(config: FetchiConfig, refId: string): CachedReference | null {
+  const { references } = listCached(config);
+  return references.find(r => r.refId === refId) || null;
+}
+
+/**
+ * Promote a reference from temp to docs folder
+ */
+export function promoteReference(config: FetchiConfig, refId: string): PromoteResult {
+  try {
+    const cached = findCached(config, refId);
+
+    if (!cached) {
+      return {
+        success: false,
+        fromPath: '',
+        toPath: '',
+        error: `Reference ${refId} not found in ${config.paths.tempDir}`,
+      };
+    }
+
+    const docsDir = config.paths.docsDir;
+
+    mkdirSync(docsDir, { recursive: true });
+
+    let content = readFileSync(cached.filepath, 'utf-8');
+
+    content = content.replace(/^status:\s*temporary$/m, 'status: permanent');
+
+    const filename = basename(cached.filepath);
+    const toPath = join(docsDir, filename);
+
+    writeFileSync(toPath, content, 'utf-8');
+
+    unlinkSync(cached.filepath);
+
+    return {
+      success: true,
+      fromPath: cached.filepath,
+      toPath,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      success: false,
+      fromPath: '',
+      toPath: '',
+      error: message,
+    };
+  }
+}
+
+/**
+ * Delete a cached reference
+ */
+export function deleteCached(config: FetchiConfig, refId: string): DeleteResult {
+  try {
+    const cached = findCached(config, refId);
+
+    if (!cached) {
+      return {
+        success: false,
+        filepath: '',
+        error: `Reference ${refId} not found in ${config.paths.tempDir}`,
+      };
+    }
+
+    unlinkSync(cached.filepath);
+
+    return {
+      success: true,
+      filepath: cached.filepath,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      success: false,
+      filepath: '',
+      error: message,
+    };
+  }
+}
+
+/**
+ * Get cache root (for backwards compatibility)
+ */
+export function findCacheRoot(): string {
+  return process.cwd();
+}

package/src/core/extractor.ts

@@ -0,0 +1,87 @@
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import { cleanMarkdownComplete } from '../utils/markdown-cleaner.js';
+
+export interface ExtractionResult {
+  markdown?: string;
+  title?: string;
+  byline?: string;
+  excerpt?: string;
+  siteName?: string;
+  error?: string;
+}
+
+const turndown = new TurndownService({
+  headingStyle: 'atx',
+  codeBlockStyle: 'fenced',
+  bulletListMarker: '-',
+  emDelimiter: '*',
+  strongDelimiter: '**',
+  hr: '---',
+});
+
+turndown.use(gfm);
+
+turndown.addRule('removeComments', {
+  filter: (node) => (node as unknown as { nodeType: number }).nodeType === 8,
+  replacement: () => '',
+});
+
+export async function processHtmlToMarkdown(
+  html: string,
+  url: string,
+  verbose = false
+): Promise<ExtractionResult> {
+  try {
+    if (verbose) {
+      console.error(`📝 Processing HTML (${html.length} chars)`);
+    }
+
+    const { document } = parseHTML(html, { url });
+
+    const reader = new Readability(document, {
+      debug: false,
+      maxElemsToParse: 0,
+      nbTopCandidates: 5,
+      charThreshold: 500,
+      keepClasses: false,
+    });
+
+    const article = reader.parse();
+
+    if (!article) {
+      return {
+        error: 'Could not extract article content. Page may not contain article-like content.',
+      };
+    }
+
+    const content = article.content ?? '';
+
+    if (verbose) {
+      console.error(`📝 Extracted: "${article.title}" (${content.length} chars)`);
+    }
+
+    let markdown = turndown.turndown(content);
+
+    markdown = cleanMarkdownComplete(markdown);
+
+    let header = `# ${article.title}\n\n`;
+    if (article.byline) header += `**By:** ${article.byline}\n\n`;
+    if (article.siteName) header += `**Source:** ${article.siteName}\n\n`;
+    if (article.excerpt) header += `**Summary:** ${article.excerpt}\n\n`;
+    header += `**URL:** ${url}\n\n---\n\n`;
+
+    return {
+      markdown: header + markdown,
+      title: article.title ?? undefined,
+      byline: article.byline ?? undefined,
+      excerpt: article.excerpt ?? undefined,
+      siteName: article.siteName ?? undefined,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { error: message };
+  }
+}

package/src/core/index.ts

@@ -0,0 +1,4 @@
+export * from './pipeline.js';
+export * from './extractor.js';
+export * from './cache.js';
+export * from './playwright/index.js';

package/src/core/pipeline.ts

@@ -0,0 +1,189 @@
+import type { FetchiConfig } from '../config/schema.js';
+import { validateMarkdown, type ValidationResult } from '../utils/markdown-validator.js';
+import { fetchWithBrowser, closeBrowser } from './playwright/manager.js';
+import { processHtmlToMarkdown } from './extractor.js';
+
+export interface FetchResult {
+  success: boolean;
+  markdown?: string;
+  title?: string;
+  byline?: string;
+  excerpt?: string;
+  siteName?: string;
+  quality?: ValidationResult;
+  error?: string;
+  suggestion?: string;
+  usedPlaywright?: boolean;
+  playwrightReason?: string;
+}
+
+interface SimpleFetchResult {
+  html: string;
+  error?: string;
+}
+
+async function simpleFetch(url: string, verbose = false): Promise<SimpleFetchResult> {
+  try {
+    if (verbose) {
+      console.error(`📡 Simple fetch: ${url}`);
+    }
+
+    const response = await fetch(url, {
+      redirect: 'follow',
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+      },
+    });
+
+    if (!response.ok) {
+      return { html: '', error: `HTTP ${response.status}: ${response.statusText}` };
+    }
+
+    const html = await response.text();
+
+    if (verbose) {
+      console.error(`📡 Simple fetch: Got ${html.length} chars`);
+    }
+
+    return { html };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { html: '', error: message };
+  }
+}
+
+async function tryPlaywright(
+  url: string,
+  config: FetchiConfig,
+  reason: string,
+  verbose = false
+): Promise<FetchResult> {
+  if (verbose) {
+    console.error(`🎭 Trying Playwright (reason: ${reason})`);
+  }
+
+  const browserResult = await fetchWithBrowser(url, config.playwright, verbose);
+
+  if (browserResult.error) {
+    return {
+      success: false,
+      error: `Playwright fetch failed: ${browserResult.error}`,
+    };
+  }
+
+  const extracted = await processHtmlToMarkdown(browserResult.html, url, verbose);
+
+  if (extracted.error) {
+    return {
+      success: false,
+      error: extracted.error,
+    };
+  }
+
+  const quality = validateMarkdown(extracted.markdown!);
+
+  if (quality.score < config.quality.minScore) {
+    return {
+      success: false,
+      error: `Content quality too low (${quality.score}/100) even with JavaScript rendering`,
+      quality,
+      suggestion: 'This page may be a login wall, forum, or complex web app not suitable for article extraction',
+      usedPlaywright: true,
+      playwrightReason: reason,
+    };
+  }
+
+  return {
+    success: true,
+    markdown: extracted.markdown,
+    title: extracted.title,
+    byline: extracted.byline,
+    excerpt: extracted.excerpt,
+    siteName: extracted.siteName,
+    quality,
+    usedPlaywright: true,
+    playwrightReason: reason,
+  };
+}
+
+export async function fetchUrl(
+  url: string,
+  config: FetchiConfig,
+  verbose = false,
+  forcePlaywright = false
+): Promise<FetchResult> {
+  if (forcePlaywright) {
+    if (verbose) {
+      console.error(`⚡ Force Playwright mode enabled`);
+    }
+    return tryPlaywright(url, config, 'forced', verbose);
+  }
+
+  const simpleResult = await simpleFetch(url, verbose);
+
+  if (simpleResult.error) {
+    if (verbose) {
+      console.error(`📡 Simple fetch failed: ${simpleResult.error}`);
+    }
+    return tryPlaywright(url, config, 'network_error', verbose);
+  }
+
+  const extracted = await processHtmlToMarkdown(simpleResult.html, url, verbose);
+
+  if (extracted.error) {
+    if (verbose) {
+      console.error(`📝 Extraction failed: ${extracted.error}`);
+    }
+    return tryPlaywright(url, config, 'extraction_failed', verbose);
+  }
+
+  const quality = validateMarkdown(extracted.markdown!);
+
+  if (verbose) {
+    console.error(`📊 Quality score: ${quality.score}/100`);
+  }
+
+  if (quality.score >= config.quality.jsRetryThreshold) {
+    return {
+      success: true,
+      markdown: extracted.markdown,
+      title: extracted.title,
+      byline: extracted.byline,
+      excerpt: extracted.excerpt,
+      siteName: extracted.siteName,
+      quality,
+    };
+  }
+
+  if (quality.score >= config.quality.minScore) {
+    if (verbose) {
+      console.error(`📊 Quality marginal (${quality.score}), trying Playwright...`);
+    }
+
+    const playwrightResult = await tryPlaywright(url, config, 'quality_marginal', verbose);
+
+    if (playwrightResult.success && playwrightResult.quality!.score > quality.score) {
+      return playwrightResult;
+    }
+
+    return {
+      success: true,
+      markdown: extracted.markdown,
+      title: extracted.title,
+      byline: extracted.byline,
+      excerpt: extracted.excerpt,
+      siteName: extracted.siteName,
+      quality,
+    };
+  }
+
+  if (verbose) {
+    console.error(`📊 Quality too low (${quality.score}), trying Playwright...`);
+  }
+
+  return tryPlaywright(url, config, 'quality_too_low', verbose);
+}
+
+export { closeBrowser };

package/src/core/playwright/index.ts

@@ -0,0 +1,2 @@
+export * from './types.js';
+export * from './manager.js';

package/src/core/playwright/local.ts

@@ -0,0 +1,38 @@
+import { chromium } from 'playwright-extra';
+import type { Browser } from 'playwright';
+import stealth from 'puppeteer-extra-plugin-stealth';
+import type { PlaywrightConfig } from '../../config/schema.js';
+import type { BrowserManager } from './types.js';
+
+chromium.use(stealth());
+
+let browserInstance: Browser | null = null;
+
+export class LocalBrowserManager implements BrowserManager {
+  private config: PlaywrightConfig;
+  
+  constructor(config: PlaywrightConfig) {
+    this.config = config;
+  }
+  
+  async getBrowser(): Promise<Browser> {
+    if (!browserInstance) {
+      browserInstance = await chromium.launch({ 
+        headless: true,
+        timeout: this.config.timeout,
+      });
+    }
+    return browserInstance;
+  }
+  
+  async closeBrowser(): Promise<void> {
+    if (browserInstance) {
+      await browserInstance.close();
+      browserInstance = null;
+    }
+  }
+  
+  isDocker(): boolean {
+    return false;
+  }
+}

package/src/core/playwright/manager.ts

@@ -0,0 +1,61 @@
+import type { PlaywrightConfig } from '../../config/schema.js';
+import type { BrowserManager, FetchWithBrowserResult } from './types.js';
+import { LocalBrowserManager } from './local.js';
+
+let currentManager: BrowserManager | null = null;
+
+export async function getBrowserManager(config: PlaywrightConfig): Promise<BrowserManager> {
+  if (currentManager) {
+    return currentManager;
+  }
+
+  // Only local mode is supported
+  currentManager = new LocalBrowserManager(config);
+  return currentManager;
+}
+
+export async function fetchWithBrowser(
+  url: string,
+  config: PlaywrightConfig,
+  verbose = false
+): Promise<FetchWithBrowserResult> {
+  const manager = await getBrowserManager(config);
+  const browser = await manager.getBrowser();
+  
+  try {
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    });
+    const page = await context.newPage();
+    
+    if (verbose) {
+      console.error(`🎭 Playwright: Navigating to ${url}`);
+    }
+    
+    await page.goto(url, {
+      waitUntil: config.waitStrategy,
+      timeout: config.timeout,
+    });
+    
+    const html = await page.content();
+    
+    if (verbose) {
+      console.error(`🎭 Playwright: Got ${html.length} chars of HTML`);
+    }
+    
+    await page.close();
+    await context.close();
+
+    return { html };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { html: '', error: message };
+  }
+}
+
+export async function closeBrowser(): Promise<void> {
+  if (currentManager) {
+    await currentManager.closeBrowser();
+    currentManager = null;
+  }
+}

package/src/core/playwright/types.ts

@@ -0,0 +1,12 @@
+import type { Browser } from 'playwright';
+
+export interface BrowserManager {
+  getBrowser(): Promise<Browser>;
+  closeBrowser(): Promise<void>;
+  isDocker(): boolean;
+}
+
+export interface FetchWithBrowserResult {
+  html: string;
+  error?: string;
+}

package/src/types/turndown-plugin-gfm.d.ts

@@ -0,0 +1,8 @@
+declare module 'turndown-plugin-gfm' {
+  import TurndownService from 'turndown';
+
+  export function gfm(turndownService: TurndownService): void;
+  export function strikethrough(turndownService: TurndownService): void;
+  export function tables(turndownService: TurndownService): void;
+  export function taskListItems(turndownService: TurndownService): void;
+}