@wenathlan/saddle 1.8.2 → 1.8.5

package/scrape/crawler.ts

@@ -0,0 +1,115 @@
+import type { ScrapeOptions, ScrapeResult, CrawlStats } from './types.js';
+import { scrapeUrl } from './scrape.js';
+
+export interface CrawlOptions extends ScrapeOptions {
+  maxDepth?: number;
+  maxPages?: number;
+  maxConcurrent?: number;
+  sameDomain?: boolean;
+  delayMs?: number;
+  onDiscover?: (url: string, depth: number) => void;
+  onResult?: (result: ScrapeResult) => void;
+  onError?: (url: string, error: Error) => void;
+}
+
+interface CrawlEntry {
+  url: string;
+  depth: number;
+}
+
+function isSameDomain(url1: string, url2: string): boolean {
+  try {
+    return new URL(url1).hostname === new URL(url2).hostname;
+  } catch {
+    return false;
+  }
+}
+
+function normalizeForDedup(url: string): string {
+  try {
+    const u = new URL(url);
+    u.hash = '';
+    u.search = '';
+    return u.href.replace(/\/+$/, '') || u.href;
+  } catch {
+    return url;
+  }
+}
+
+export async function crawl(
+  startUrl: string,
+  options: CrawlOptions = {}
+): Promise<{ results: ScrapeResult[]; stats: CrawlStats }> {
+  const {
+    maxDepth = 2,
+    maxPages = 50,
+    maxConcurrent = 3,
+    sameDomain = true,
+    delayMs = 1000,
+    onDiscover,
+    onResult,
+    onError,
+    ...scrapeOpts
+  } = options;
+
+  const startTime = Date.now();
+  const visited = new Set<string>();
+  const queue: CrawlEntry[] = [{ url: startUrl, depth: 0 }];
+  const results: ScrapeResult[] = [];
+  let failed = 0;
+
+  visited.add(normalizeForDedup(startUrl));
+
+  while (queue.length > 0 && results.length < maxPages) {
+    const batch = queue.splice(0, maxConcurrent);
+    const promises = batch.map(async (entry) => {
+      if (entry.depth > maxDepth) return;
+
+      try {
+        const result = await scrapeUrl(entry.url, scrapeOpts);
+        results.push(result);
+        onResult?.(result);
+
+        // Extract links for further crawling
+        if (entry.depth < maxDepth) {
+          for (const link of result.links) {
+            const normalized = normalizeForDedup(link.href);
+            if (visited.has(normalized)) continue;
+            if (sameDomain && !isSameDomain(startUrl, link.href)) continue;
+            try {
+              new URL(link.href);
+            } catch {
+              continue;
+            }
+            visited.add(normalized);
+            queue.push({ url: link.href, depth: entry.depth + 1 });
+            onDiscover?.(link.href, entry.depth + 1);
+          }
+        }
+      } catch (error) {
+        failed++;
+        onError?.(entry.url, error as Error);
+      }
+    });
+
+    await Promise.all(promises);
+
+    if (delayMs > 0 && queue.length > 0) {
+      await new Promise(r => setTimeout(r, delayMs));
+    }
+  }
+
+  const duration = Date.now() - startTime;
+
+  return {
+    results,
+    stats: {
+      totalUrls: visited.size,
+      successful: results.length,
+      failed,
+      skipped: visited.size - results.length - failed,
+      duration,
+      avgResponseTime: results.length > 0 ? results.reduce((sum, r) => sum + r.duration, 0) / results.length : 0,
+    },
+  };
+}

package/scrape/dev-server.ts

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+import { createServer } from 'node:http';
+
+import { scrapeUrl } from './scrape.js';
+import { serializeResult } from './serialize.js';
+import { resolveFormat, buildSerializeOptions } from './formats.js';
+import { formatForAgent } from './agent.js';
+import { randomPort } from './utils/port.js';
+import type { ScrapeOptions } from './types.js';
+
+const PORT = randomPort();
+
+const server = createServer(async (req, res) => {
+  res.setHeader('Access-Control-Allow-Origin', '*');
+  res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
+  res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+  if (req.method === 'OPTIONS') {
+    res.writeHead(200);
+    res.end();
+    return;
+  }
+
+  if (req.method !== 'POST' || req.url !== '/api/scrape') {
+    res.writeHead(404);
+    res.end(JSON.stringify({ error: 'Not found' }));
+    return;
+  }
+
+  let body = '';
+  req.on('data', (chunk: Buffer) => { body += chunk.toString(); });
+  req.on('end', async () => {
+    try {
+      const { url, format: fmt, options = {} }: {
+        url: string;
+        format?: string;
+        options?: Record<string, unknown>;
+      } = JSON.parse(body);
+      if (!url) throw new Error('URL is required');
+
+      const format = resolveFormat(fmt || 'markdown');
+      const result = await scrapeUrl(url, {
+        timeout: 30000,
+        scroll: options.scroll as boolean,
+        extractLinks: (options.extractLinks as boolean) ?? true,
+        extractImages: (options.extractImages as boolean) ?? true,
+        extractTables: (options.extractTables as boolean) ?? true,
+      } as ScrapeOptions);
+
+      let response: Record<string, unknown>;
+
+      if (options.agent) {
+        const agentOutput = formatForAgent(result);
+        response = {
+          success: true,
+          title: result.title,
+          url: result.url,
+          duration: result.duration,
+          size: result.size,
+          agentOutput,
+        };
+      } else {
+        const serializeOpts = buildSerializeOptions(format);
+        const serialized = serializeResult(result, serializeOpts);
+        response = {
+          success: true,
+          title: result.title,
+          url: result.url,
+          format,
+          content: serialized.content,
+          duration: result.duration,
+          size: serialized.size,
+          metadata: result.metadata,
+          links: result.links,
+          images: result.images,
+          tables: result.tables,
+          result,
+        };
+      }
+
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify(response));
+    } catch (err) {
+      const error = err as Error;
+      res.writeHead(500, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: error.message }));
+    }
+  });
+});
+
+server.listen(PORT, () => {
+  console.log(`API server running at http://localhost:${PORT}`);
+  console.log(`POST /api/scrape with { url, format?, options? }`);
+});

package/scrape/errors.ts

@@ -0,0 +1,132 @@
+export const ErrorCode = {
+  VALIDATION_FAILED: 'VALIDATION_FAILED',
+  INVALID_URL: 'INVALID_URL',
+  TIMEOUT: 'TIMEOUT',
+  BLOCKED: 'BLOCKED',
+  RATE_LIMITED: 'RATE_LIMITED',
+  PROXY_ERROR: 'PROXY_ERROR',
+  PARSE_ERROR: 'PARSE_ERROR',
+  AUTH_REQUIRED: 'AUTH_REQUIRED',
+  NETWORK_ERROR: 'NETWORK_ERROR',
+  BROWSER_NOT_AVAILABLE: 'BROWSER_NOT_AVAILABLE',
+  CRAWL_DEPTH_EXCEEDED: 'CRAWL_DEPTH_EXCEEDED',
+  MAX_RETRIES_EXCEEDED: 'MAX_RETRIES_EXCEEDED',
+} as const;
+
+export type ErrorCode = typeof ErrorCode[keyof typeof ErrorCode];
+
+export class WebScrapeError extends Error {
+  public readonly code: ErrorCode;
+  public readonly statusCode: number;
+  public readonly isRetryable: boolean;
+  public readonly timestamp: string;
+  public readonly details?: Record<string, unknown>;
+  public readonly cause?: Error;
+
+  constructor(
+    message: string,
+    code: ErrorCode,
+    statusCode: number,
+    isRetryable: boolean,
+    details?: Record<string, unknown>,
+    cause?: Error
+  ) {
+    super(message);
+    this.name = 'WebScrapeError';
+    this.code = code;
+    this.statusCode = statusCode;
+    this.isRetryable = isRetryable;
+    this.timestamp = new Date().toISOString();
+    this.details = details;
+    this.cause = cause;
+    Object.setPrototypeOf(this, new.target.prototype);
+    Error.captureStackTrace?.(this, this.constructor);
+  }
+
+  toJSON() {
+    return {
+      error: {
+        name: this.name,
+        code: this.code,
+        message: this.message,
+        statusCode: this.statusCode,
+        isRetryable: this.isRetryable,
+        timestamp: this.timestamp,
+        details: this.details,
+      },
+    };
+  }
+}
+
+export class ValidationError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.VALIDATION_FAILED, 400, false, details, cause);
+    this.name = 'ValidationError';
+  }
+}
+
+export class TimeoutError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.TIMEOUT, 504, true, details, cause);
+    this.name = 'TimeoutError';
+  }
+}
+
+export class BlockedError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.BLOCKED, 403, false, details, cause);
+    this.name = 'BlockedError';
+  }
+}
+
+export class RateLimitError extends WebScrapeError {
+  public readonly retryAfterMs?: number;
+  constructor(
+    message: string,
+    retryAfterMs?: number,
+    details?: Record<string, unknown>,
+    cause?: Error
+  ) {
+    super(message, ErrorCode.RATE_LIMITED, 429, true, { ...details, retryAfterMs }, cause);
+    this.name = 'RateLimitError';
+    this.retryAfterMs = retryAfterMs;
+  }
+}
+
+export class ProxyError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.PROXY_ERROR, 502, true, details, cause);
+    this.name = 'ProxyError';
+  }
+}
+
+export class ParseError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.PARSE_ERROR, 422, false, details, cause);
+    this.name = 'ParseError';
+  }
+}
+
+export class AuthError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.AUTH_REQUIRED, 401, false, details, cause);
+    this.name = 'AuthError';
+  }
+}
+
+export class NetworkError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.NETWORK_ERROR, 503, true, details, cause);
+    this.name = 'NetworkError';
+  }
+}
+
+export class BrowserNotAvailableError extends WebScrapeError {
+  constructor(
+    message: string = 'Playwright is not installed. Install it with: npm install playwright',
+    details?: Record<string, unknown>
+  ) {
+    super(message, ErrorCode.BROWSER_NOT_AVAILABLE, 500, false, details);
+    this.name = 'BrowserNotAvailableError';
+  }
+}

package/scrape/events.ts

@@ -0,0 +1,26 @@
+import Emittery from 'emittery';
+import type { ScrapeOptions } from './types.js';
+
+export interface WebScrapeEvents {
+  'request:start': { url: string; options: ScrapeOptions; timestamp: number };
+  'request:response': { url: string; status: number; attempt: number; duration: number };
+  'request:error': { url: string; error: Error; attempt: number };
+  'request:retry': { url: string; attempt: number; maxRetries: number; delayMs: number };
+  'request:complete': { url: string; duration: number; success: boolean };
+  'cache:hit': { key: string };
+  'cache:miss': { key: string };
+  'cache:set': { key: string; ttlMs?: number };
+  'proxy:rotate': { proxy: string; reason: string };
+  'proxy:error': { proxy: string; error: Error };
+  'proxy:disabled': { proxy: string; reason: string };
+  'crawl:discover': { url: string; depth: number; parentUrl?: string };
+  'crawl:complete': { totalUrls: number; successful: number; failed: number; duration: number };
+}
+
+export type EventEmitter = Emittery<WebScrapeEvents>;
+
+export function createEventEmitter(): EventEmitter {
+  return new Emittery<WebScrapeEvents>();
+}
+
+export const globalEmitter = createEventEmitter();

package/scrape/extract.ts

@@ -0,0 +1,165 @@
+import * as cheerio from 'cheerio';
+import type { ExtractOptions, LinkInfo, ImageInfo, TableInfo, PageMetadata } from './types.js';
+import { normalizeUrl, isInternalUrl } from './utils.js';
+
+export interface ExtractedContent {
+  content: string;
+  text: string;
+  links: LinkInfo[];
+  images: ImageInfo[];
+  tables: TableInfo[];
+  metadata: PageMetadata;
+  jsonLd: unknown[];
+}
+
+function extractReadableContent($: cheerio.CheerioAPI): string {
+  const articleSelectors = [
+    'article', '[role="main"]', 'main', '.post-content', '.article-content',
+    '.entry-content', '.content', '#content', '.prose', '.markdown-body',
+  ];
+  for (const sel of articleSelectors) {
+    const el = $(sel).first();
+    if (el.length) return el.text().trim();
+  }
+  return $('body').text().trim();
+}
+
+function extractMetadata($: cheerio.CheerioAPI): PageMetadata {
+  const getMeta = (name: string): string | undefined => {
+    return $(`meta[name="${name}"], meta[property="${name}"]`).attr('content') || undefined;
+  };
+  return {
+    title: $('title').text() || getMeta('og:title') || '',
+    description: getMeta('description') || getMeta('og:description'),
+    favicon: $('link[rel="icon"]').attr('href') || $('link[rel="shortcut icon"]').attr('href'),
+    charset: $('meta[charset]').attr('charset') || $('meta[http-equiv="Content-Type"]').attr('content'),
+    language: $('html').attr('lang') || undefined,
+    author: getMeta('author'),
+    publishedDate: getMeta('article:published_time') || getMeta('date'),
+    ogImage: getMeta('og:image'),
+    ogType: getMeta('og:type'),
+    keywords: getMeta('keywords')?.split(',').map(k => k.trim()),
+  };
+}
+
+function extractJsonLd($: cheerio.CheerioAPI): unknown[] {
+  const results: unknown[] = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    try {
+      const data = JSON.parse($(el).html() || '');
+      if (data['@graph']) {
+        results.push(...(Array.isArray(data['@graph']) ? data['@graph'] : [data['@graph']]));
+      } else {
+        results.push(data);
+      }
+    } catch {}
+  });
+  return results;
+}
+
+function extractLinks($: cheerio.CheerioAPI, baseUrl: string): LinkInfo[] {
+  const links: LinkInfo[] = [];
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href') || '';
+    const text = $(el).text().trim().slice(0, 200);
+    if (!href || href.startsWith('#') || href.startsWith('javascript:')) return;
+    const fullUrl = normalizeUrl(href, baseUrl);
+    links.push({
+      href: fullUrl,
+      text: text || fullUrl,
+      isInternal: isInternalUrl(fullUrl, baseUrl),
+      isExternal: !isInternalUrl(fullUrl, baseUrl),
+    });
+  });
+  return links;
+}
+
+function extractImages($: cheerio.CheerioAPI, baseUrl: string): ImageInfo[] {
+  const images: ImageInfo[] = [];
+  $('img[src]').each((_, el) => {
+    const src = normalizeUrl($(el).attr('src') || '', baseUrl);
+    const alt = $(el).attr('alt') || '';
+    const width = parseInt($(el).attr('width') || '') || undefined;
+    const height = parseInt($(el).attr('height') || '') || undefined;
+    if (src) images.push({ src, alt: alt.slice(0, 200), width, height });
+  });
+  return images;
+}
+
+function extractTables($: cheerio.CheerioAPI): TableInfo[] {
+  const tables: TableInfo[] = [];
+  $('table').each((_, table) => {
+    const headers: string[] = [];
+    const rows: string[][] = [];
+    $('thead tr th, thead tr td', table).each((_, th) => {
+      headers.push($(th).text().trim());
+    });
+    $('tbody tr, > tr', table).each((_, tr) => {
+      if ($(tr).parent().is('thead')) return;
+      const row: string[] = [];
+      $('td, th', tr).each((_, td) => {
+        row.push($(td).text().trim());
+      });
+      if (row.length) rows.push(row);
+    });
+    const caption = $('caption', table).text().trim() || undefined;
+    if (rows.length) tables.push({ headers, rows, caption });
+  });
+  return tables;
+}
+
+function htmlToText(html: string): string {
+  return html
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/p>/gi, '\n\n')
+    .replace(/<\/div>/gi, '\n')
+    .replace(/<\/h[1-6]>/gi, '\n')
+    .replace(/<\/li>/gi, '\n')
+    .replace(/<[^>]+>/g, '')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+export async function extractContent(html: string, options: ExtractOptions = {}): Promise<ExtractedContent> {
+  const $ = cheerio.load(html);
+  const baseUrl = $('base').attr('href') || '';
+
+  if (options.removeSelectors?.length) {
+    for (const sel of options.removeSelectors) {
+      $(sel).remove();
+    }
+  }
+
+  const metadata = extractMetadata($);
+  const jsonLd = extractJsonLd($);
+  const links = options.preserveLinks !== false ? extractLinks($, baseUrl) : [];
+  const images = options.preserveImages !== false ? extractImages($, baseUrl) : [];
+  const tables = options.preserveTables !== false ? extractTables($) : [];
+
+  const text = options.readable
+    ? extractReadableContent($)
+    : $('body').text().trim();
+
+  const content = htmlToText(text);
+
+  return {
+    content: options.maxLength ? content.slice(0, options.maxLength) : content,
+    text,
+    links,
+    images,
+    tables,
+    metadata,
+    jsonLd,
+  };
+}
+
+export async function extractReadable(html: string): Promise<string> {
+  const result = await extractContent(html, { readable: true });
+  return result.content;
+}

package/scrape/fetch.ts

@@ -0,0 +1,105 @@
+import * as cheerio from 'cheerio';
+import type { ScrapeOptions } from './types.js';
+import { getHeaders, getRandomProfile } from './headers.js';
+import { WebScrapeError, ErrorCode } from './errors.js';
+
+export interface FetchResult {
+  html: string;
+  url: string;
+  status: number;
+  headers: Record<string, string>;
+  duration: number;
+}
+
+export async function fetchHtml(url: string, options: {
+  timeout?: number;
+  headers?: Record<string, string>;
+  userAgent?: string;
+  proxy?: string;
+  signal?: AbortSignal;
+} = {}): Promise<FetchResult> {
+  const startTime = Date.now();
+  const profile = getRandomProfile();
+  const baseHeaders = getHeaders(profile);
+  const headers = { ...baseHeaders, ...options.headers };
+  if (options.userAgent) headers['User-Agent'] = options.userAgent;
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), options.timeout || 30000);
+  const signal = options.signal ? AbortSignal.any([controller.signal, options.signal]) : controller.signal;
+
+  try {
+    const response = await fetch(url, {
+      method: 'GET',
+      headers,
+      signal,
+      redirect: 'follow',
+    });
+
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      throw new WebScrapeError(
+        `HTTP ${response.status}: ${response.statusText}`,
+        response.status === 403 ? ErrorCode.BLOCKED :
+        response.status === 429 ? ErrorCode.RATE_LIMITED :
+        response.status >= 500 ? ErrorCode.NETWORK_ERROR :
+        ErrorCode.NETWORK_ERROR,
+        response.status,
+        [408, 429, 500, 502, 503, 504].includes(response.status),
+        { url, status: response.status }
+      );
+    }
+
+    const html = await response.text();
+    const responseHeaders: Record<string, string> = {};
+    response.headers.forEach((value, key) => { responseHeaders[key] = value; });
+
+    return {
+      html,
+      url: response.url || url,
+      status: response.status,
+      headers: responseHeaders,
+      duration: Date.now() - startTime,
+    };
+  } catch (error) {
+    clearTimeout(timeoutId);
+    if (error instanceof WebScrapeError) throw error;
+    if ((error as Error).name === 'AbortError') {
+      throw new WebScrapeError(
+        `Request timeout after ${options.timeout || 30000}ms`,
+        ErrorCode.TIMEOUT,
+        504,
+        true,
+        { url, timeout: options.timeout || 30000 }
+      );
+    }
+    throw new WebScrapeError(
+      `Network error: ${(error as Error).message}`,
+      ErrorCode.NETWORK_ERROR,
+      503,
+      true,
+      { url },
+      error as Error
+    );
+  }
+}
+
+export async function fetchAndParse(url: string, options: Partial<ScrapeOptions> = {}): Promise<FetchResult> {
+  return fetchHtml(url, {
+    timeout: options.timeout,
+    headers: options.headers,
+    userAgent: options.userAgent,
+    proxy: options.proxy,
+  });
+}
+
+export function detectRenderingMode(html: string): 'static' | 'spa' | 'hydrated' {
+  if (/<div id=["'](?:root|app|__next|__nuxt)["']>\s*<\/div>/.test(html)) return 'spa';
+  if (html.includes('__NEXT_DATA__') || html.includes('__NUXT__') || html.includes('__VUE_SSR_DATA__')) return 'hydrated';
+  const $ = cheerio.load(html);
+  if ($('script[type="application/ld+json"]').length > 0) return 'static';
+  const textContent = $.text().replace(/\s+/g, ' ').trim();
+  if (textContent.length > 500) return 'static';
+  return 'spa';
+}