@wenathlan/saddle 1.8.8 → 1.8.9

package/scrape/errors.ts

@@ -1,132 +0,0 @@
-export const ErrorCode = {
-  VALIDATION_FAILED: 'VALIDATION_FAILED',
-  INVALID_URL: 'INVALID_URL',
-  TIMEOUT: 'TIMEOUT',
-  BLOCKED: 'BLOCKED',
-  RATE_LIMITED: 'RATE_LIMITED',
-  PROXY_ERROR: 'PROXY_ERROR',
-  PARSE_ERROR: 'PARSE_ERROR',
-  AUTH_REQUIRED: 'AUTH_REQUIRED',
-  NETWORK_ERROR: 'NETWORK_ERROR',
-  BROWSER_NOT_AVAILABLE: 'BROWSER_NOT_AVAILABLE',
-  CRAWL_DEPTH_EXCEEDED: 'CRAWL_DEPTH_EXCEEDED',
-  MAX_RETRIES_EXCEEDED: 'MAX_RETRIES_EXCEEDED',
-} as const;
-
-export type ErrorCode = typeof ErrorCode[keyof typeof ErrorCode];
-
-export class WebScrapeError extends Error {
-  public readonly code: ErrorCode;
-  public readonly statusCode: number;
-  public readonly isRetryable: boolean;
-  public readonly timestamp: string;
-  public readonly details?: Record<string, unknown>;
-  public readonly cause?: Error;
-
-  constructor(
-    message: string,
-    code: ErrorCode,
-    statusCode: number,
-    isRetryable: boolean,
-    details?: Record<string, unknown>,
-    cause?: Error
-  ) {
-    super(message);
-    this.name = 'WebScrapeError';
-    this.code = code;
-    this.statusCode = statusCode;
-    this.isRetryable = isRetryable;
-    this.timestamp = new Date().toISOString();
-    this.details = details;
-    this.cause = cause;
-    Object.setPrototypeOf(this, new.target.prototype);
-    Error.captureStackTrace?.(this, this.constructor);
-  }
-
-  toJSON() {
-    return {
-      error: {
-        name: this.name,
-        code: this.code,
-        message: this.message,
-        statusCode: this.statusCode,
-        isRetryable: this.isRetryable,
-        timestamp: this.timestamp,
-        details: this.details,
-      },
-    };
-  }
-}
-
-export class ValidationError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.VALIDATION_FAILED, 400, false, details, cause);
-    this.name = 'ValidationError';
-  }
-}
-
-export class TimeoutError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.TIMEOUT, 504, true, details, cause);
-    this.name = 'TimeoutError';
-  }
-}
-
-export class BlockedError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.BLOCKED, 403, false, details, cause);
-    this.name = 'BlockedError';
-  }
-}
-
-export class RateLimitError extends WebScrapeError {
-  public readonly retryAfterMs?: number;
-  constructor(
-    message: string,
-    retryAfterMs?: number,
-    details?: Record<string, unknown>,
-    cause?: Error
-  ) {
-    super(message, ErrorCode.RATE_LIMITED, 429, true, { ...details, retryAfterMs }, cause);
-    this.name = 'RateLimitError';
-    this.retryAfterMs = retryAfterMs;
-  }
-}
-
-export class ProxyError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.PROXY_ERROR, 502, true, details, cause);
-    this.name = 'ProxyError';
-  }
-}
-
-export class ParseError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.PARSE_ERROR, 422, false, details, cause);
-    this.name = 'ParseError';
-  }
-}
-
-export class AuthError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.AUTH_REQUIRED, 401, false, details, cause);
-    this.name = 'AuthError';
-  }
-}
-
-export class NetworkError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.NETWORK_ERROR, 503, true, details, cause);
-    this.name = 'NetworkError';
-  }
-}
-
-export class BrowserNotAvailableError extends WebScrapeError {
-  constructor(
-    message: string = 'Playwright is not installed. Install it with: npm install playwright',
-    details?: Record<string, unknown>
-  ) {
-    super(message, ErrorCode.BROWSER_NOT_AVAILABLE, 500, false, details);
-    this.name = 'BrowserNotAvailableError';
-  }
-}

package/scrape/events.ts

@@ -1,26 +0,0 @@
-import Emittery from 'emittery';
-import type { ScrapeOptions } from './types.js';
-
-export interface WebScrapeEvents {
-  'request:start': { url: string; options: ScrapeOptions; timestamp: number };
-  'request:response': { url: string; status: number; attempt: number; duration: number };
-  'request:error': { url: string; error: Error; attempt: number };
-  'request:retry': { url: string; attempt: number; maxRetries: number; delayMs: number };
-  'request:complete': { url: string; duration: number; success: boolean };
-  'cache:hit': { key: string };
-  'cache:miss': { key: string };
-  'cache:set': { key: string; ttlMs?: number };
-  'proxy:rotate': { proxy: string; reason: string };
-  'proxy:error': { proxy: string; error: Error };
-  'proxy:disabled': { proxy: string; reason: string };
-  'crawl:discover': { url: string; depth: number; parentUrl?: string };
-  'crawl:complete': { totalUrls: number; successful: number; failed: number; duration: number };
-}
-
-export type EventEmitter = Emittery<WebScrapeEvents>;
-
-export function createEventEmitter(): EventEmitter {
-  return new Emittery<WebScrapeEvents>();
-}
-
-export const globalEmitter = createEventEmitter();

package/scrape/extract.js

@@ -1,14 +0,0 @@
-/**
- * structured first extraction uses small built in heuristics and returns serializable data.
- */
-export function extracthtml(html, url) {
-  const title = match(html, /<title[^>]*>([\s\S]*?)<\/title>/i);
-  const description = match(html, /<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']/i) ?? match(html, /<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["']/i);
-  const links = [...html.matchAll(/<a[^>]+href=["']([^"']+)["'][^>]*>/gi)].map((item) => resolveurl(item[1], url)).filter(Boolean);
-  const text = html.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
-  return { url, title: decode(title ?? ""), description: decode(description ?? ""), text, links: [...new Set(links)] };
-}
-
-function match(value, expression) { return value.match(expression)?.[1]?.trim(); }
-function decode(value) { return value.replaceAll("&amp;", "&").replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&quot;", '"'); }
-function resolveurl(value, base) { try { return base ? new URL(value, base).href : new URL(value).href; } catch { return null; } }

package/scrape/extract.ts

@@ -1,165 +0,0 @@
-import * as cheerio from 'cheerio';
-import type { ExtractOptions, LinkInfo, ImageInfo, TableInfo, PageMetadata } from './types.js';
-import { normalizeUrl, isInternalUrl } from './utils.js';
-
-export interface ExtractedContent {
-  content: string;
-  text: string;
-  links: LinkInfo[];
-  images: ImageInfo[];
-  tables: TableInfo[];
-  metadata: PageMetadata;
-  jsonLd: unknown[];
-}
-
-function extractReadableContent($: cheerio.CheerioAPI): string {
-  const articleSelectors = [
-    'article', '[role="main"]', 'main', '.post-content', '.article-content',
-    '.entry-content', '.content', '#content', '.prose', '.markdown-body',
-  ];
-  for (const sel of articleSelectors) {
-    const el = $(sel).first();
-    if (el.length) return el.text().trim();
-  }
-  return $('body').text().trim();
-}
-
-function extractMetadata($: cheerio.CheerioAPI): PageMetadata {
-  const getMeta = (name: string): string | undefined => {
-    return $(`meta[name="${name}"], meta[property="${name}"]`).attr('content') || undefined;
-  };
-  return {
-    title: $('title').text() || getMeta('og:title') || '',
-    description: getMeta('description') || getMeta('og:description'),
-    favicon: $('link[rel="icon"]').attr('href') || $('link[rel="shortcut icon"]').attr('href'),
-    charset: $('meta[charset]').attr('charset') || $('meta[http-equiv="Content-Type"]').attr('content'),
-    language: $('html').attr('lang') || undefined,
-    author: getMeta('author'),
-    publishedDate: getMeta('article:published_time') || getMeta('date'),
-    ogImage: getMeta('og:image'),
-    ogType: getMeta('og:type'),
-    keywords: getMeta('keywords')?.split(',').map(k => k.trim()),
-  };
-}
-
-function extractJsonLd($: cheerio.CheerioAPI): unknown[] {
-  const results: unknown[] = [];
-  $('script[type="application/ld+json"]').each((_, el) => {
-    try {
-      const data = JSON.parse($(el).html() || '');
-      if (data['@graph']) {
-        results.push(...(Array.isArray(data['@graph']) ? data['@graph'] : [data['@graph']]));
-      } else {
-        results.push(data);
-      }
-    } catch {}
-  });
-  return results;
-}
-
-function extractLinks($: cheerio.CheerioAPI, baseUrl: string): LinkInfo[] {
-  const links: LinkInfo[] = [];
-  $('a[href]').each((_, el) => {
-    const href = $(el).attr('href') || '';
-    const text = $(el).text().trim().slice(0, 200);
-    if (!href || href.startsWith('#') || href.startsWith('javascript:')) return;
-    const fullUrl = normalizeUrl(href, baseUrl);
-    links.push({
-      href: fullUrl,
-      text: text || fullUrl,
-      isInternal: isInternalUrl(fullUrl, baseUrl),
-      isExternal: !isInternalUrl(fullUrl, baseUrl),
-    });
-  });
-  return links;
-}
-
-function extractImages($: cheerio.CheerioAPI, baseUrl: string): ImageInfo[] {
-  const images: ImageInfo[] = [];
-  $('img[src]').each((_, el) => {
-    const src = normalizeUrl($(el).attr('src') || '', baseUrl);
-    const alt = $(el).attr('alt') || '';
-    const width = parseInt($(el).attr('width') || '') || undefined;
-    const height = parseInt($(el).attr('height') || '') || undefined;
-    if (src) images.push({ src, alt: alt.slice(0, 200), width, height });
-  });
-  return images;
-}
-
-function extractTables($: cheerio.CheerioAPI): TableInfo[] {
-  const tables: TableInfo[] = [];
-  $('table').each((_, table) => {
-    const headers: string[] = [];
-    const rows: string[][] = [];
-    $('thead tr th, thead tr td', table).each((_, th) => {
-      headers.push($(th).text().trim());
-    });
-    $('tbody tr, > tr', table).each((_, tr) => {
-      if ($(tr).parent().is('thead')) return;
-      const row: string[] = [];
-      $('td, th', tr).each((_, td) => {
-        row.push($(td).text().trim());
-      });
-      if (row.length) rows.push(row);
-    });
-    const caption = $('caption', table).text().trim() || undefined;
-    if (rows.length) tables.push({ headers, rows, caption });
-  });
-  return tables;
-}
-
-function htmlToText(html: string): string {
-  return html
-    .replace(/<br\s*\/?>/gi, '\n')
-    .replace(/<\/p>/gi, '\n\n')
-    .replace(/<\/div>/gi, '\n')
-    .replace(/<\/h[1-6]>/gi, '\n')
-    .replace(/<\/li>/gi, '\n')
-    .replace(/<[^>]+>/g, '')
-    .replace(/&nbsp;/g, ' ')
-    .replace(/&amp;/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/\n{3,}/g, '\n\n')
-    .trim();
-}
-
-export async function extractContent(html: string, options: ExtractOptions = {}): Promise<ExtractedContent> {
-  const $ = cheerio.load(html);
-  const baseUrl = $('base').attr('href') || '';
-
-  if (options.removeSelectors?.length) {
-    for (const sel of options.removeSelectors) {
-      $(sel).remove();
-    }
-  }
-
-  const metadata = extractMetadata($);
-  const jsonLd = extractJsonLd($);
-  const links = options.preserveLinks !== false ? extractLinks($, baseUrl) : [];
-  const images = options.preserveImages !== false ? extractImages($, baseUrl) : [];
-  const tables = options.preserveTables !== false ? extractTables($) : [];
-
-  const text = options.readable
-    ? extractReadableContent($)
-    : $('body').text().trim();
-
-  const content = htmlToText(text);
-
-  return {
-    content: options.maxLength ? content.slice(0, options.maxLength) : content,
-    text,
-    links,
-    images,
-    tables,
-    metadata,
-    jsonLd,
-  };
-}
-
-export async function extractReadable(html: string): Promise<string> {
-  const result = await extractContent(html, { readable: true });
-  return result.content;
-}

package/scrape/fetch.ts

@@ -1,105 +0,0 @@
-import * as cheerio from 'cheerio';
-import type { ScrapeOptions } from './types.js';
-import { getHeaders, getRandomProfile } from './headers.js';
-import { WebScrapeError, ErrorCode } from './errors.js';
-
-export interface FetchResult {
-  html: string;
-  url: string;
-  status: number;
-  headers: Record<string, string>;
-  duration: number;
-}
-
-export async function fetchHtml(url: string, options: {
-  timeout?: number;
-  headers?: Record<string, string>;
-  userAgent?: string;
-  proxy?: string;
-  signal?: AbortSignal;
-} = {}): Promise<FetchResult> {
-  const startTime = Date.now();
-  const profile = getRandomProfile();
-  const baseHeaders = getHeaders(profile);
-  const headers = { ...baseHeaders, ...options.headers };
-  if (options.userAgent) headers['User-Agent'] = options.userAgent;
-
-  const controller = new AbortController();
-  const timeoutId = setTimeout(() => controller.abort(), options.timeout || 30000);
-  const signal = options.signal ? AbortSignal.any([controller.signal, options.signal]) : controller.signal;
-
-  try {
-    const response = await fetch(url, {
-      method: 'GET',
-      headers,
-      signal,
-      redirect: 'follow',
-    });
-
-    clearTimeout(timeoutId);
-
-    if (!response.ok) {
-      throw new WebScrapeError(
-        `HTTP ${response.status}: ${response.statusText}`,
-        response.status === 403 ? ErrorCode.BLOCKED :
-        response.status === 429 ? ErrorCode.RATE_LIMITED :
-        response.status >= 500 ? ErrorCode.NETWORK_ERROR :
-        ErrorCode.NETWORK_ERROR,
-        response.status,
-        [408, 429, 500, 502, 503, 504].includes(response.status),
-        { url, status: response.status }
-      );
-    }
-
-    const html = await response.text();
-    const responseHeaders: Record<string, string> = {};
-    response.headers.forEach((value, key) => { responseHeaders[key] = value; });
-
-    return {
-      html,
-      url: response.url || url,
-      status: response.status,
-      headers: responseHeaders,
-      duration: Date.now() - startTime,
-    };
-  } catch (error) {
-    clearTimeout(timeoutId);
-    if (error instanceof WebScrapeError) throw error;
-    if ((error as Error).name === 'AbortError') {
-      throw new WebScrapeError(
-        `Request timeout after ${options.timeout || 30000}ms`,
-        ErrorCode.TIMEOUT,
-        504,
-        true,
-        { url, timeout: options.timeout || 30000 }
-      );
-    }
-    throw new WebScrapeError(
-      `Network error: ${(error as Error).message}`,
-      ErrorCode.NETWORK_ERROR,
-      503,
-      true,
-      { url },
-      error as Error
-    );
-  }
-}
-
-export async function fetchAndParse(url: string, options: Partial<ScrapeOptions> = {}): Promise<FetchResult> {
-  return fetchHtml(url, {
-    timeout: options.timeout,
-    headers: options.headers,
-    userAgent: options.userAgent,
-    proxy: options.proxy,
-  });
-}
-
-export function detectRenderingMode(html: string): 'static' | 'spa' | 'hydrated' {
-  if (/<div id=["'](?:root|app|__next|__nuxt)["']>\s*<\/div>/.test(html)) return 'spa';
-  if (html.includes('__NEXT_DATA__') || html.includes('__NUXT__') || html.includes('__VUE_SSR_DATA__')) return 'hydrated';
-  const $ = cheerio.load(html);
-  if ($('script[type="application/ld+json"]').length > 0) return 'static';
-  const textContent = $.text().replace(/\s+/g, ' ').trim();
-  if (textContent.length > 500) return 'static';
-  return 'spa';
-}

package/scrape/formats.ts

@@ -1,85 +0,0 @@
-import type { ScrapeFormat, SerializeTarget, SerializeOptions, ScrapeResult } from './types.js';
-import { serializeResult as sr } from './serialize.js';
-
-const FORMAT_ALIASES: Record<string, ScrapeFormat> = {
-  md: 'markdown',
-  txt: 'text',
-  plain: 'text',
-  xml: 'xml',
-  json: 'json',
-  redis: 'redis',
-  html: 'html',
-  h: 'html',
-};
-
-const FORMAT_EXTENSIONS: Record<string, ScrapeFormat> = {
-  '.md': 'markdown',
-  '.markdown': 'markdown',
-  '.html': 'html',
-  '.htm': 'html',
-  '.txt': 'text',
-  '.json': 'json',
-  '.xml': 'xml',
-  '.redis': 'redis',
-};
-
-export function resolveFormat(format: string): ScrapeFormat {
-  const lower = format.toLowerCase().trim();
-  return FORMAT_ALIASES[lower] || (lower as ScrapeFormat);
-}
-
-export function formatFromExtension(ext: string): ScrapeFormat | null {
-  return FORMAT_EXTENSIONS[ext.toLowerCase()] ?? null;
-}
-
-export function extensionForFormat(format: ScrapeFormat): string {
-  const map: Record<ScrapeFormat, string> = {
-    markdown: '.md',
-    html: '.html',
-    text: '.txt',
-    json: '.json',
-    xml: '.xml',
-    redis: '.redis',
-  };
-  return map[format] || '.md';
-}
-
-export function detectContentType(html: string): 'article' | 'list' | 'page' | 'other' {
-  if (html.includes('<article') || html.includes('entry-content') || html.includes('class="post-content"') || html.includes('class="article-content"') || html.includes('role="main"')) {
-    return 'article';
-  }
-  const listCount = (html.match(/<li>/gi) || []).length;
-  if (listCount > 20) return 'list';
-  if (html.includes('<article') || html.includes('entry-content')) return 'article';
-  return 'page';
-}
-
-export function buildSerializeOptions(format: ScrapeFormat, pretty = true): SerializeOptions {
-  const targetMap: Record<ScrapeFormat, SerializeTarget> = {
-    markdown: 'markdown',
-    html: 'markdown',
-    text: 'text',
-    json: 'json',
-    xml: 'xml',
-    redis: 'redis',
-  };
-  return {
-    format: targetMap[format] || 'markdown',
-    pretty,
-    includeMetadata: true,
-  };
-}
-
-export async function convertResult(result: ScrapeResult, targetFormat: ScrapeFormat): Promise<ScrapeResult> {
-  if (result.format === targetFormat) return result;
-
-  const opts = buildSerializeOptions(targetFormat);
-  const serialized = sr(result, opts);
-
-  return {
-    ...result,
-    content: serialized.content,
-    format: targetFormat,
-    size: serialized.size,
-  };
-}

package/scrape/headers.ts

@@ -1,71 +0,0 @@
-export interface HeaderProfile {
-  userAgent: string;
-  accept: string;
-  acceptLanguage: string;
-  secChUa: string;
-  secChUaPlatform: string;
-}
-
-const CHROME_PROFILES: HeaderProfile[] = [
-  {
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
-    secChUaPlatform: '"Windows"',
-  },
-  {
-    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
-    secChUaPlatform: '"macOS"',
-  },
-  {
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-    acceptLanguage: 'en-US,en;q=0.5',
-    secChUa: '',
-    secChUaPlatform: '"Windows"',
-  },
-  {
-    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '',
-    secChUaPlatform: '"macOS"',
-  },
-];
-
-export function getRandomProfile(): HeaderProfile {
-  return CHROME_PROFILES[Math.floor(Math.random() * CHROME_PROFILES.length)];
-}
-
-export function getHeaders(profile?: HeaderProfile): Record<string, string> {
-  const p = profile || getRandomProfile();
-  const headers: Record<string, string> = {
-    'User-Agent': p.userAgent,
-    'Accept': p.accept,
-    'Accept-Language': p.acceptLanguage,
-    'Accept-Encoding': 'gzip, deflate, br',
-    'Connection': 'keep-alive',
-    'Upgrade-Insecure-Requests': '1',
-    'Sec-Fetch-Dest': 'document',
-    'Sec-Fetch-Mode': 'navigate',
-    'Sec-Fetch-Site': 'none',
-    'Sec-Fetch-User': '?1',
-  };
-  if (p.secChUa) {
-    headers['sec-ch-ua'] = p.secChUa;
-    headers['sec-ch-ua-mobile'] = '?0';
-    headers['sec-ch-ua-platform'] = p.secChUaPlatform;
-  }
-  return headers;
-}
-
-export function mergeHeaders(
-  base: Record<string, string>,
-  override: Record<string, string>
-): Record<string, string> {
-  return { ...base, ...override };
-}