@wenathlan/saddle 1.8.4 → 1.8.6

package/scrape/extract.ts

@@ -0,0 +1,165 @@
+import * as cheerio from 'cheerio';
+import type { ExtractOptions, LinkInfo, ImageInfo, TableInfo, PageMetadata } from './types.js';
+import { normalizeUrl, isInternalUrl } from './utils.js';
+
+export interface ExtractedContent {
+  content: string;
+  text: string;
+  links: LinkInfo[];
+  images: ImageInfo[];
+  tables: TableInfo[];
+  metadata: PageMetadata;
+  jsonLd: unknown[];
+}
+
+function extractReadableContent($: cheerio.CheerioAPI): string {
+  const articleSelectors = [
+    'article', '[role="main"]', 'main', '.post-content', '.article-content',
+    '.entry-content', '.content', '#content', '.prose', '.markdown-body',
+  ];
+  for (const sel of articleSelectors) {
+    const el = $(sel).first();
+    if (el.length) return el.text().trim();
+  }
+  return $('body').text().trim();
+}
+
+function extractMetadata($: cheerio.CheerioAPI): PageMetadata {
+  const getMeta = (name: string): string | undefined => {
+    return $(`meta[name="${name}"], meta[property="${name}"]`).attr('content') || undefined;
+  };
+  return {
+    title: $('title').text() || getMeta('og:title') || '',
+    description: getMeta('description') || getMeta('og:description'),
+    favicon: $('link[rel="icon"]').attr('href') || $('link[rel="shortcut icon"]').attr('href'),
+    charset: $('meta[charset]').attr('charset') || $('meta[http-equiv="Content-Type"]').attr('content'),
+    language: $('html').attr('lang') || undefined,
+    author: getMeta('author'),
+    publishedDate: getMeta('article:published_time') || getMeta('date'),
+    ogImage: getMeta('og:image'),
+    ogType: getMeta('og:type'),
+    keywords: getMeta('keywords')?.split(',').map(k => k.trim()),
+  };
+}
+
+function extractJsonLd($: cheerio.CheerioAPI): unknown[] {
+  const results: unknown[] = [];
+  $('script[type="application/ld+json"]').each((_, el) => {
+    try {
+      const data = JSON.parse($(el).html() || '');
+      if (data['@graph']) {
+        results.push(...(Array.isArray(data['@graph']) ? data['@graph'] : [data['@graph']]));
+      } else {
+        results.push(data);
+      }
+    } catch {}
+  });
+  return results;
+}
+
+function extractLinks($: cheerio.CheerioAPI, baseUrl: string): LinkInfo[] {
+  const links: LinkInfo[] = [];
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href') || '';
+    const text = $(el).text().trim().slice(0, 200);
+    if (!href || href.startsWith('#') || href.startsWith('javascript:')) return;
+    const fullUrl = normalizeUrl(href, baseUrl);
+    links.push({
+      href: fullUrl,
+      text: text || fullUrl,
+      isInternal: isInternalUrl(fullUrl, baseUrl),
+      isExternal: !isInternalUrl(fullUrl, baseUrl),
+    });
+  });
+  return links;
+}
+
+function extractImages($: cheerio.CheerioAPI, baseUrl: string): ImageInfo[] {
+  const images: ImageInfo[] = [];
+  $('img[src]').each((_, el) => {
+    const src = normalizeUrl($(el).attr('src') || '', baseUrl);
+    const alt = $(el).attr('alt') || '';
+    const width = parseInt($(el).attr('width') || '') || undefined;
+    const height = parseInt($(el).attr('height') || '') || undefined;
+    if (src) images.push({ src, alt: alt.slice(0, 200), width, height });
+  });
+  return images;
+}
+
+function extractTables($: cheerio.CheerioAPI): TableInfo[] {
+  const tables: TableInfo[] = [];
+  $('table').each((_, table) => {
+    const headers: string[] = [];
+    const rows: string[][] = [];
+    $('thead tr th, thead tr td', table).each((_, th) => {
+      headers.push($(th).text().trim());
+    });
+    $('tbody tr, > tr', table).each((_, tr) => {
+      if ($(tr).parent().is('thead')) return;
+      const row: string[] = [];
+      $('td, th', tr).each((_, td) => {
+        row.push($(td).text().trim());
+      });
+      if (row.length) rows.push(row);
+    });
+    const caption = $('caption', table).text().trim() || undefined;
+    if (rows.length) tables.push({ headers, rows, caption });
+  });
+  return tables;
+}
+
+function htmlToText(html: string): string {
+  return html
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/p>/gi, '\n\n')
+    .replace(/<\/div>/gi, '\n')
+    .replace(/<\/h[1-6]>/gi, '\n')
+    .replace(/<\/li>/gi, '\n')
+    .replace(/<[^>]+>/g, '')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+export async function extractContent(html: string, options: ExtractOptions = {}): Promise<ExtractedContent> {
+  const $ = cheerio.load(html);
+  const baseUrl = $('base').attr('href') || '';
+
+  if (options.removeSelectors?.length) {
+    for (const sel of options.removeSelectors) {
+      $(sel).remove();
+    }
+  }
+
+  const metadata = extractMetadata($);
+  const jsonLd = extractJsonLd($);
+  const links = options.preserveLinks !== false ? extractLinks($, baseUrl) : [];
+  const images = options.preserveImages !== false ? extractImages($, baseUrl) : [];
+  const tables = options.preserveTables !== false ? extractTables($) : [];
+
+  const text = options.readable
+    ? extractReadableContent($)
+    : $('body').text().trim();
+
+  const content = htmlToText(text);
+
+  return {
+    content: options.maxLength ? content.slice(0, options.maxLength) : content,
+    text,
+    links,
+    images,
+    tables,
+    metadata,
+    jsonLd,
+  };
+}
+
+export async function extractReadable(html: string): Promise<string> {
+  const result = await extractContent(html, { readable: true });
+  return result.content;
+}

package/scrape/fetch.ts

@@ -0,0 +1,105 @@
+import * as cheerio from 'cheerio';
+import type { ScrapeOptions } from './types.js';
+import { getHeaders, getRandomProfile } from './headers.js';
+import { WebScrapeError, ErrorCode } from './errors.js';
+
+export interface FetchResult {
+  html: string;
+  url: string;
+  status: number;
+  headers: Record<string, string>;
+  duration: number;
+}
+
+export async function fetchHtml(url: string, options: {
+  timeout?: number;
+  headers?: Record<string, string>;
+  userAgent?: string;
+  proxy?: string;
+  signal?: AbortSignal;
+} = {}): Promise<FetchResult> {
+  const startTime = Date.now();
+  const profile = getRandomProfile();
+  const baseHeaders = getHeaders(profile);
+  const headers = { ...baseHeaders, ...options.headers };
+  if (options.userAgent) headers['User-Agent'] = options.userAgent;
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), options.timeout || 30000);
+  const signal = options.signal ? AbortSignal.any([controller.signal, options.signal]) : controller.signal;
+
+  try {
+    const response = await fetch(url, {
+      method: 'GET',
+      headers,
+      signal,
+      redirect: 'follow',
+    });
+
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      throw new WebScrapeError(
+        `HTTP ${response.status}: ${response.statusText}`,
+        response.status === 403 ? ErrorCode.BLOCKED :
+        response.status === 429 ? ErrorCode.RATE_LIMITED :
+        response.status >= 500 ? ErrorCode.NETWORK_ERROR :
+        ErrorCode.NETWORK_ERROR,
+        response.status,
+        [408, 429, 500, 502, 503, 504].includes(response.status),
+        { url, status: response.status }
+      );
+    }
+
+    const html = await response.text();
+    const responseHeaders: Record<string, string> = {};
+    response.headers.forEach((value, key) => { responseHeaders[key] = value; });
+
+    return {
+      html,
+      url: response.url || url,
+      status: response.status,
+      headers: responseHeaders,
+      duration: Date.now() - startTime,
+    };
+  } catch (error) {
+    clearTimeout(timeoutId);
+    if (error instanceof WebScrapeError) throw error;
+    if ((error as Error).name === 'AbortError') {
+      throw new WebScrapeError(
+        `Request timeout after ${options.timeout || 30000}ms`,
+        ErrorCode.TIMEOUT,
+        504,
+        true,
+        { url, timeout: options.timeout || 30000 }
+      );
+    }
+    throw new WebScrapeError(
+      `Network error: ${(error as Error).message}`,
+      ErrorCode.NETWORK_ERROR,
+      503,
+      true,
+      { url },
+      error as Error
+    );
+  }
+}
+
+export async function fetchAndParse(url: string, options: Partial<ScrapeOptions> = {}): Promise<FetchResult> {
+  return fetchHtml(url, {
+    timeout: options.timeout,
+    headers: options.headers,
+    userAgent: options.userAgent,
+    proxy: options.proxy,
+  });
+}
+
+export function detectRenderingMode(html: string): 'static' | 'spa' | 'hydrated' {
+  if (/<div id=["'](?:root|app|__next|__nuxt)["']>\s*<\/div>/.test(html)) return 'spa';
+  if (html.includes('__NEXT_DATA__') || html.includes('__NUXT__') || html.includes('__VUE_SSR_DATA__')) return 'hydrated';
+  const $ = cheerio.load(html);
+  if ($('script[type="application/ld+json"]').length > 0) return 'static';
+  const textContent = $.text().replace(/\s+/g, ' ').trim();
+  if (textContent.length > 500) return 'static';
+  return 'spa';
+}

package/scrape/formats.ts

@@ -0,0 +1,85 @@
+import type { ScrapeFormat, SerializeTarget, SerializeOptions, ScrapeResult } from './types.js';
+import { serializeResult as sr } from './serialize.js';
+
+const FORMAT_ALIASES: Record<string, ScrapeFormat> = {
+  md: 'markdown',
+  txt: 'text',
+  plain: 'text',
+  xml: 'xml',
+  json: 'json',
+  redis: 'redis',
+  html: 'html',
+  h: 'html',
+};
+
+const FORMAT_EXTENSIONS: Record<string, ScrapeFormat> = {
+  '.md': 'markdown',
+  '.markdown': 'markdown',
+  '.html': 'html',
+  '.htm': 'html',
+  '.txt': 'text',
+  '.json': 'json',
+  '.xml': 'xml',
+  '.redis': 'redis',
+};
+
+export function resolveFormat(format: string): ScrapeFormat {
+  const lower = format.toLowerCase().trim();
+  return FORMAT_ALIASES[lower] || (lower as ScrapeFormat);
+}
+
+export function formatFromExtension(ext: string): ScrapeFormat | null {
+  return FORMAT_EXTENSIONS[ext.toLowerCase()] ?? null;
+}
+
+export function extensionForFormat(format: ScrapeFormat): string {
+  const map: Record<ScrapeFormat, string> = {
+    markdown: '.md',
+    html: '.html',
+    text: '.txt',
+    json: '.json',
+    xml: '.xml',
+    redis: '.redis',
+  };
+  return map[format] || '.md';
+}
+
+export function detectContentType(html: string): 'article' | 'list' | 'page' | 'other' {
+  if (html.includes('<article') || html.includes('entry-content') || html.includes('class="post-content"') || html.includes('class="article-content"') || html.includes('role="main"')) {
+    return 'article';
+  }
+  const listCount = (html.match(/<li>/gi) || []).length;
+  if (listCount > 20) return 'list';
+  if (html.includes('<article') || html.includes('entry-content')) return 'article';
+  return 'page';
+}
+
+export function buildSerializeOptions(format: ScrapeFormat, pretty = true): SerializeOptions {
+  const targetMap: Record<ScrapeFormat, SerializeTarget> = {
+    markdown: 'markdown',
+    html: 'markdown',
+    text: 'text',
+    json: 'json',
+    xml: 'xml',
+    redis: 'redis',
+  };
+  return {
+    format: targetMap[format] || 'markdown',
+    pretty,
+    includeMetadata: true,
+  };
+}
+
+export async function convertResult(result: ScrapeResult, targetFormat: ScrapeFormat): Promise<ScrapeResult> {
+  if (result.format === targetFormat) return result;
+
+  const opts = buildSerializeOptions(targetFormat);
+  const serialized = sr(result, opts);
+
+  return {
+    ...result,
+    content: serialized.content,
+    format: targetFormat,
+    size: serialized.size,
+  };
+}

package/scrape/headers.ts

@@ -0,0 +1,71 @@
+export interface HeaderProfile {
+  userAgent: string;
+  accept: string;
+  acceptLanguage: string;
+  secChUa: string;
+  secChUaPlatform: string;
+}
+
+const CHROME_PROFILES: HeaderProfile[] = [
+  {
+    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+    acceptLanguage: 'en-US,en;q=0.9',
+    secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
+    secChUaPlatform: '"Windows"',
+  },
+  {
+    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+    acceptLanguage: 'en-US,en;q=0.9',
+    secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
+    secChUaPlatform: '"macOS"',
+  },
+  {
+    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    acceptLanguage: 'en-US,en;q=0.5',
+    secChUa: '',
+    secChUaPlatform: '"Windows"',
+  },
+  {
+    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    acceptLanguage: 'en-US,en;q=0.9',
+    secChUa: '',
+    secChUaPlatform: '"macOS"',
+  },
+];
+
+export function getRandomProfile(): HeaderProfile {
+  return CHROME_PROFILES[Math.floor(Math.random() * CHROME_PROFILES.length)];
+}
+
+export function getHeaders(profile?: HeaderProfile): Record<string, string> {
+  const p = profile || getRandomProfile();
+  const headers: Record<string, string> = {
+    'User-Agent': p.userAgent,
+    'Accept': p.accept,
+    'Accept-Language': p.acceptLanguage,
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+  };
+  if (p.secChUa) {
+    headers['sec-ch-ua'] = p.secChUa;
+    headers['sec-ch-ua-mobile'] = '?0';
+    headers['sec-ch-ua-platform'] = p.secChUaPlatform;
+  }
+  return headers;
+}
+
+export function mergeHeaders(
+  base: Record<string, string>,
+  override: Record<string, string>
+): Record<string, string> {
+  return { ...base, ...override };
+}

package/scrape/index.ts

@@ -0,0 +1,92 @@
+// ─── Core Scraping ───
+export { AgentBrowser, createBrowser } from './browser.js';
+export { scrapeUrl, scrapeHtml, scrapeWithBrowser } from './scrape.js';
+export { extractContent, extractReadable } from './extract.js';
+export type { ExtractedContent } from './extract.js';
+export { fetchHtml, fetchAndParse, detectRenderingMode } from './fetch.js';
+export type { FetchResult } from './fetch.js';
+
+// ─── Serialization ───
+export { serializeResult, serializeHtml } from './serialize.js';
+export { resolveFormat, formatFromExtension, extensionForFormat, buildSerializeOptions, convertResult } from './formats.js';
+
+// ─── Renderer ───
+export { PygameRenderer, createRenderer } from './renderer.js';
+
+// ─── AI/LLM ───
+export { formatForAgent, buildContext } from './agent.js';
+export { estimateTokens, countTokens, fitsInContext, truncateToTokens, tokenCost } from './tokens.js';
+export type { ModelType } from './tokens.js';
+export { chunkMarkdown, chunkText, formatChunksForRAG } from './chunking.js';
+export type { Chunk, ChunkOptions } from './chunking.js';
+export { generateLlmsTxt, generateLlmsFullTxt } from './llms-txt.js';
+export type { LlmsTxtOptions } from './llms-txt.js';
+
+// ─── Infrastructure ───
+export { WebScrapeError, ValidationError, TimeoutError, BlockedError, RateLimitError, ProxyError, ParseError, AuthError, NetworkError, BrowserNotAvailableError } from './errors.js';
+export type { ErrorCode } from './errors.js';
+export { createEventEmitter, globalEmitter } from './events.js';
+export type { WebScrapeEvents, EventEmitter } from './events.js';
+export { withRetry, isRetryableError, AbortError } from './retry.js';
+export { RateLimiter, createRateLimiter } from './rate-limiter.js';
+export type { RateLimiterConfig } from './rate-limiter.js';
+export { ProxyPool, createProxyPool } from './proxy.js';
+export type { ProxyPoolConfig, ProxyRotationStrategy } from './proxy.js';
+export { CookieJar, ScrapingSession, createSession } from './session.js';
+export type { Cookie } from './session.js';
+export { getRandomProfile, getHeaders, mergeHeaders } from './headers.js';
+export type { HeaderProfile } from './headers.js';
+export { WebScrapeCache, createCache } from './cache.js';
+export type { CacheConfig, CacheEntry } from './cache.js';
+export { MiddlewarePipeline, createPipeline, loggingMiddleware, timeoutMiddleware, retryMiddleware } from './middleware.js';
+export type { Middleware, MiddlewareContext, MiddlewareNext } from './middleware.js';
+
+// ─── Crawling ───
+export { crawl } from './crawler.js';
+export type { CrawlOptions } from './crawler.js';
+export { fetchSitemap, discoverSitemaps, parseSitemap } from './sitemap.js';
+export type { SitemapUrl, SitemapResult } from './sitemap.js';
+export { fetchRobotsTxt, isAllowed, getCrawlDelay, getSitemaps } from './robots.js';
+export type { RobotsTxt, RobotsDirective } from './robots.js';
+
+// ─── Batch & Pool ───
+export { batchScrape, batchScrapeSequential } from './batch.js';
+export type { BatchResult } from './batch.js';
+export { BrowserPool, createPool } from './pool.js';
+export type { PoolConfig } from './pool.js';
+
+// ─── Server ───
+export { createServer, app } from './server.js';
+export type { ServerConfig } from './server.js';
+
+// ─── Utilities ───
+export { ensureDir, writeOutput, slugify, truncate, chunkText as chunkTextUtil, delay, isValidUrl, normalizeUrl, isInternalUrl } from './utils.js';
+export { randomPort, resetPort } from './utils/port.js';
+
+// ─── Types ───
+export {
+  ScrapeOptionsSchema,
+  BrowserAgentConfigSchema,
+} from './types.js';
+export type {
+  ScrapeFormat,
+  SerializeTarget,
+  ScrapeOptions,
+  ScrapeResult,
+  PageMetadata,
+  LinkInfo,
+  ImageInfo,
+  TableInfo,
+  BrowserAgentConfig,
+  ExtractOptions,
+  SerializeOptions,
+  SerializedOutput,
+  AgentOutput,
+  RenderOptions,
+  ChromeCommand,
+  CliOptions,
+  RetryConfig,
+  ProxyConfig,
+  CrawlStats,
+  BatchOptions,
+} from './types.js';

package/scrape/jsdom.d.ts

@@ -0,0 +1,6 @@
+declare module 'jsdom' {
+  export class JSDOM {
+    constructor(html?: string, options?: Record<string, unknown>);
+    window: Window;
+  }
+}

package/scrape/llms-txt.ts

@@ -0,0 +1,84 @@
+import type { ScrapeResult } from './types.js';
+
+export interface LlmsTxtOptions {
+  siteName?: string;
+  description?: string;
+  includeOptional?: boolean;
+}
+
+export function generateLlmsTxt(
+  results: ScrapeResult[],
+  options: LlmsTxtOptions = {}
+): string {
+  const siteName = options.siteName || results[0]?.metadata?.title || 'Site';
+  const description = options.description || results[0]?.metadata?.description || '';
+
+  const lines: string[] = [];
+  lines.push(`# ${siteName}`);
+  lines.push('');
+  if (description) {
+    lines.push(`> ${description}`);
+    lines.push('');
+  }
+
+  const pages: ScrapeResult[] = [];
+  const docs: ScrapeResult[] = [];
+
+  for (const result of results) {
+    const url = result.url.toLowerCase();
+    if (url.includes('doc') || url.includes('guide') || url.includes('api')) {
+      docs.push(result);
+    } else {
+      pages.push(result);
+    }
+  }
+
+  if (pages.length > 0) {
+    lines.push('## Pages');
+    lines.push('');
+    for (const page of pages.slice(0, 20)) {
+      const title = page.metadata.title || page.title;
+      const desc = page.metadata.description || '';
+      lines.push(`- ${title}: ${desc || page.url}`);
+    }
+    lines.push('');
+  }
+
+  if (docs.length > 0) {
+    lines.push('## Documentation');
+    lines.push('');
+    for (const doc of docs.slice(0, 20)) {
+      const title = doc.metadata.title || doc.title;
+      const desc = doc.metadata.description || '';
+      lines.push(`- ${title}: ${desc || doc.url}`);
+    }
+    lines.push('');
+  }
+
+  if (options.includeOptional && results.length > 5) {
+    lines.push('## Optional');
+    lines.push('');
+    lines.push(`- Full content: ${results.length} pages scraped`);
+    lines.push(`- Last updated: ${new Date().toISOString().split('T')[0]}`);
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+export function generateLlmsFullTxt(results: ScrapeResult[]): string {
+  const parts: string[] = [];
+
+  for (const result of results) {
+    parts.push(`# ${result.title || result.metadata.title || 'Untitled'}`);
+    parts.push('');
+    parts.push(`Source: ${result.url}`);
+    parts.push('');
+    parts.push(result.content);
+    parts.push('');
+    parts.push('---');
+    parts.push('');
+  }
+
+  return parts.join('\n');
+}