@wenathlan/saddle 1.8.7 → 1.8.9

package/scrape/extract.ts

@@ -1,165 +0,0 @@
-import * as cheerio from 'cheerio';
-import type { ExtractOptions, LinkInfo, ImageInfo, TableInfo, PageMetadata } from './types.js';
-import { normalizeUrl, isInternalUrl } from './utils.js';
-
-export interface ExtractedContent {
-  content: string;
-  text: string;
-  links: LinkInfo[];
-  images: ImageInfo[];
-  tables: TableInfo[];
-  metadata: PageMetadata;
-  jsonLd: unknown[];
-}
-
-function extractReadableContent($: cheerio.CheerioAPI): string {
-  const articleSelectors = [
-    'article', '[role="main"]', 'main', '.post-content', '.article-content',
-    '.entry-content', '.content', '#content', '.prose', '.markdown-body',
-  ];
-  for (const sel of articleSelectors) {
-    const el = $(sel).first();
-    if (el.length) return el.text().trim();
-  }
-  return $('body').text().trim();
-}
-
-function extractMetadata($: cheerio.CheerioAPI): PageMetadata {
-  const getMeta = (name: string): string | undefined => {
-    return $(`meta[name="${name}"], meta[property="${name}"]`).attr('content') || undefined;
-  };
-  return {
-    title: $('title').text() || getMeta('og:title') || '',
-    description: getMeta('description') || getMeta('og:description'),
-    favicon: $('link[rel="icon"]').attr('href') || $('link[rel="shortcut icon"]').attr('href'),
-    charset: $('meta[charset]').attr('charset') || $('meta[http-equiv="Content-Type"]').attr('content'),
-    language: $('html').attr('lang') || undefined,
-    author: getMeta('author'),
-    publishedDate: getMeta('article:published_time') || getMeta('date'),
-    ogImage: getMeta('og:image'),
-    ogType: getMeta('og:type'),
-    keywords: getMeta('keywords')?.split(',').map(k => k.trim()),
-  };
-}
-
-function extractJsonLd($: cheerio.CheerioAPI): unknown[] {
-  const results: unknown[] = [];
-  $('script[type="application/ld+json"]').each((_, el) => {
-    try {
-      const data = JSON.parse($(el).html() || '');
-      if (data['@graph']) {
-        results.push(...(Array.isArray(data['@graph']) ? data['@graph'] : [data['@graph']]));
-      } else {
-        results.push(data);
-      }
-    } catch {}
-  });
-  return results;
-}
-
-function extractLinks($: cheerio.CheerioAPI, baseUrl: string): LinkInfo[] {
-  const links: LinkInfo[] = [];
-  $('a[href]').each((_, el) => {
-    const href = $(el).attr('href') || '';
-    const text = $(el).text().trim().slice(0, 200);
-    if (!href || href.startsWith('#') || href.startsWith('javascript:')) return;
-    const fullUrl = normalizeUrl(href, baseUrl);
-    links.push({
-      href: fullUrl,
-      text: text || fullUrl,
-      isInternal: isInternalUrl(fullUrl, baseUrl),
-      isExternal: !isInternalUrl(fullUrl, baseUrl),
-    });
-  });
-  return links;
-}
-
-function extractImages($: cheerio.CheerioAPI, baseUrl: string): ImageInfo[] {
-  const images: ImageInfo[] = [];
-  $('img[src]').each((_, el) => {
-    const src = normalizeUrl($(el).attr('src') || '', baseUrl);
-    const alt = $(el).attr('alt') || '';
-    const width = parseInt($(el).attr('width') || '') || undefined;
-    const height = parseInt($(el).attr('height') || '') || undefined;
-    if (src) images.push({ src, alt: alt.slice(0, 200), width, height });
-  });
-  return images;
-}
-
-function extractTables($: cheerio.CheerioAPI): TableInfo[] {
-  const tables: TableInfo[] = [];
-  $('table').each((_, table) => {
-    const headers: string[] = [];
-    const rows: string[][] = [];
-    $('thead tr th, thead tr td', table).each((_, th) => {
-      headers.push($(th).text().trim());
-    });
-    $('tbody tr, > tr', table).each((_, tr) => {
-      if ($(tr).parent().is('thead')) return;
-      const row: string[] = [];
-      $('td, th', tr).each((_, td) => {
-        row.push($(td).text().trim());
-      });
-      if (row.length) rows.push(row);
-    });
-    const caption = $('caption', table).text().trim() || undefined;
-    if (rows.length) tables.push({ headers, rows, caption });
-  });
-  return tables;
-}
-
-function htmlToText(html: string): string {
-  return html
-    .replace(/<br\s*\/?>/gi, '\n')
-    .replace(/<\/p>/gi, '\n\n')
-    .replace(/<\/div>/gi, '\n')
-    .replace(/<\/h[1-6]>/gi, '\n')
-    .replace(/<\/li>/gi, '\n')
-    .replace(/<[^>]+>/g, '')
-    .replace(/&nbsp;/g, ' ')
-    .replace(/&amp;/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/\n{3,}/g, '\n\n')
-    .trim();
-}
-
-export async function extractContent(html: string, options: ExtractOptions = {}): Promise<ExtractedContent> {
-  const $ = cheerio.load(html);
-  const baseUrl = $('base').attr('href') || '';
-
-  if (options.removeSelectors?.length) {
-    for (const sel of options.removeSelectors) {
-      $(sel).remove();
-    }
-  }
-
-  const metadata = extractMetadata($);
-  const jsonLd = extractJsonLd($);
-  const links = options.preserveLinks !== false ? extractLinks($, baseUrl) : [];
-  const images = options.preserveImages !== false ? extractImages($, baseUrl) : [];
-  const tables = options.preserveTables !== false ? extractTables($) : [];
-
-  const text = options.readable
-    ? extractReadableContent($)
-    : $('body').text().trim();
-
-  const content = htmlToText(text);
-
-  return {
-    content: options.maxLength ? content.slice(0, options.maxLength) : content,
-    text,
-    links,
-    images,
-    tables,
-    metadata,
-    jsonLd,
-  };
-}
-
-export async function extractReadable(html: string): Promise<string> {
-  const result = await extractContent(html, { readable: true });
-  return result.content;
-}

package/scrape/fetch.ts

@@ -1,105 +0,0 @@
-import * as cheerio from 'cheerio';
-import type { ScrapeOptions } from './types.js';
-import { getHeaders, getRandomProfile } from './headers.js';
-import { WebScrapeError, ErrorCode } from './errors.js';
-
-export interface FetchResult {
-  html: string;
-  url: string;
-  status: number;
-  headers: Record<string, string>;
-  duration: number;
-}
-
-export async function fetchHtml(url: string, options: {
-  timeout?: number;
-  headers?: Record<string, string>;
-  userAgent?: string;
-  proxy?: string;
-  signal?: AbortSignal;
-} = {}): Promise<FetchResult> {
-  const startTime = Date.now();
-  const profile = getRandomProfile();
-  const baseHeaders = getHeaders(profile);
-  const headers = { ...baseHeaders, ...options.headers };
-  if (options.userAgent) headers['User-Agent'] = options.userAgent;
-
-  const controller = new AbortController();
-  const timeoutId = setTimeout(() => controller.abort(), options.timeout || 30000);
-  const signal = options.signal ? AbortSignal.any([controller.signal, options.signal]) : controller.signal;
-
-  try {
-    const response = await fetch(url, {
-      method: 'GET',
-      headers,
-      signal,
-      redirect: 'follow',
-    });
-
-    clearTimeout(timeoutId);
-
-    if (!response.ok) {
-      throw new WebScrapeError(
-        `HTTP ${response.status}: ${response.statusText}`,
-        response.status === 403 ? ErrorCode.BLOCKED :
-        response.status === 429 ? ErrorCode.RATE_LIMITED :
-        response.status >= 500 ? ErrorCode.NETWORK_ERROR :
-        ErrorCode.NETWORK_ERROR,
-        response.status,
-        [408, 429, 500, 502, 503, 504].includes(response.status),
-        { url, status: response.status }
-      );
-    }
-
-    const html = await response.text();
-    const responseHeaders: Record<string, string> = {};
-    response.headers.forEach((value, key) => { responseHeaders[key] = value; });
-
-    return {
-      html,
-      url: response.url || url,
-      status: response.status,
-      headers: responseHeaders,
-      duration: Date.now() - startTime,
-    };
-  } catch (error) {
-    clearTimeout(timeoutId);
-    if (error instanceof WebScrapeError) throw error;
-    if ((error as Error).name === 'AbortError') {
-      throw new WebScrapeError(
-        `Request timeout after ${options.timeout || 30000}ms`,
-        ErrorCode.TIMEOUT,
-        504,
-        true,
-        { url, timeout: options.timeout || 30000 }
-      );
-    }
-    throw new WebScrapeError(
-      `Network error: ${(error as Error).message}`,
-      ErrorCode.NETWORK_ERROR,
-      503,
-      true,
-      { url },
-      error as Error
-    );
-  }
-}
-
-export async function fetchAndParse(url: string, options: Partial<ScrapeOptions> = {}): Promise<FetchResult> {
-  return fetchHtml(url, {
-    timeout: options.timeout,
-    headers: options.headers,
-    userAgent: options.userAgent,
-    proxy: options.proxy,
-  });
-}
-
-export function detectRenderingMode(html: string): 'static' | 'spa' | 'hydrated' {
-  if (/<div id=["'](?:root|app|__next|__nuxt)["']>\s*<\/div>/.test(html)) return 'spa';
-  if (html.includes('__NEXT_DATA__') || html.includes('__NUXT__') || html.includes('__VUE_SSR_DATA__')) return 'hydrated';
-  const $ = cheerio.load(html);
-  if ($('script[type="application/ld+json"]').length > 0) return 'static';
-  const textContent = $.text().replace(/\s+/g, ' ').trim();
-  if (textContent.length > 500) return 'static';
-  return 'spa';
-}

package/scrape/formats.ts

@@ -1,85 +0,0 @@
-import type { ScrapeFormat, SerializeTarget, SerializeOptions, ScrapeResult } from './types.js';
-import { serializeResult as sr } from './serialize.js';
-
-const FORMAT_ALIASES: Record<string, ScrapeFormat> = {
-  md: 'markdown',
-  txt: 'text',
-  plain: 'text',
-  xml: 'xml',
-  json: 'json',
-  redis: 'redis',
-  html: 'html',
-  h: 'html',
-};
-
-const FORMAT_EXTENSIONS: Record<string, ScrapeFormat> = {
-  '.md': 'markdown',
-  '.markdown': 'markdown',
-  '.html': 'html',
-  '.htm': 'html',
-  '.txt': 'text',
-  '.json': 'json',
-  '.xml': 'xml',
-  '.redis': 'redis',
-};
-
-export function resolveFormat(format: string): ScrapeFormat {
-  const lower = format.toLowerCase().trim();
-  return FORMAT_ALIASES[lower] || (lower as ScrapeFormat);
-}
-
-export function formatFromExtension(ext: string): ScrapeFormat | null {
-  return FORMAT_EXTENSIONS[ext.toLowerCase()] ?? null;
-}
-
-export function extensionForFormat(format: ScrapeFormat): string {
-  const map: Record<ScrapeFormat, string> = {
-    markdown: '.md',
-    html: '.html',
-    text: '.txt',
-    json: '.json',
-    xml: '.xml',
-    redis: '.redis',
-  };
-  return map[format] || '.md';
-}
-
-export function detectContentType(html: string): 'article' | 'list' | 'page' | 'other' {
-  if (html.includes('<article') || html.includes('entry-content') || html.includes('class="post-content"') || html.includes('class="article-content"') || html.includes('role="main"')) {
-    return 'article';
-  }
-  const listCount = (html.match(/<li>/gi) || []).length;
-  if (listCount > 20) return 'list';
-  if (html.includes('<article') || html.includes('entry-content')) return 'article';
-  return 'page';
-}
-
-export function buildSerializeOptions(format: ScrapeFormat, pretty = true): SerializeOptions {
-  const targetMap: Record<ScrapeFormat, SerializeTarget> = {
-    markdown: 'markdown',
-    html: 'markdown',
-    text: 'text',
-    json: 'json',
-    xml: 'xml',
-    redis: 'redis',
-  };
-  return {
-    format: targetMap[format] || 'markdown',
-    pretty,
-    includeMetadata: true,
-  };
-}
-
-export async function convertResult(result: ScrapeResult, targetFormat: ScrapeFormat): Promise<ScrapeResult> {
-  if (result.format === targetFormat) return result;
-
-  const opts = buildSerializeOptions(targetFormat);
-  const serialized = sr(result, opts);
-
-  return {
-    ...result,
-    content: serialized.content,
-    format: targetFormat,
-    size: serialized.size,
-  };
-}

package/scrape/headers.ts

@@ -1,71 +0,0 @@
-export interface HeaderProfile {
-  userAgent: string;
-  accept: string;
-  acceptLanguage: string;
-  secChUa: string;
-  secChUaPlatform: string;
-}
-
-const CHROME_PROFILES: HeaderProfile[] = [
-  {
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
-    secChUaPlatform: '"Windows"',
-  },
-  {
-    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
-    secChUaPlatform: '"macOS"',
-  },
-  {
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-    acceptLanguage: 'en-US,en;q=0.5',
-    secChUa: '',
-    secChUaPlatform: '"Windows"',
-  },
-  {
-    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
-    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    acceptLanguage: 'en-US,en;q=0.9',
-    secChUa: '',
-    secChUaPlatform: '"macOS"',
-  },
-];
-
-export function getRandomProfile(): HeaderProfile {
-  return CHROME_PROFILES[Math.floor(Math.random() * CHROME_PROFILES.length)];
-}
-
-export function getHeaders(profile?: HeaderProfile): Record<string, string> {
-  const p = profile || getRandomProfile();
-  const headers: Record<string, string> = {
-    'User-Agent': p.userAgent,
-    'Accept': p.accept,
-    'Accept-Language': p.acceptLanguage,
-    'Accept-Encoding': 'gzip, deflate, br',
-    'Connection': 'keep-alive',
-    'Upgrade-Insecure-Requests': '1',
-    'Sec-Fetch-Dest': 'document',
-    'Sec-Fetch-Mode': 'navigate',
-    'Sec-Fetch-Site': 'none',
-    'Sec-Fetch-User': '?1',
-  };
-  if (p.secChUa) {
-    headers['sec-ch-ua'] = p.secChUa;
-    headers['sec-ch-ua-mobile'] = '?0';
-    headers['sec-ch-ua-platform'] = p.secChUaPlatform;
-  }
-  return headers;
-}
-
-export function mergeHeaders(
-  base: Record<string, string>,
-  override: Record<string, string>
-): Record<string, string> {
-  return { ...base, ...override };
-}

package/scrape/index.ts

@@ -1,92 +0,0 @@
-// ─── Core Scraping ───
-export { AgentBrowser, createBrowser } from './browser.js';
-export { scrapeUrl, scrapeHtml, scrapeWithBrowser } from './scrape.js';
-export { extractContent, extractReadable } from './extract.js';
-export type { ExtractedContent } from './extract.js';
-export { fetchHtml, fetchAndParse, detectRenderingMode } from './fetch.js';
-export type { FetchResult } from './fetch.js';
-
-// ─── Serialization ───
-export { serializeResult, serializeHtml } from './serialize.js';
-export { resolveFormat, formatFromExtension, extensionForFormat, buildSerializeOptions, convertResult } from './formats.js';
-
-// ─── Renderer ───
-export { PygameRenderer, createRenderer } from './renderer.js';
-
-// ─── AI/LLM ───
-export { formatForAgent, buildContext } from './agent.js';
-export { estimateTokens, countTokens, fitsInContext, truncateToTokens, tokenCost } from './tokens.js';
-export type { ModelType } from './tokens.js';
-export { chunkMarkdown, chunkText, formatChunksForRAG } from './chunking.js';
-export type { Chunk, ChunkOptions } from './chunking.js';
-export { generateLlmsTxt, generateLlmsFullTxt } from './llms-txt.js';
-export type { LlmsTxtOptions } from './llms-txt.js';
-
-// ─── Infrastructure ───
-export { WebScrapeError, ValidationError, TimeoutError, BlockedError, RateLimitError, ProxyError, ParseError, AuthError, NetworkError, BrowserNotAvailableError } from './errors.js';
-export type { ErrorCode } from './errors.js';
-export { createEventEmitter, globalEmitter } from './events.js';
-export type { WebScrapeEvents, EventEmitter } from './events.js';
-export { withRetry, isRetryableError, AbortError } from './retry.js';
-export { RateLimiter, createRateLimiter } from './rate-limiter.js';
-export type { RateLimiterConfig } from './rate-limiter.js';
-export { ProxyPool, createProxyPool } from './proxy.js';
-export type { ProxyPoolConfig, ProxyRotationStrategy } from './proxy.js';
-export { CookieJar, ScrapingSession, createSession } from './session.js';
-export type { Cookie } from './session.js';
-export { getRandomProfile, getHeaders, mergeHeaders } from './headers.js';
-export type { HeaderProfile } from './headers.js';
-export { WebScrapeCache, createCache } from './cache.js';
-export type { CacheConfig, CacheEntry } from './cache.js';
-export { MiddlewarePipeline, createPipeline, loggingMiddleware, timeoutMiddleware, retryMiddleware } from './middleware.js';
-export type { Middleware, MiddlewareContext, MiddlewareNext } from './middleware.js';
-
-// ─── Crawling ───
-export { crawl } from './crawler.js';
-export type { CrawlOptions } from './crawler.js';
-export { fetchSitemap, discoverSitemaps, parseSitemap } from './sitemap.js';
-export type { SitemapUrl, SitemapResult } from './sitemap.js';
-export { fetchRobotsTxt, isAllowed, getCrawlDelay, getSitemaps } from './robots.js';
-export type { RobotsTxt, RobotsDirective } from './robots.js';
-
-// ─── Batch & Pool ───
-export { batchScrape, batchScrapeSequential } from './batch.js';
-export type { BatchResult } from './batch.js';
-export { BrowserPool, createPool } from './pool.js';
-export type { PoolConfig } from './pool.js';
-
-// ─── Server ───
-export { createServer, app } from './server.js';
-export type { ServerConfig } from './server.js';
-
-// ─── Utilities ───
-export { ensureDir, writeOutput, slugify, truncate, chunkText as chunkTextUtil, delay, isValidUrl, normalizeUrl, isInternalUrl } from './utils.js';
-export { randomPort, resetPort } from './utils/port.js';
-
-// ─── Types ───
-export {
-  ScrapeOptionsSchema,
-  BrowserAgentConfigSchema,
-} from './types.js';
-export type {
-  ScrapeFormat,
-  SerializeTarget,
-  ScrapeOptions,
-  ScrapeResult,
-  PageMetadata,
-  LinkInfo,
-  ImageInfo,
-  TableInfo,
-  BrowserAgentConfig,
-  ExtractOptions,
-  SerializeOptions,
-  SerializedOutput,
-  AgentOutput,
-  RenderOptions,
-  ChromeCommand,
-  CliOptions,
-  RetryConfig,
-  ProxyConfig,
-  CrawlStats,
-  BatchOptions,
-} from './types.js';

package/scrape/jsdom.d.ts

@@ -1,6 +0,0 @@
-declare module 'jsdom' {
-  export class JSDOM {
-    constructor(html?: string, options?: Record<string, unknown>);
-    window: Window;
-  }
-}

package/scrape/llms-txt.ts

@@ -1,84 +0,0 @@
-import type { ScrapeResult } from './types.js';
-
-export interface LlmsTxtOptions {
-  siteName?: string;
-  description?: string;
-  includeOptional?: boolean;
-}
-
-export function generateLlmsTxt(
-  results: ScrapeResult[],
-  options: LlmsTxtOptions = {}
-): string {
-  const siteName = options.siteName || results[0]?.metadata?.title || 'Site';
-  const description = options.description || results[0]?.metadata?.description || '';
-
-  const lines: string[] = [];
-  lines.push(`# ${siteName}`);
-  lines.push('');
-  if (description) {
-    lines.push(`> ${description}`);
-    lines.push('');
-  }
-
-  const pages: ScrapeResult[] = [];
-  const docs: ScrapeResult[] = [];
-
-  for (const result of results) {
-    const url = result.url.toLowerCase();
-    if (url.includes('doc') || url.includes('guide') || url.includes('api')) {
-      docs.push(result);
-    } else {
-      pages.push(result);
-    }
-  }
-
-  if (pages.length > 0) {
-    lines.push('## Pages');
-    lines.push('');
-    for (const page of pages.slice(0, 20)) {
-      const title = page.metadata.title || page.title;
-      const desc = page.metadata.description || '';
-      lines.push(`- ${title}: ${desc || page.url}`);
-    }
-    lines.push('');
-  }
-
-  if (docs.length > 0) {
-    lines.push('## Documentation');
-    lines.push('');
-    for (const doc of docs.slice(0, 20)) {
-      const title = doc.metadata.title || doc.title;
-      const desc = doc.metadata.description || '';
-      lines.push(`- ${title}: ${desc || doc.url}`);
-    }
-    lines.push('');
-  }
-
-  if (options.includeOptional && results.length > 5) {
-    lines.push('## Optional');
-    lines.push('');
-    lines.push(`- Full content: ${results.length} pages scraped`);
-    lines.push(`- Last updated: ${new Date().toISOString().split('T')[0]}`);
-    lines.push('');
-  }
-
-  return lines.join('\n');
-}
-
-export function generateLlmsFullTxt(results: ScrapeResult[]): string {
-  const parts: string[] = [];
-
-  for (const result of results) {
-    parts.push(`# ${result.title || result.metadata.title || 'Untitled'}`);
-    parts.push('');
-    parts.push(`Source: ${result.url}`);
-    parts.push('');
-    parts.push(result.content);
-    parts.push('');
-    parts.push('---');
-    parts.push('');
-  }
-
-  return parts.join('\n');
-}