@wenathlan/saddle 1.8.2 → 1.8.5

package/scrape/formats.ts

@@ -0,0 +1,85 @@
+import type { ScrapeFormat, SerializeTarget, SerializeOptions, ScrapeResult } from './types.js';
+import { serializeResult as sr } from './serialize.js';
+
+const FORMAT_ALIASES: Record<string, ScrapeFormat> = {
+  md: 'markdown',
+  txt: 'text',
+  plain: 'text',
+  xml: 'xml',
+  json: 'json',
+  redis: 'redis',
+  html: 'html',
+  h: 'html',
+};
+
+const FORMAT_EXTENSIONS: Record<string, ScrapeFormat> = {
+  '.md': 'markdown',
+  '.markdown': 'markdown',
+  '.html': 'html',
+  '.htm': 'html',
+  '.txt': 'text',
+  '.json': 'json',
+  '.xml': 'xml',
+  '.redis': 'redis',
+};
+
+export function resolveFormat(format: string): ScrapeFormat {
+  const lower = format.toLowerCase().trim();
+  return FORMAT_ALIASES[lower] || (lower as ScrapeFormat);
+}
+
+export function formatFromExtension(ext: string): ScrapeFormat | null {
+  return FORMAT_EXTENSIONS[ext.toLowerCase()] ?? null;
+}
+
+export function extensionForFormat(format: ScrapeFormat): string {
+  const map: Record<ScrapeFormat, string> = {
+    markdown: '.md',
+    html: '.html',
+    text: '.txt',
+    json: '.json',
+    xml: '.xml',
+    redis: '.redis',
+  };
+  return map[format] || '.md';
+}
+
+export function detectContentType(html: string): 'article' | 'list' | 'page' | 'other' {
+  if (html.includes('<article') || html.includes('entry-content') || html.includes('class="post-content"') || html.includes('class="article-content"') || html.includes('role="main"')) {
+    return 'article';
+  }
+  const listCount = (html.match(/<li>/gi) || []).length;
+  if (listCount > 20) return 'list';
+  if (html.includes('<article') || html.includes('entry-content')) return 'article';
+  return 'page';
+}
+
+export function buildSerializeOptions(format: ScrapeFormat, pretty = true): SerializeOptions {
+  const targetMap: Record<ScrapeFormat, SerializeTarget> = {
+    markdown: 'markdown',
+    html: 'markdown',
+    text: 'text',
+    json: 'json',
+    xml: 'xml',
+    redis: 'redis',
+  };
+  return {
+    format: targetMap[format] || 'markdown',
+    pretty,
+    includeMetadata: true,
+  };
+}
+
+export async function convertResult(result: ScrapeResult, targetFormat: ScrapeFormat): Promise<ScrapeResult> {
+  if (result.format === targetFormat) return result;
+
+  const opts = buildSerializeOptions(targetFormat);
+  const serialized = sr(result, opts);
+
+  return {
+    ...result,
+    content: serialized.content,
+    format: targetFormat,
+    size: serialized.size,
+  };
+}

package/scrape/headers.ts

@@ -0,0 +1,71 @@
+export interface HeaderProfile {
+  userAgent: string;
+  accept: string;
+  acceptLanguage: string;
+  secChUa: string;
+  secChUaPlatform: string;
+}
+
+const CHROME_PROFILES: HeaderProfile[] = [
+  {
+    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+    acceptLanguage: 'en-US,en;q=0.9',
+    secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
+    secChUaPlatform: '"Windows"',
+  },
+  {
+    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+    acceptLanguage: 'en-US,en;q=0.9',
+    secChUa: '"Chromium";v="131", "Not_A Brand";v="24"',
+    secChUaPlatform: '"macOS"',
+  },
+  {
+    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    acceptLanguage: 'en-US,en;q=0.5',
+    secChUa: '',
+    secChUaPlatform: '"Windows"',
+  },
+  {
+    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.2 Safari/605.1.15',
+    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    acceptLanguage: 'en-US,en;q=0.9',
+    secChUa: '',
+    secChUaPlatform: '"macOS"',
+  },
+];
+
+export function getRandomProfile(): HeaderProfile {
+  return CHROME_PROFILES[Math.floor(Math.random() * CHROME_PROFILES.length)];
+}
+
+export function getHeaders(profile?: HeaderProfile): Record<string, string> {
+  const p = profile || getRandomProfile();
+  const headers: Record<string, string> = {
+    'User-Agent': p.userAgent,
+    'Accept': p.accept,
+    'Accept-Language': p.acceptLanguage,
+    'Accept-Encoding': 'gzip, deflate, br',
+    'Connection': 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+  };
+  if (p.secChUa) {
+    headers['sec-ch-ua'] = p.secChUa;
+    headers['sec-ch-ua-mobile'] = '?0';
+    headers['sec-ch-ua-platform'] = p.secChUaPlatform;
+  }
+  return headers;
+}
+
+export function mergeHeaders(
+  base: Record<string, string>,
+  override: Record<string, string>
+): Record<string, string> {
+  return { ...base, ...override };
+}

package/scrape/index.ts

@@ -0,0 +1,92 @@
+// ─── Core Scraping ───
+export { AgentBrowser, createBrowser } from './browser.js';
+export { scrapeUrl, scrapeHtml, scrapeWithBrowser } from './scrape.js';
+export { extractContent, extractReadable } from './extract.js';
+export type { ExtractedContent } from './extract.js';
+export { fetchHtml, fetchAndParse, detectRenderingMode } from './fetch.js';
+export type { FetchResult } from './fetch.js';
+
+// ─── Serialization ───
+export { serializeResult, serializeHtml } from './serialize.js';
+export { resolveFormat, formatFromExtension, extensionForFormat, buildSerializeOptions, convertResult } from './formats.js';
+
+// ─── Renderer ───
+export { PygameRenderer, createRenderer } from './renderer.js';
+
+// ─── AI/LLM ───
+export { formatForAgent, buildContext } from './agent.js';
+export { estimateTokens, countTokens, fitsInContext, truncateToTokens, tokenCost } from './tokens.js';
+export type { ModelType } from './tokens.js';
+export { chunkMarkdown, chunkText, formatChunksForRAG } from './chunking.js';
+export type { Chunk, ChunkOptions } from './chunking.js';
+export { generateLlmsTxt, generateLlmsFullTxt } from './llms-txt.js';
+export type { LlmsTxtOptions } from './llms-txt.js';
+
+// ─── Infrastructure ───
+export { WebScrapeError, ValidationError, TimeoutError, BlockedError, RateLimitError, ProxyError, ParseError, AuthError, NetworkError, BrowserNotAvailableError } from './errors.js';
+export type { ErrorCode } from './errors.js';
+export { createEventEmitter, globalEmitter } from './events.js';
+export type { WebScrapeEvents, EventEmitter } from './events.js';
+export { withRetry, isRetryableError, AbortError } from './retry.js';
+export { RateLimiter, createRateLimiter } from './rate-limiter.js';
+export type { RateLimiterConfig } from './rate-limiter.js';
+export { ProxyPool, createProxyPool } from './proxy.js';
+export type { ProxyPoolConfig, ProxyRotationStrategy } from './proxy.js';
+export { CookieJar, ScrapingSession, createSession } from './session.js';
+export type { Cookie } from './session.js';
+export { getRandomProfile, getHeaders, mergeHeaders } from './headers.js';
+export type { HeaderProfile } from './headers.js';
+export { WebScrapeCache, createCache } from './cache.js';
+export type { CacheConfig, CacheEntry } from './cache.js';
+export { MiddlewarePipeline, createPipeline, loggingMiddleware, timeoutMiddleware, retryMiddleware } from './middleware.js';
+export type { Middleware, MiddlewareContext, MiddlewareNext } from './middleware.js';
+
+// ─── Crawling ───
+export { crawl } from './crawler.js';
+export type { CrawlOptions } from './crawler.js';
+export { fetchSitemap, discoverSitemaps, parseSitemap } from './sitemap.js';
+export type { SitemapUrl, SitemapResult } from './sitemap.js';
+export { fetchRobotsTxt, isAllowed, getCrawlDelay, getSitemaps } from './robots.js';
+export type { RobotsTxt, RobotsDirective } from './robots.js';
+
+// ─── Batch & Pool ───
+export { batchScrape, batchScrapeSequential } from './batch.js';
+export type { BatchResult } from './batch.js';
+export { BrowserPool, createPool } from './pool.js';
+export type { PoolConfig } from './pool.js';
+
+// ─── Server ───
+export { createServer, app } from './server.js';
+export type { ServerConfig } from './server.js';
+
+// ─── Utilities ───
+export { ensureDir, writeOutput, slugify, truncate, chunkText as chunkTextUtil, delay, isValidUrl, normalizeUrl, isInternalUrl } from './utils.js';
+export { randomPort, resetPort } from './utils/port.js';
+
+// ─── Types ───
+export {
+  ScrapeOptionsSchema,
+  BrowserAgentConfigSchema,
+} from './types.js';
+export type {
+  ScrapeFormat,
+  SerializeTarget,
+  ScrapeOptions,
+  ScrapeResult,
+  PageMetadata,
+  LinkInfo,
+  ImageInfo,
+  TableInfo,
+  BrowserAgentConfig,
+  ExtractOptions,
+  SerializeOptions,
+  SerializedOutput,
+  AgentOutput,
+  RenderOptions,
+  ChromeCommand,
+  CliOptions,
+  RetryConfig,
+  ProxyConfig,
+  CrawlStats,
+  BatchOptions,
+} from './types.js';

package/scrape/jsdom.d.ts

@@ -0,0 +1,6 @@
+declare module 'jsdom' {
+  export class JSDOM {
+    constructor(html?: string, options?: Record<string, unknown>);
+    window: Window;
+  }
+}

package/scrape/llms-txt.ts

@@ -0,0 +1,84 @@
+import type { ScrapeResult } from './types.js';
+
+export interface LlmsTxtOptions {
+  siteName?: string;
+  description?: string;
+  includeOptional?: boolean;
+}
+
+export function generateLlmsTxt(
+  results: ScrapeResult[],
+  options: LlmsTxtOptions = {}
+): string {
+  const siteName = options.siteName || results[0]?.metadata?.title || 'Site';
+  const description = options.description || results[0]?.metadata?.description || '';
+
+  const lines: string[] = [];
+  lines.push(`# ${siteName}`);
+  lines.push('');
+  if (description) {
+    lines.push(`> ${description}`);
+    lines.push('');
+  }
+
+  const pages: ScrapeResult[] = [];
+  const docs: ScrapeResult[] = [];
+
+  for (const result of results) {
+    const url = result.url.toLowerCase();
+    if (url.includes('doc') || url.includes('guide') || url.includes('api')) {
+      docs.push(result);
+    } else {
+      pages.push(result);
+    }
+  }
+
+  if (pages.length > 0) {
+    lines.push('## Pages');
+    lines.push('');
+    for (const page of pages.slice(0, 20)) {
+      const title = page.metadata.title || page.title;
+      const desc = page.metadata.description || '';
+      lines.push(`- ${title}: ${desc || page.url}`);
+    }
+    lines.push('');
+  }
+
+  if (docs.length > 0) {
+    lines.push('## Documentation');
+    lines.push('');
+    for (const doc of docs.slice(0, 20)) {
+      const title = doc.metadata.title || doc.title;
+      const desc = doc.metadata.description || '';
+      lines.push(`- ${title}: ${desc || doc.url}`);
+    }
+    lines.push('');
+  }
+
+  if (options.includeOptional && results.length > 5) {
+    lines.push('## Optional');
+    lines.push('');
+    lines.push(`- Full content: ${results.length} pages scraped`);
+    lines.push(`- Last updated: ${new Date().toISOString().split('T')[0]}`);
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+export function generateLlmsFullTxt(results: ScrapeResult[]): string {
+  const parts: string[] = [];
+
+  for (const result of results) {
+    parts.push(`# ${result.title || result.metadata.title || 'Untitled'}`);
+    parts.push('');
+    parts.push(`Source: ${result.url}`);
+    parts.push('');
+    parts.push(result.content);
+    parts.push('');
+    parts.push('---');
+    parts.push('');
+  }
+
+  return parts.join('\n');
+}

package/scrape/middleware.ts

@@ -0,0 +1,90 @@
+export interface MiddlewareContext<T = Record<string, unknown>> {
+  data: T;
+  state: Record<string, unknown>;
+  startTime: number;
+  aborted: boolean;
+  abort(): void;
+}
+
+export type MiddlewareNext = () => Promise<void>;
+export type Middleware<T = Record<string, unknown>> = (
+  ctx: MiddlewareContext<T>,
+  next: MiddlewareNext
+) => Promise<void>;
+
+export class MiddlewarePipeline<T = Record<string, unknown>> {
+  private middlewares: Middleware<T>[] = [];
+
+  use(middleware: Middleware<T>): this {
+    this.middlewares.push(middleware);
+    return this;
+  }
+
+  async execute(initialData: T): Promise<MiddlewareContext<T>> {
+    const ctx: MiddlewareContext<T> = {
+      data: initialData,
+      state: {},
+      startTime: Date.now(),
+      aborted: false,
+      abort() { this.aborted = true; },
+    };
+
+    let index = 0;
+
+    const next = async (): Promise<void> => {
+      if (ctx.aborted) return;
+      if (index < this.middlewares.length) {
+        const middleware = this.middlewares[index++];
+        await middleware(ctx, next);
+      }
+    };
+
+    await next();
+    return ctx;
+  }
+
+  clear(): void {
+    this.middlewares = [];
+  }
+}
+
+export function loggingMiddleware(logger?: { info: (msg: string) => void }): Middleware {
+  return async (_ctx, next) => {
+    const start = Date.now();
+    await next();
+    const duration = Date.now() - start;
+    logger?.info(`Request completed in ${duration}ms`);
+  };
+}
+
+export function timeoutMiddleware(timeoutMs: number): Middleware {
+  return async (ctx, next) => {
+    const timer = setTimeout(() => ctx.abort(), timeoutMs);
+    try {
+      await next();
+    } finally {
+      clearTimeout(timer);
+    }
+  };
+}
+
+export function retryMiddleware(maxRetries: number, delayMs: number): Middleware {
+  return async (ctx, next) => {
+    let attempt = 0;
+    while (attempt <= maxRetries) {
+      ctx.state.attempt = attempt;
+      try {
+        await next();
+        return;
+      } catch (error) {
+        if (attempt === maxRetries) throw error;
+        await new Promise(r => setTimeout(r, delayMs * Math.pow(2, attempt)));
+        attempt++;
+      }
+    }
+  };
+}
+
+export function createPipeline<T = Record<string, unknown>>(): MiddlewarePipeline<T> {
+  return new MiddlewarePipeline<T>();
+}