@wenathlan/saddle 1.8.8 → 1.8.9

package/scrape/index.ts

@@ -1,92 +0,0 @@
-// ─── Core Scraping ───
-export { AgentBrowser, createBrowser } from './browser.js';
-export { scrapeUrl, scrapeHtml, scrapeWithBrowser } from './scrape.js';
-export { extractContent, extractReadable } from './extract.js';
-export type { ExtractedContent } from './extract.js';
-export { fetchHtml, fetchAndParse, detectRenderingMode } from './fetch.js';
-export type { FetchResult } from './fetch.js';
-
-// ─── Serialization ───
-export { serializeResult, serializeHtml } from './serialize.js';
-export { resolveFormat, formatFromExtension, extensionForFormat, buildSerializeOptions, convertResult } from './formats.js';
-
-// ─── Renderer ───
-export { PygameRenderer, createRenderer } from './renderer.js';
-
-// ─── AI/LLM ───
-export { formatForAgent, buildContext } from './agent.js';
-export { estimateTokens, countTokens, fitsInContext, truncateToTokens, tokenCost } from './tokens.js';
-export type { ModelType } from './tokens.js';
-export { chunkMarkdown, chunkText, formatChunksForRAG } from './chunking.js';
-export type { Chunk, ChunkOptions } from './chunking.js';
-export { generateLlmsTxt, generateLlmsFullTxt } from './llms-txt.js';
-export type { LlmsTxtOptions } from './llms-txt.js';
-
-// ─── Infrastructure ───
-export { WebScrapeError, ValidationError, TimeoutError, BlockedError, RateLimitError, ProxyError, ParseError, AuthError, NetworkError, BrowserNotAvailableError } from './errors.js';
-export type { ErrorCode } from './errors.js';
-export { createEventEmitter, globalEmitter } from './events.js';
-export type { WebScrapeEvents, EventEmitter } from './events.js';
-export { withRetry, isRetryableError, AbortError } from './retry.js';
-export { RateLimiter, createRateLimiter } from './rate-limiter.js';
-export type { RateLimiterConfig } from './rate-limiter.js';
-export { ProxyPool, createProxyPool } from './proxy.js';
-export type { ProxyPoolConfig, ProxyRotationStrategy } from './proxy.js';
-export { CookieJar, ScrapingSession, createSession } from './session.js';
-export type { Cookie } from './session.js';
-export { getRandomProfile, getHeaders, mergeHeaders } from './headers.js';
-export type { HeaderProfile } from './headers.js';
-export { WebScrapeCache, createCache } from './cache.js';
-export type { CacheConfig, CacheEntry } from './cache.js';
-export { MiddlewarePipeline, createPipeline, loggingMiddleware, timeoutMiddleware, retryMiddleware } from './middleware.js';
-export type { Middleware, MiddlewareContext, MiddlewareNext } from './middleware.js';
-
-// ─── Crawling ───
-export { crawl } from './crawler.js';
-export type { CrawlOptions } from './crawler.js';
-export { fetchSitemap, discoverSitemaps, parseSitemap } from './sitemap.js';
-export type { SitemapUrl, SitemapResult } from './sitemap.js';
-export { fetchRobotsTxt, isAllowed, getCrawlDelay, getSitemaps } from './robots.js';
-export type { RobotsTxt, RobotsDirective } from './robots.js';
-
-// ─── Batch & Pool ───
-export { batchScrape, batchScrapeSequential } from './batch.js';
-export type { BatchResult } from './batch.js';
-export { BrowserPool, createPool } from './pool.js';
-export type { PoolConfig } from './pool.js';
-
-// ─── Server ───
-export { createServer, app } from './server.js';
-export type { ServerConfig } from './server.js';
-
-// ─── Utilities ───
-export { ensureDir, writeOutput, slugify, truncate, chunkText as chunkTextUtil, delay, isValidUrl, normalizeUrl, isInternalUrl } from './utils.js';
-export { randomPort, resetPort } from './utils/port.js';
-
-// ─── Types ───
-export {
-  ScrapeOptionsSchema,
-  BrowserAgentConfigSchema,
-} from './types.js';
-export type {
-  ScrapeFormat,
-  SerializeTarget,
-  ScrapeOptions,
-  ScrapeResult,
-  PageMetadata,
-  LinkInfo,
-  ImageInfo,
-  TableInfo,
-  BrowserAgentConfig,
-  ExtractOptions,
-  SerializeOptions,
-  SerializedOutput,
-  AgentOutput,
-  RenderOptions,
-  ChromeCommand,
-  CliOptions,
-  RetryConfig,
-  ProxyConfig,
-  CrawlStats,
-  BatchOptions,
-} from './types.js';

package/scrape/jsdom.d.ts

@@ -1,6 +0,0 @@
-declare module 'jsdom' {
-  export class JSDOM {
-    constructor(html?: string, options?: Record<string, unknown>);
-    window: Window;
-  }
-}

package/scrape/llms-txt.ts

@@ -1,84 +0,0 @@
-import type { ScrapeResult } from './types.js';
-
-export interface LlmsTxtOptions {
-  siteName?: string;
-  description?: string;
-  includeOptional?: boolean;
-}
-
-export function generateLlmsTxt(
-  results: ScrapeResult[],
-  options: LlmsTxtOptions = {}
-): string {
-  const siteName = options.siteName || results[0]?.metadata?.title || 'Site';
-  const description = options.description || results[0]?.metadata?.description || '';
-
-  const lines: string[] = [];
-  lines.push(`# ${siteName}`);
-  lines.push('');
-  if (description) {
-    lines.push(`> ${description}`);
-    lines.push('');
-  }
-
-  const pages: ScrapeResult[] = [];
-  const docs: ScrapeResult[] = [];
-
-  for (const result of results) {
-    const url = result.url.toLowerCase();
-    if (url.includes('doc') || url.includes('guide') || url.includes('api')) {
-      docs.push(result);
-    } else {
-      pages.push(result);
-    }
-  }
-
-  if (pages.length > 0) {
-    lines.push('## Pages');
-    lines.push('');
-    for (const page of pages.slice(0, 20)) {
-      const title = page.metadata.title || page.title;
-      const desc = page.metadata.description || '';
-      lines.push(`- ${title}: ${desc || page.url}`);
-    }
-    lines.push('');
-  }
-
-  if (docs.length > 0) {
-    lines.push('## Documentation');
-    lines.push('');
-    for (const doc of docs.slice(0, 20)) {
-      const title = doc.metadata.title || doc.title;
-      const desc = doc.metadata.description || '';
-      lines.push(`- ${title}: ${desc || doc.url}`);
-    }
-    lines.push('');
-  }
-
-  if (options.includeOptional && results.length > 5) {
-    lines.push('## Optional');
-    lines.push('');
-    lines.push(`- Full content: ${results.length} pages scraped`);
-    lines.push(`- Last updated: ${new Date().toISOString().split('T')[0]}`);
-    lines.push('');
-  }
-
-  return lines.join('\n');
-}
-
-export function generateLlmsFullTxt(results: ScrapeResult[]): string {
-  const parts: string[] = [];
-
-  for (const result of results) {
-    parts.push(`# ${result.title || result.metadata.title || 'Untitled'}`);
-    parts.push('');
-    parts.push(`Source: ${result.url}`);
-    parts.push('');
-    parts.push(result.content);
-    parts.push('');
-    parts.push('---');
-    parts.push('');
-  }
-
-  return parts.join('\n');
-}

package/scrape/middleware.ts

@@ -1,90 +0,0 @@
-export interface MiddlewareContext<T = Record<string, unknown>> {
-  data: T;
-  state: Record<string, unknown>;
-  startTime: number;
-  aborted: boolean;
-  abort(): void;
-}
-
-export type MiddlewareNext = () => Promise<void>;
-export type Middleware<T = Record<string, unknown>> = (
-  ctx: MiddlewareContext<T>,
-  next: MiddlewareNext
-) => Promise<void>;
-
-export class MiddlewarePipeline<T = Record<string, unknown>> {
-  private middlewares: Middleware<T>[] = [];
-
-  use(middleware: Middleware<T>): this {
-    this.middlewares.push(middleware);
-    return this;
-  }
-
-  async execute(initialData: T): Promise<MiddlewareContext<T>> {
-    const ctx: MiddlewareContext<T> = {
-      data: initialData,
-      state: {},
-      startTime: Date.now(),
-      aborted: false,
-      abort() { this.aborted = true; },
-    };
-
-    let index = 0;
-
-    const next = async (): Promise<void> => {
-      if (ctx.aborted) return;
-      if (index < this.middlewares.length) {
-        const middleware = this.middlewares[index++];
-        await middleware(ctx, next);
-      }
-    };
-
-    await next();
-    return ctx;
-  }
-
-  clear(): void {
-    this.middlewares = [];
-  }
-}
-
-export function loggingMiddleware(logger?: { info: (msg: string) => void }): Middleware {
-  return async (_ctx, next) => {
-    const start = Date.now();
-    await next();
-    const duration = Date.now() - start;
-    logger?.info(`Request completed in ${duration}ms`);
-  };
-}
-
-export function timeoutMiddleware(timeoutMs: number): Middleware {
-  return async (ctx, next) => {
-    const timer = setTimeout(() => ctx.abort(), timeoutMs);
-    try {
-      await next();
-    } finally {
-      clearTimeout(timer);
-    }
-  };
-}
-
-export function retryMiddleware(maxRetries: number, delayMs: number): Middleware {
-  return async (ctx, next) => {
-    let attempt = 0;
-    while (attempt <= maxRetries) {
-      ctx.state.attempt = attempt;
-      try {
-        await next();
-        return;
-      } catch (error) {
-        if (attempt === maxRetries) throw error;
-        await new Promise(r => setTimeout(r, delayMs * Math.pow(2, attempt)));
-        attempt++;
-      }
-    }
-  };
-}
-
-export function createPipeline<T = Record<string, unknown>>(): MiddlewarePipeline<T> {
-  return new MiddlewarePipeline<T>();
-}

package/scrape/normalize.js

@@ -1,86 +0,0 @@
-/**
- * content normalization classifies bounded response bytes without owning a parser, transport or storage backend.
- */
-
-const extensions = Object.freeze({
-  ".json": "application/json",
-  ".map": "application/json",
-  ".xml": "application/xml",
-  ".rss": "application/rss+xml",
-  ".atom": "application/atom+xml",
-  ".md": "text/markdown",
-  ".markdown": "text/markdown",
-  ".html": "text/html",
-  ".htm": "text/html",
-  ".txt": "text/plain",
-  ".csv": "text/csv"
-});
-
-/** Detects a normalized media type from a header, URL suffix or caller fallback. */
-export function detectcontenttype(header, url, fallback = "application/octet-stream") {
-  const value = String(header ?? "").split(";", 1)[0].trim().toLowerCase();
-  if (value) return value;
-  try {
-    const pathname = new URL(url).pathname.toLowerCase();
-    const suffix = Object.keys(extensions).find((extension) => pathname.endsWith(extension));
-    if (suffix) return extensions[suffix];
-  } catch { /* invalid or absent URLs remain caller-owned metadata */ }
-  return String(fallback).toLowerCase();
-}
-
-/** Normalizes bounded input bytes into a serializable text, JSON or binary result. */
-export function normalizeresult(input, options = {}) {
-  const contenttype = detectcontenttype(options.contenttype, options.url, options.defaultcontenttype ?? "application/octet-stream");
-  const bytes = tobytes(input);
-  const maxbytes = boundedlimit(options.maxbytes, 2_000_000);
-  if (bytes.byteLength > maxbytes) throw normalizationerror("CONTENT_TOO_LARGE", `content exceeds ${maxbytes} bytes`);
-  const kind = contentkind(contenttype);
-  if (kind === "binary") return { contenttype, kind, size: bytes.byteLength, bytes };
-  const text = new TextDecoder(options.charset ?? "utf-8", { fatal: false }).decode(bytes).replaceAll("\r\n", "\n");
-  if (text.length > boundedlimit(options.maxtext, maxbytes)) throw normalizationerror("TEXT_TOO_LARGE", "decoded content exceeds the configured text limit");
-  if (kind === "json") {
-    try { return { contenttype, kind, size: bytes.byteLength, content: text, data: JSON.parse(text) }; } catch (error) { throw normalizationerror("INVALID_JSON", `invalid JSON content: ${error.message}`); }
-  }
-  return { contenttype, kind, size: bytes.byteLength, content: text };
-}
-
-/** Reads a caller-provided response body after checking declared and actual size limits. */
-export async function normalizeresponse(response, options = {}) {
-  if (!response || typeof response !== "object") throw new TypeError("response is required");
-  const headers = response.headers;
-  const header = typeof headers?.get === "function" ? headers.get("content-type") : headers?.["content-type"];
-  const declared = Number(typeof headers?.get === "function" ? headers.get("content-length") : headers?.["content-length"]);
-  const maxbytes = boundedlimit(options.maxbytes, 2_000_000);
-  if (Number.isFinite(declared) && declared > maxbytes) throw normalizationerror("CONTENT_TOO_LARGE", `content exceeds ${maxbytes} bytes`);
-  const body = typeof response.arrayBuffer === "function" ? await response.arrayBuffer() : await response.text();
-  return normalizeresult(body, { ...options, contenttype: header ?? options.contenttype, url: options.url ?? response.url });
-}
-
-function contentkind(contenttype) {
-  if (contenttype === "application/json" || contenttype.endsWith("+json")) return "json";
-  if (contenttype === "text/html" || contenttype === "application/xhtml+xml") return "html";
-  if (contenttype === "application/xml" || contenttype === "text/xml" || contenttype.endsWith("+xml")) return "xml";
-  if (contenttype === "text/markdown") return "markdown";
-  if (contenttype.startsWith("text/")) return "text";
-  return "binary";
-}
-
-function tobytes(input) {
-  if (typeof input === "string") return new TextEncoder().encode(input);
-  if (input instanceof Uint8Array) return input;
-  if (input instanceof ArrayBuffer) return new Uint8Array(input);
-  if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
-  throw new TypeError("content must be a string, ArrayBuffer or typed array");
-}
-
-function boundedlimit(value, fallback) {
-  const limit = Number(value ?? fallback);
-  if (!Number.isSafeInteger(limit) || limit < 1) throw new TypeError("content limit must be a positive safe integer");
-  return limit;
-}
-
-function normalizationerror(code, message) {
-  const error = new Error(message);
-  error.code = code;
-  return error;
-}

package/scrape/pool.ts

@@ -1,95 +0,0 @@
-import { AgentBrowser } from './browser.js';
-import type { BrowserAgentConfig } from './types.js';
-
-export interface PoolConfig {
-  maxSize?: number;
-  acquireTimeoutMs?: number;
-}
-
-interface PoolEntry {
-  browser: AgentBrowser;
-  inUse: boolean;
-  createdAt: number;
-}
-
-export class BrowserPool {
-  private pool: PoolEntry[] = [];
-  private waitQueue: Array<(browser: AgentBrowser) => void> = [];
-  private config: Required<PoolConfig>;
-  private browserConfig: BrowserAgentConfig;
-
-  constructor(config: PoolConfig = {}, browserConfig: BrowserAgentConfig = {}) {
-    this.config = {
-      maxSize: config.maxSize || 3,
-      acquireTimeoutMs: config.acquireTimeoutMs || 30000,
-    };
-    this.browserConfig = browserConfig;
-  }
-
-  async acquire(): Promise<AgentBrowser> {
-    const idle = this.pool.find(e => !e.inUse && e.browser.isConnected());
-    if (idle) {
-      idle.inUse = true;
-      return idle.browser;
-    }
-
-    if (this.pool.length < this.config.maxSize) {
-      const browser = new AgentBrowser(this.browserConfig);
-      await browser.launch();
-      const entry: PoolEntry = { browser, inUse: true, createdAt: Date.now() };
-      this.pool.push(entry);
-      return browser;
-    }
-
-    return new Promise<AgentBrowser>((resolve, reject) => {
-      const timeout = setTimeout(() => {
-        const idx = this.waitQueue.indexOf(waiter);
-        if (idx !== -1) this.waitQueue.splice(idx, 1);
-        reject(new Error('Timeout waiting for browser'));
-      }, this.config.acquireTimeoutMs);
-
-      const waiter = (browser: AgentBrowser) => {
-        clearTimeout(timeout);
-        resolve(browser);
-      };
-      this.waitQueue.push(waiter);
-    });
-  }
-
-  release(browser: AgentBrowser): void {
-    const entry = this.pool.find(e => e.browser === browser);
-    if (entry) {
-      entry.inUse = false;
-
-      if (this.waitQueue.length > 0) {
-        const waiter = this.waitQueue.shift()!;
-        entry.inUse = true;
-        waiter(browser);
-      }
-    }
-  }
-
-  async destroy(): Promise<void> {
-    for (const entry of this.pool) {
-      await entry.browser.close().catch(() => {});
-    }
-    this.pool = [];
-    for (const waiter of this.waitQueue) {
-      waiter(new AgentBrowser());
-    }
-    this.waitQueue = [];
-  }
-
-  getStatus(): { total: number; idle: number; inUse: number; waiting: number } {
-    return {
-      total: this.pool.length,
-      idle: this.pool.filter(e => !e.inUse).length,
-      inUse: this.pool.filter(e => e.inUse).length,
-      waiting: this.waitQueue.length,
-    };
-  }
-}
-
-export function createPool(config?: PoolConfig, browserConfig?: BrowserAgentConfig): BrowserPool {
-  return new BrowserPool(config, browserConfig);
-}

package/scrape/port.ts

@@ -1,18 +0,0 @@
-/**
- * Generate a random port number in the ephemeral port range (1024-65535).
- * The port is calculated once and then fixed for the lifetime of the process.
- */
-let _cachedPort: number | null = null;
-
-export function randomPort(): number {
-  if (_cachedPort !== null) return _cachedPort;
-  _cachedPort = Math.floor(Math.random() * 64511) + 1024;
-  return _cachedPort;
-}
-
-/**
- * Reset the cached port (for testing).
- */
-export function resetPort(): void {
-  _cachedPort = null;
-}

package/scrape/proxy.ts

@@ -1,103 +0,0 @@
-import type { ProxyConfig } from './types.js';
-
-interface ProxyStats {
-  failures: number;
-  successes: number;
-  consecutiveFailures: number;
-  lastFailure?: number;
-  lastSuccess?: number;
-  disabled: boolean;
-  disabledAt?: number;
-}
-
-export type ProxyRotationStrategy = 'round-robin' | 'random';
-
-export interface ProxyPoolConfig {
-  maxFailures?: number;
-  reviveAfterMs?: number;
-  strategy?: ProxyRotationStrategy;
-}
-
-const DEFAULT_POOL_CONFIG: Required<ProxyPoolConfig> = {
-  maxFailures: 3,
-  reviveAfterMs: 30 * 60 * 1000,
-  strategy: 'round-robin',
-};
-
-export class ProxyPool {
-  private proxies: ProxyConfig[];
-  private stats = new Map<string, ProxyStats>();
-  private currentIndex = 0;
-  private config: Required<ProxyPoolConfig>;
-
-  constructor(proxies: ProxyConfig[], config: Partial<ProxyPoolConfig> = {}) {
-    this.proxies = proxies;
-    this.config = { ...DEFAULT_POOL_CONFIG, ...config };
-    for (const p of proxies) {
-      this.stats.set(p.url, { failures: 0, successes: 0, consecutiveFailures: 0, disabled: false });
-    }
-  }
-
-  private getActiveProxies(): ProxyConfig[] {
-    const now = Date.now();
-    return this.proxies.filter(p => {
-      const s = this.stats.get(p.url);
-      if (!s) return true;
-      if (s.disabled && s.disabledAt && (now - s.disabledAt) >= this.config.reviveAfterMs) {
-        s.disabled = false;
-        s.failures = 0;
-        s.consecutiveFailures = 0;
-        return true;
-      }
-      return !s.disabled;
-    });
-  }
-
-  next(): ProxyConfig | null {
-    const active = this.getActiveProxies();
-    if (active.length === 0) return null;
-
-    if (this.config.strategy === 'random') {
-      return active[Math.floor(Math.random() * active.length)];
-    }
-
-    const proxy = active[this.currentIndex % active.length];
-    this.currentIndex = (this.currentIndex + 1) % active.length;
-    return proxy;
-  }
-
-  reportSuccess(proxy: ProxyConfig): void {
-    const s = this.stats.get(proxy.url);
-    if (s) {
-      s.successes++;
-      s.consecutiveFailures = 0;
-      s.lastSuccess = Date.now();
-    }
-  }
-
-  reportFailure(proxy: ProxyConfig): void {
-    const s = this.stats.get(proxy.url);
-    if (s) {
-      s.failures++;
-      s.consecutiveFailures++;
-      s.lastFailure = Date.now();
-      if (s.consecutiveFailures >= this.config.maxFailures) {
-        s.disabled = true;
-        s.disabledAt = Date.now();
-      }
-    }
-  }
-
-  getStatus(): { total: number; active: number; disabled: number } {
-    const active = this.getActiveProxies();
-    return {
-      total: this.proxies.length,
-      active: active.length,
-      disabled: this.proxies.length - active.length,
-    };
-  }
-}
-
-export function createProxyPool(proxies: ProxyConfig[], config?: Partial<ProxyPoolConfig>): ProxyPool {
-  return new ProxyPool(proxies, config);
-}

package/scrape/rate-limiter.ts

@@ -1,95 +0,0 @@
-interface TokenBucket {
-	tokens: number;
-	lastRefill: number;
-	maxTokens: number;
-	refillRate: number;
-}
-
-export interface RateLimiterConfig {
-	maxTokensPerInterval: number;
-	intervalMs: number;
-	maxConcurrent?: number;
-}
-
-const DEFAULT_CONFIG: Required<RateLimiterConfig> = {
-	maxTokensPerInterval: 10,
-	intervalMs: 1000,
-	maxConcurrent: 5,
-};
-
-export class RateLimiter {
-	private buckets = new Map<string, TokenBucket>();
-	private active = new Map<string, number>();
-	private config: Required<RateLimiterConfig>;
-
-	constructor(config: Partial<RateLimiterConfig> = {}) {
-		this.config = { ...DEFAULT_CONFIG, ...config };
-	}
-
-	private getBucket(key: string): TokenBucket {
-		if (!this.buckets.has(key)) {
-			this.buckets.set(key, {
-				tokens: this.config.maxTokensPerInterval,
-				lastRefill: Date.now(),
-				maxTokens: this.config.maxTokensPerInterval,
-				refillRate: this.config.maxTokensPerInterval / (this.config.intervalMs / 1000),
-			});
-		}
-		return this.buckets.get(key)!;
-	}
-
-	private refill(bucket: TokenBucket): void {
-		const now = Date.now();
-		const elapsed = (now - bucket.lastRefill) / 1000;
-		bucket.tokens = Math.min(bucket.maxTokens, bucket.tokens + elapsed * bucket.refillRate);
-		bucket.lastRefill = now;
-	}
-
-	async acquire(key: string): Promise<void> {
-		const bucket = this.getBucket(key);
-		const currentActive = this.active.get(key) || 0;
-
-		if (currentActive >= this.config.maxConcurrent) {
-			await new Promise((resolve) => setTimeout(resolve, 100));
-			return this.acquire(key);
-		}
-
-		this.refill(bucket);
-
-		if (bucket.tokens < 1) {
-			const waitMs = ((1 - bucket.tokens) / bucket.refillRate) * 1000;
-			await new Promise((resolve) => setTimeout(resolve, Math.ceil(waitMs)));
-			this.refill(bucket);
-		}
-
-		bucket.tokens -= 1;
-		this.active.set(key, currentActive + 1);
-	}
-
-	release(key: string): void {
-		const current = this.active.get(key) || 0;
-		if (current <= 1) {
-			this.active.delete(key);
-		} else {
-			this.active.set(key, current - 1);
-		}
-	}
-
-	getStats(key: string): { tokens: number; active: number } {
-		const bucket = this.buckets.get(key);
-		if (bucket) this.refill(bucket);
-		return {
-			tokens: bucket?.tokens ?? this.config.maxTokensPerInterval,
-			active: this.active.get(key) || 0,
-		};
-	}
-
-	clear(): void {
-		this.buckets.clear();
-		this.active.clear();
-	}
-}
-
-export function createRateLimiter(config?: Partial<RateLimiterConfig>): RateLimiter {
-	return new RateLimiter(config);
-}