@wenathlan/saddle 1.8.7 → 1.8.9

package/scrape/retry.ts

@@ -1,64 +0,0 @@
-import pRetry, { AbortError } from "p-retry";
-import type { RetryConfig } from "./types.js";
-
-export const DEFAULT_RETRY_CONFIG: Required<RetryConfig> = {
-	retries: 3,
-	delay: 1000,
-	backoff: "exponential",
-	maxDelay: 30000,
-	statusCodes: [408, 429, 500, 502, 503, 504],
-};
-
-const RETRYABLE_NETWORK_CODES = ["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EPIPE", "ENOTFOUND"];
-
-export function isRetryableError(error: Error): boolean {
-	const code = (error as any).code;
-	if (code && RETRYABLE_NETWORK_CODES.includes(code)) return true;
-	const status = (error as any).statusCode || (error as any).status;
-	if (status && [408, 429, 500, 502, 503, 504].includes(status)) return true;
-	if (error.name === "AbortError") return false;
-	return false;
-}
-
-function calculateDelay(attempt: number, config: Required<RetryConfig>): number {
-	switch (config.backoff) {
-		case "exponential":
-			return Math.min(config.delay * 2 ** attempt, config.maxDelay);
-		case "linear":
-			return Math.min(config.delay * (attempt + 1), config.maxDelay);
-		case "constant":
-			return config.delay;
-	}
-}
-
-export async function withRetry<T>(
-	fn: (attemptNumber: number) => Promise<T>,
-	config: Partial<RetryConfig> = {},
-	onRetry?: (error: Error, attempt: number) => void,
-): Promise<T> {
-	const cfg = { ...DEFAULT_RETRY_CONFIG, ...config };
-
-	return pRetry(
-		async (attemptNumber) => {
-			try {
-				return await fn(attemptNumber);
-			} catch (error) {
-				if (!isRetryableError(error as Error)) {
-					throw new AbortError((error as Error).message);
-				}
-				throw error;
-			}
-		},
-		{
-			retries: cfg.retries,
-			minTimeout: calculateDelay(0, cfg),
-			maxTimeout: cfg.maxDelay,
-			randomize: cfg.backoff === "exponential",
-			onFailedAttempt: (context) => {
-				onRetry?.(context.error, context.attemptNumber);
-			},
-		},
-	);
-}
-
-export { AbortError };

package/scrape/robots.js

@@ -1,32 +0,0 @@
-/**
- * robots rules are parsed locally and enforced before a fetch is attempted.
- */
-export function robotsrules(text = "") {
-  const groups = [];
-  let current = null;
-  for (const raw of text.split(/\r?\n/)) {
-    const line = raw.split("#", 1)[0].trim();
-    if (!line) continue;
-    const separator = line.indexOf(":");
-    if (separator < 0) continue;
-    const key = line.slice(0, separator).trim().toLowerCase();
-    const value = line.slice(separator + 1).trim();
-    if (key === "user-agent") { current = { agents: [value.toLowerCase()], disallow: [], allow: [], delay: undefined, sitemaps: [] }; groups.push(current); }
-    else if (current && key === "disallow" && value) current.disallow.push(value);
-    else if (current && key === "allow" && value) current.allow.push(value);
-    else if (current && key === "crawl-delay" && Number.isFinite(Number(value))) current.delay = Number(value);
-    else if (key === "sitemap") (current ?? { sitemaps: [] }).sitemaps.push(value);
-  }
-  return { groups };
-}
-
-export function robotsallowed(rules, target, agent = "*") {
-  const pathname = new URL(target).pathname || "/";
-  const candidates = rules.groups.filter((group) => group.agents.includes(agent.toLowerCase()) || group.agents.includes("*"));
-  if (!candidates.length) return true;
-  const disallowed = candidates.flatMap((group) => group.disallow).filter((path) => pathname.startsWith(path));
-  const allowed = candidates.flatMap((group) => group.allow).filter((path) => pathname.startsWith(path));
-  return allowed.some((path) => path.length >= Math.max(...disallowed.map((item) => item.length), 0)) || disallowed.length === 0;
-}
-
-export function robotsdelay(rules, agent = "*") { return rules.groups.find((group) => group.agents.includes(agent.toLowerCase()) || group.agents.includes("*"))?.delay ?? 0; }

package/scrape/robots.ts

@@ -1,137 +0,0 @@
-export interface RobotsDirective {
-  userAgent: string;
-  allow: string[];
-  disallow: string[];
-  crawlDelay?: number;
-}
-
-export interface RobotsTxt {
-  directives: RobotsDirective[];
-  sitemaps: string[];
-  crawlDelay?: number;
-}
-
-const cache = new Map<string, { robots: RobotsTxt; fetchedAt: number }>();
-const CACHE_TTL = 24 * 60 * 60 * 1000;
-
-function parseRobotsTxt(content: string): RobotsTxt {
-  const lines = content.split('\n').map(l => l.trim());
-  const directives: RobotsDirective[] = [];
-  const sitemaps: string[] = [];
-  let current: RobotsDirective | null = null;
-  let globalCrawlDelay: number | undefined;
-
-  for (const line of lines) {
-    if (!line || line.startsWith('#')) continue;
-
-    const colonIndex = line.indexOf(':');
-    if (colonIndex === -1) continue;
-
-    const key = line.slice(0, colonIndex).trim().toLowerCase();
-    const value = line.slice(colonIndex + 1).trim();
-
-    if (key === 'user-agent') {
-      current = { userAgent: value, allow: [], disallow: [] };
-      directives.push(current);
-    } else if (key === 'allow' && current) {
-      current.allow.push(value);
-    } else if (key === 'disallow' && current) {
-      if (value) current.disallow.push(value);
-    } else if (key === 'crawl-delay' && current) {
-      const delay = parseInt(value);
-      if (!isNaN(delay)) {
-        current.crawlDelay = delay;
-        globalCrawlDelay = delay;
-      }
-    } else if (key === 'sitemap') {
-      sitemaps.push(value);
-    }
-  }
-
-  return { directives, sitemaps, crawlDelay: globalCrawlDelay };
-}
-
-function matchUserAgent(pattern: string, userAgent: string): boolean {
-  const lowerPattern = pattern.toLowerCase();
-  const lowerUA = userAgent.toLowerCase();
-  if (lowerPattern === '*') return true;
-  return lowerUA.includes(lowerPattern.replace(/\*/g, ''));
-}
-
-function isPathAllowed(path: string, directive: RobotsDirective): boolean {
-  // Check disallow rules first
-  for (const pattern of directive.disallow) {
-    if (pattern === '/') return false;
-    if (pattern && path.startsWith(pattern)) return false;
-  }
-  // Then check allow rules
-  for (const pattern of directive.allow) {
-    if (pattern === '/') return true;
-    if (pattern && path.startsWith(pattern)) return true;
-  }
-  // Default: allowed
-  return true;
-}
-
-export async function fetchRobotsTxt(siteUrl: string, options: { timeout?: number; userAgent?: string } = {}): Promise<RobotsTxt> {
-  const base = siteUrl.replace(/\/$/, '');
-  const robotsUrl = `${base}/robots.txt`;
-  const userAgent = options.userAgent || 'DevThink-WebScrape/2.0';
-
-  const cached = cache.get(robotsUrl);
-  if (cached && (Date.now() - cached.fetchedAt) < CACHE_TTL) {
-    return cached.robots;
-  }
-
-  try {
-    const response = await fetch(robotsUrl, {
-      headers: { 'User-Agent': userAgent },
-      signal: AbortSignal.timeout(options.timeout || 5000),
-    });
-
-    if (!response.ok) {
-      const robots: RobotsTxt = { directives: [], sitemaps: [], crawlDelay: undefined };
-      cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
-      return robots;
-    }
-
-    const text = await response.text();
-    const robots = parseRobotsTxt(text);
-    cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
-    return robots;
-  } catch {
-    const robots: RobotsTxt = { directives: [], sitemaps: [], crawlDelay: undefined };
-    cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
-    return robots;
-  }
-}
-
-export function isAllowed(url: string, robots: RobotsTxt, userAgent: string = '*'): boolean {
-  try {
-    const parsed = new URL(url);
-    const path = parsed.pathname;
-
-    for (const directive of robots.directives) {
-      if (matchUserAgent(directive.userAgent, userAgent)) {
-        return isPathAllowed(path, directive);
-      }
-    }
-
-    return true;
-  } catch {
-    return true;
-  }
-}
-
-export function getCrawlDelay(robots: RobotsTxt, userAgent: string = '*'): number | undefined {
-  for (const directive of robots.directives) {
-    if (matchUserAgent(directive.userAgent, userAgent)) {
-      return directive.crawlDelay;
-    }
-  }
-  return robots.crawlDelay;
-}
-
-export function getSitemaps(robots: RobotsTxt): string[] {
-  return robots.sitemaps;
-}

package/scrape/schema.js

@@ -1,21 +0,0 @@
-/**
- * schema extraction accepts safe field descriptors and never evaluates source strings.
- */
-export function extractwithschema(html, schema = {}, url) {
-  const result = {};
-  for (const [name, descriptor] of Object.entries(schema)) result[name] = extractfield(html, descriptor, url);
-  return result;
-}
-
-function extractfield(html, descriptor, url) {
-  if (typeof descriptor === "function") return descriptor({ html, url });
-  if (typeof descriptor === "string") return textfromselector(html, descriptor);
-  if (!descriptor || typeof descriptor.selector !== "string") throw new TypeError("schema field requires selector or function");
-  if (descriptor.selector === "title") return html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.trim() ?? null;
-  const escaped = descriptor.selector.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-  const attribute = descriptor.attribute;
-  const pattern = attribute ? new RegExp(`<[^>]+${escaped}[^>]*${attribute}=["']([^"']+)["'][^>]*>`, "i") : new RegExp(`<${escaped}[^>]*>([\\s\\S]*?)<\\/${escaped}>`, "i");
-  return pattern.exec(html)?.[1]?.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() ?? null;
-}
-
-function textfromselector(html, selector) { return extractfield(html, { selector }); }

package/scrape/scrape.ts

@@ -1,123 +0,0 @@
-import { AgentBrowser } from './browser.js';
-import { extractContent } from './extract.js';
-import { fetchHtml, detectRenderingMode } from './fetch.js';
-import type { ScrapeOptions, ScrapeResult } from './types.js';
-import { WebScrapeError, ErrorCode } from './errors.js';
-
-export async function scrapeUrl(url: string, options: ScrapeOptions = {}): Promise<ScrapeResult> {
-  if (!url || !url.startsWith('http')) {
-    throw new WebScrapeError(`Invalid URL: ${url}`, ErrorCode.VALIDATION_FAILED, 400, false);
-  }
-
-  const startTime = Date.now();
-  const mode = options.mode || 'auto';
-
-  if (mode === 'fetch' || mode === 'auto') {
-    try {
-      const fetchResult = await fetchHtml(url, {
-        timeout: options.timeout,
-        headers: options.headers,
-        userAgent: options.userAgent,
-      });
-
-      if (mode === 'auto') {
-        const renderMode = detectRenderingMode(fetchResult.html);
-        if (renderMode === 'static') {
-          return buildResult(fetchResult.html, fetchResult.url, options, startTime);
-        }
-      } else {
-        return buildResult(fetchResult.html, fetchResult.url, options, startTime);
-      }
-    } catch (error) {
-      if (mode === 'fetch') throw error;
-      if (!(error instanceof WebScrapeError) || error.isRetryable) {
-        // Fall through to browser mode
-      } else {
-        throw error;
-      }
-    }
-  }
-
-  const browser = new AgentBrowser({
-    headless: true,
-    timeout: options.timeout ?? 30000,
-    proxy: options.proxy ?? '',
-    userAgent: options.userAgent,
-  });
-
-  try {
-    await browser.launch();
-    await browser.navigate(url, (options.waitUntil as 'load' | 'domcontentloaded' | 'networkidle') ?? 'networkidle');
-
-    if (options.scroll) {
-      await browser.scrollToBottom(800, options.scrollDelay ?? 300, options.maxScrolls ?? 50);
-    }
-
-    const html = await browser.html();
-    const pageTitle = await browser.title();
-    const pageUrl = await browser.url();
-
-    return buildResult(html, pageUrl, options, startTime, pageTitle);
-  } finally {
-    await browser.close();
-  }
-}
-
-export async function scrapeHtml(html: string, options: ScrapeOptions = {}): Promise<ScrapeResult> {
-  const startTime = Date.now();
-  return buildResult(html, 'about:blank', options, startTime);
-}
-
-export async function scrapeWithBrowser(
-  browser: AgentBrowser,
-  url: string,
-  options: ScrapeOptions = {}
-): Promise<ScrapeResult> {
-  const startTime = Date.now();
-
-  await browser.navigate(url, (options.waitUntil as 'load' | 'domcontentloaded' | 'networkidle') ?? 'networkidle');
-
-  if (options.scroll) {
-    await browser.scrollToBottom(800, options.scrollDelay ?? 300, options.maxScrolls ?? 50);
-  }
-
-  const html = await browser.html();
-  const pageTitle = await browser.title();
-  const pageUrl = await browser.url();
-
-  return buildResult(html, pageUrl, options, startTime, pageTitle);
-}
-
-async function buildResult(
-  html: string,
-  url: string,
-  options: ScrapeOptions,
-  startTime: number,
-  title?: string,
-): Promise<ScrapeResult> {
-  const extracted = await extractContent(html, {
-    readable: true,
-    preserveLinks: options.extractLinks ?? true,
-    preserveImages: options.extractImages ?? true,
-    preserveTables: options.extractTables ?? true,
-    maxLength: options.maxContentLength,
-    removeSelectors: options.removeSelectors,
-  });
-
-  const duration = Date.now() - startTime;
-
-  return {
-    url,
-    title: title || extracted.metadata.title || '',
-    content: extracted.content,
-    format: options.format ?? 'markdown',
-    text: extracted.text,
-    links: extracted.links,
-    images: extracted.images,
-    tables: extracted.tables,
-    metadata: extracted.metadata,
-    extractedAt: new Date().toISOString(),
-    duration,
-    size: extracted.content.length + extracted.text.length,
-  };
-}

package/scrape/scraper.js

@@ -1,40 +0,0 @@
-/**
- * scraper composes robots policy cache transport and extraction without browser assumptions.
- */
-import { transport } from "../adapters/transport.js";
-import { extracthtml } from "./extract.js";
-import { robotsallowed, robotsdelay, robotsrules } from "./robots.js";
-import { ttlcache } from "./cache.js";
-
-export function scraper(options = {}) {
-  const client = options.transport ?? transport({ fetcher: options.fetcher, timeout: options.timeout, attempts: options.attempts });
-  const cache = options.cache ?? ttlcache(options.cacheoptions);
-  const agent = options.agent ?? "*";
-  const policies = new Map();
-  return {
-    async robots(origin) {
-      if (policies.has(origin)) return policies.get(origin);
-      const url = new URL("/robots.txt", origin).href;
-      const response = await client.request(url);
-      const rules = robotsrules(response.ok ? await response.text() : "");
-      policies.set(origin, rules);
-      return rules;
-    },
-    async scrape(url) {
-      const target = new URL(url);
-      if (!["http:", "https:"].includes(target.protocol)) throw new TypeError("scraper accepts http and https only");
-      const rules = await this.robots(target.origin);
-      if (!robotsallowed(rules, target.href, agent)) throw new Error("robots policy disallows target");
-      const cached = cache.get(target.href);
-      if (cached) return cached;
-      const wait = robotsdelay(rules, agent);
-      if (wait) await new Promise((resolve) => setTimeout(resolve, wait * 1000));
-      const response = await client.request(target.href, { headers: options.headers });
-      if (!response.ok) throw new Error(`scrape request failed with ${response.status}`);
-      const result = extracthtml(await response.text(), target.href);
-      cache.set(target.href, result, { ttl: options.ttl });
-      return result;
-    },
-    cache
-  };
-}

package/scrape/semantic.js

@@ -1,22 +0,0 @@
-/**
- * semantic extraction exposes bounded headings, landmarks, controls and links without evaluating page code.
- */
-
-/** Extracts semantic page facts from HTML using safe built-in parsing heuristics. */
-export function extractsemantic(html, url) {
-  const source = String(html ?? "");
-  const headings = [...source.matchAll(/<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi)].slice(0, 100).map((match) => ({ level: Number(match[1]), text: clean(match[2]) })).filter((item) => item.text);
-  const landmarks = [...source.matchAll(/<(main|nav|header|footer|aside|section|article)\b([^>]*)>/gi)].slice(0, 100).map((match) => ({ role: match[1].toLowerCase(), label: attribute(match[2], "aria-label") ?? attribute(match[2], "id") ?? "" }));
-  const controls = [...source.matchAll(/<(button|input|textarea|select|a)\b([^>]*)>([\s\S]*?)<\/\1>|<(input)\b([^>]*)\/?\s*>/gi)].slice(0, 200).map((match, index) => {
-    const tag = (match[1] ?? match[4]).toLowerCase();
-    const attrs = match[2] ?? match[5] ?? "";
-    return { ref: `e${index + 1}`, role: attribute(attrs, "role") ?? tag, name: attribute(attrs, "aria-label") ?? attribute(attrs, "placeholder") ?? clean(match[3] ?? ""), type: attribute(attrs, "type") };
-  });
-  const links = [...source.matchAll(/<a\b([^>]*)href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi)].slice(0, 500).map((match) => ({ url: resolve(match[2], url), text: clean(match[3]), rel: attribute(match[1], "rel") })).filter((link) => link.url);
-  return { url, title: clean(source.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? ""), headings, landmarks, controls, links: dedupe(links, (item) => item.url), semantictext: clean(source.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ")).slice(0, 100000) };
-}
-
-function clean(value) { return String(value ?? "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").replaceAll("&amp;", "&").replaceAll("&lt;", "<").replaceAll("&gt;", ">").trim(); }
-function attribute(value, name) { return value.match(new RegExp(`${name}=["']([^"']*)["']`, "i"))?.[1] ?? null; }
-function resolve(value, base) { try { return new URL(value, base).href; } catch { return null; } }
-function dedupe(values, key) { const seen = new Set(); return values.filter((value) => { const item = key(value); if (seen.has(item)) return false; seen.add(item); return true; }); }