@wenathlan/saddle 1.8.7 → 1.8.9

package/scrape/chunking.ts

@@ -1,193 +0,0 @@
-import { estimateTokens } from './tokens.js';
-
-export interface ChunkOptions {
-  maxTokens?: number;
-  overlapTokens?: number;
-  preserveCodeBlocks?: boolean;
-  includeHeadingPath?: boolean;
-}
-
-export interface Chunk {
-  content: string;
-  headingPath: string[];
-  tokenCount: number;
-  index: number;
-}
-
-const DEFAULT_CHUNK_OPTIONS: Required<ChunkOptions> = {
-  maxTokens: 512,
-  overlapTokens: 50,
-  preserveCodeBlocks: true,
-  includeHeadingPath: true,
-};
-
-function splitByHeaders(markdown: string): { heading: string; level: number; content: string }[] {
-  const sections: { heading: string; level: number; content: string }[] = [];
-  const lines = markdown.split('\n');
-  let currentHeading = '';
-  let currentLevel = 0;
-  let currentContent: string[] = [];
-
-  for (const line of lines) {
-    const headerMatch = line.match(/^(#{1,6})\s+(.+)/);
-    if (headerMatch) {
-      if (currentContent.length > 0 || currentHeading) {
-        sections.push({
-          heading: currentHeading,
-          level: currentLevel,
-          content: currentContent.join('\n').trim(),
-        });
-      }
-      currentLevel = headerMatch[1].length;
-      currentHeading = headerMatch[2];
-      currentContent = [];
-    } else {
-      currentContent.push(line);
-    }
-  }
-
-  if (currentContent.length > 0 || currentHeading) {
-    sections.push({
-      heading: currentHeading,
-      level: currentLevel,
-      content: currentContent.join('\n').trim(),
-    });
-  }
-
-  return sections;
-}
-
-function splitByParagraphs(text: string): string[] {
-  return text.split(/\n\n+/).filter(p => p.trim().length > 0);
-}
-
-function splitBySentences(text: string): string[] {
-  return text.match(/[^.!?]+[.!?]+/g) || [text];
-}
-
-export function chunkMarkdown(markdown: string, options: ChunkOptions = {}): Chunk[] {
-  const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
-  const chunks: Chunk[] = [];
-  const sections = splitByHeaders(markdown);
-  const headingPath: string[] = [];
-  let headingDepth = 0;
-
-  for (const section of sections) {
-    while (headingDepth > 0 && headingDepth >= section.level) {
-      headingDepth--;
-      headingPath.pop();
-    }
-    if (section.heading) {
-      headingDepth = section.level;
-      headingPath.push(section.heading);
-    }
-
-    const sectionText = section.heading ? `#${'#'.repeat(section.level - 1)} ${section.heading}\n\n${section.content}` : section.content;
-
-    if (estimateTokens(sectionText) <= opts.maxTokens) {
-      chunks.push({
-        content: sectionText,
-        headingPath: opts.includeHeadingPath ? [...headingPath] : [],
-        tokenCount: estimateTokens(sectionText),
-        index: chunks.length,
-      });
-    } else {
-      const paragraphs = splitByParagraphs(sectionText);
-      let currentChunk = '';
-
-      for (const paragraph of paragraphs) {
-        const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
-
-        if (estimateTokens(testChunk) <= opts.maxTokens) {
-          currentChunk = testChunk;
-        } else {
-          if (currentChunk) {
-            chunks.push({
-              content: currentChunk,
-              headingPath: opts.includeHeadingPath ? [...headingPath] : [],
-              tokenCount: estimateTokens(currentChunk),
-              index: chunks.length,
-            });
-          }
-
-          if (estimateTokens(paragraph) > opts.maxTokens) {
-            const sentences = splitBySentences(paragraph);
-            let sentenceChunk = '';
-            for (const sentence of sentences) {
-              const testSentence = sentenceChunk ? `${sentenceChunk} ${sentence}` : sentence;
-              if (estimateTokens(testSentence) <= opts.maxTokens) {
-                sentenceChunk = testSentence;
-              } else {
-                if (sentenceChunk) {
-                  chunks.push({
-                    content: sentenceChunk,
-                    headingPath: opts.includeHeadingPath ? [...headingPath] : [],
-                    tokenCount: estimateTokens(sentenceChunk),
-                    index: chunks.length,
-                  });
-                }
-                sentenceChunk = sentence;
-              }
-            }
-            currentChunk = sentenceChunk;
-          } else {
-            currentChunk = paragraph;
-          }
-        }
-      }
-
-      if (currentChunk) {
-        chunks.push({
-          content: currentChunk,
-          headingPath: opts.includeHeadingPath ? [...headingPath] : [],
-          tokenCount: estimateTokens(currentChunk),
-          index: chunks.length,
-        });
-      }
-    }
-  }
-
-  return chunks;
-}
-
-export function chunkText(text: string, options: ChunkOptions = {}): Chunk[] {
-  const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
-  const chunks: Chunk[] = [];
-  const paragraphs = splitByParagraphs(text);
-  let currentChunk = '';
-
-  for (const paragraph of paragraphs) {
-    const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
-    if (estimateTokens(testChunk) <= opts.maxTokens) {
-      currentChunk = testChunk;
-    } else {
-      if (currentChunk) {
-        chunks.push({
-          content: currentChunk,
-          headingPath: [],
-          tokenCount: estimateTokens(currentChunk),
-          index: chunks.length,
-        });
-      }
-      currentChunk = paragraph;
-    }
-  }
-
-  if (currentChunk) {
-    chunks.push({
-      content: currentChunk,
-      headingPath: [],
-      tokenCount: estimateTokens(currentChunk),
-      index: chunks.length,
-    });
-  }
-
-  return chunks;
-}
-
-export function formatChunksForRAG(chunks: Chunk[]): string {
-  return chunks.map(chunk => {
-    const heading = chunk.headingPath.length > 0 ? `## ${chunk.headingPath.join(' > ')}\n\n` : '';
-    return `${heading}${chunk.content}`;
-  }).join('\n\n---\n\n');
-}

package/scrape/cli.ts

@@ -1,105 +0,0 @@
-#!/usr/bin/env node
-import { Command } from 'commander';
-import { readFileSync, existsSync } from 'node:fs';
-import { resolve } from 'node:path';
-import { scrapeUrl, scrapeHtml } from './scrape.js';
-import { serializeResult } from './serialize.js';
-import { resolveFormat, extensionForFormat, buildSerializeOptions } from './formats.js';
-import { formatForAgent } from './agent.js';
-import { writeOutput } from './utils.js';
-import type { ScrapeOptions, ScrapeResult } from './types.js';
-
-const program = new Command();
-
-program
-  .name('webscrape')
-  .description('DevThink WebScrape — Universal AI-powered web scraping toolkit')
-  .version('2.0.0');
-
-program
-  .argument('[url]', 'URL to scrape')
-  .option('-f, --format <type>', 'Output format: markdown, html, text, json, xml, redis', 'markdown')
-  .option('-o, --output <file>', 'Write output to file')
-  .option('--file <path>', 'Scrape HTML from local file')
-  .option('--mode <mode>', 'Scraping mode: auto, fetch, browser', 'auto')
-  .option('--no-headless', 'Show browser window')
-  .option('--timeout <ms>', 'Navigation timeout', '30000')
-  .option('--scroll', 'Scroll page to load dynamic content')
-  .option('--readable', 'Extract readable content only')
-  .option('--agent', 'Format output for AI agent consumption')
-  .option('--pretty', 'Pretty-print JSON output')
-  .option('--proxy <url>', 'Proxy server URL')
-  .option('--retries <n>', 'Number of retries', '3')
-  .option('--user-agent <ua>', 'Custom User-Agent string')
-  .action(async (url, options) => {
-    if (!url && !options.file) {
-      program.help();
-    }
-
-    const format = resolveFormat(options.format);
-    const scrapeOpts: ScrapeOptions = {
-      format,
-      timeout: parseInt(options.timeout),
-      scroll: options.scroll,
-      readable: options.readable,
-      mode: options.mode,
-      proxy: options.proxy,
-      retries: parseInt(options.retries),
-      userAgent: options.userAgent,
-      extractLinks: true,
-      extractImages: true,
-      extractTables: true,
-    };
-
-    if (options.readable) {
-      scrapeOpts.removeSelectors = [
-        'script', 'style', 'nav', 'footer', 'header',
-        '.sidebar', '.advertisement', '.ads', '.menu',
-        '.comments', '.comment', '#comments',
-      ];
-    }
-
-    console.error(`Scraping: ${url || options.file}`);
-
-    let result: ScrapeResult;
-
-    if (options.file) {
-      const filePath = resolve(options.file);
-      if (!existsSync(filePath)) {
-        console.error(`File not found: ${filePath}`);
-        process.exit(1);
-      }
-      const html = readFileSync(filePath, 'utf-8');
-      result = await scrapeHtml(html, scrapeOpts);
-    } else {
-      result = await scrapeUrl(url, scrapeOpts);
-    }
-
-    console.error(`Done in ${result.duration}ms (${(result.size / 1024).toFixed(1)} KB)`);
-
-    if (options.agent) {
-      const agentOutput = formatForAgent(result);
-      const output = JSON.stringify(agentOutput, null, 2);
-
-      if (options.output) {
-        writeOutput(options.output, output);
-        console.error(`Saved to ${options.output}`);
-      } else {
-        console.log(output);
-      }
-    } else {
-      const serializeOpts = buildSerializeOptions(format, options.pretty);
-      const serialized = serializeResult(result, serializeOpts);
-
-      if (options.output) {
-        const ext = extensionForFormat(format);
-        const finalPath = options.output.endsWith(ext) ? options.output : options.output + ext;
-        writeOutput(finalPath, serialized.content);
-        console.error(`Saved to ${finalPath}`);
-      } else {
-        console.log(serialized.content);
-      }
-    }
-  });
-
-program.parse();

package/scrape/crawler.ts

@@ -1,115 +0,0 @@
-import type { ScrapeOptions, ScrapeResult, CrawlStats } from './types.js';
-import { scrapeUrl } from './scrape.js';
-
-export interface CrawlOptions extends ScrapeOptions {
-  maxDepth?: number;
-  maxPages?: number;
-  maxConcurrent?: number;
-  sameDomain?: boolean;
-  delayMs?: number;
-  onDiscover?: (url: string, depth: number) => void;
-  onResult?: (result: ScrapeResult) => void;
-  onError?: (url: string, error: Error) => void;
-}
-
-interface CrawlEntry {
-  url: string;
-  depth: number;
-}
-
-function isSameDomain(url1: string, url2: string): boolean {
-  try {
-    return new URL(url1).hostname === new URL(url2).hostname;
-  } catch {
-    return false;
-  }
-}
-
-function normalizeForDedup(url: string): string {
-  try {
-    const u = new URL(url);
-    u.hash = '';
-    u.search = '';
-    return u.href.replace(/\/+$/, '') || u.href;
-  } catch {
-    return url;
-  }
-}
-
-export async function crawl(
-  startUrl: string,
-  options: CrawlOptions = {}
-): Promise<{ results: ScrapeResult[]; stats: CrawlStats }> {
-  const {
-    maxDepth = 2,
-    maxPages = 50,
-    maxConcurrent = 3,
-    sameDomain = true,
-    delayMs = 1000,
-    onDiscover,
-    onResult,
-    onError,
-    ...scrapeOpts
-  } = options;
-
-  const startTime = Date.now();
-  const visited = new Set<string>();
-  const queue: CrawlEntry[] = [{ url: startUrl, depth: 0 }];
-  const results: ScrapeResult[] = [];
-  let failed = 0;
-
-  visited.add(normalizeForDedup(startUrl));
-
-  while (queue.length > 0 && results.length < maxPages) {
-    const batch = queue.splice(0, maxConcurrent);
-    const promises = batch.map(async (entry) => {
-      if (entry.depth > maxDepth) return;
-
-      try {
-        const result = await scrapeUrl(entry.url, scrapeOpts);
-        results.push(result);
-        onResult?.(result);
-
-        // Extract links for further crawling
-        if (entry.depth < maxDepth) {
-          for (const link of result.links) {
-            const normalized = normalizeForDedup(link.href);
-            if (visited.has(normalized)) continue;
-            if (sameDomain && !isSameDomain(startUrl, link.href)) continue;
-            try {
-              new URL(link.href);
-            } catch {
-              continue;
-            }
-            visited.add(normalized);
-            queue.push({ url: link.href, depth: entry.depth + 1 });
-            onDiscover?.(link.href, entry.depth + 1);
-          }
-        }
-      } catch (error) {
-        failed++;
-        onError?.(entry.url, error as Error);
-      }
-    });
-
-    await Promise.all(promises);
-
-    if (delayMs > 0 && queue.length > 0) {
-      await new Promise(r => setTimeout(r, delayMs));
-    }
-  }
-
-  const duration = Date.now() - startTime;
-
-  return {
-    results,
-    stats: {
-      totalUrls: visited.size,
-      successful: results.length,
-      failed,
-      skipped: visited.size - results.length - failed,
-      duration,
-      avgResponseTime: results.length > 0 ? results.reduce((sum, r) => sum + r.duration, 0) / results.length : 0,
-    },
-  };
-}

package/scrape/dev-server.ts

@@ -1,94 +0,0 @@
-#!/usr/bin/env node
-import { createServer } from 'node:http';
-
-import { scrapeUrl } from './scrape.js';
-import { serializeResult } from './serialize.js';
-import { resolveFormat, buildSerializeOptions } from './formats.js';
-import { formatForAgent } from './agent.js';
-import { randomPort } from './utils/port.js';
-import type { ScrapeOptions } from './types.js';
-
-const PORT = randomPort();
-
-const server = createServer(async (req, res) => {
-  res.setHeader('Access-Control-Allow-Origin', '*');
-  res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
-  res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
-
-  if (req.method === 'OPTIONS') {
-    res.writeHead(200);
-    res.end();
-    return;
-  }
-
-  if (req.method !== 'POST' || req.url !== '/api/scrape') {
-    res.writeHead(404);
-    res.end(JSON.stringify({ error: 'Not found' }));
-    return;
-  }
-
-  let body = '';
-  req.on('data', (chunk: Buffer) => { body += chunk.toString(); });
-  req.on('end', async () => {
-    try {
-      const { url, format: fmt, options = {} }: {
-        url: string;
-        format?: string;
-        options?: Record<string, unknown>;
-      } = JSON.parse(body);
-      if (!url) throw new Error('URL is required');
-
-      const format = resolveFormat(fmt || 'markdown');
-      const result = await scrapeUrl(url, {
-        timeout: 30000,
-        scroll: options.scroll as boolean,
-        extractLinks: (options.extractLinks as boolean) ?? true,
-        extractImages: (options.extractImages as boolean) ?? true,
-        extractTables: (options.extractTables as boolean) ?? true,
-      } as ScrapeOptions);
-
-      let response: Record<string, unknown>;
-
-      if (options.agent) {
-        const agentOutput = formatForAgent(result);
-        response = {
-          success: true,
-          title: result.title,
-          url: result.url,
-          duration: result.duration,
-          size: result.size,
-          agentOutput,
-        };
-      } else {
-        const serializeOpts = buildSerializeOptions(format);
-        const serialized = serializeResult(result, serializeOpts);
-        response = {
-          success: true,
-          title: result.title,
-          url: result.url,
-          format,
-          content: serialized.content,
-          duration: result.duration,
-          size: serialized.size,
-          metadata: result.metadata,
-          links: result.links,
-          images: result.images,
-          tables: result.tables,
-          result,
-        };
-      }
-
-      res.writeHead(200, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify(response));
-    } catch (err) {
-      const error = err as Error;
-      res.writeHead(500, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify({ error: error.message }));
-    }
-  });
-});
-
-server.listen(PORT, () => {
-  console.log(`API server running at http://localhost:${PORT}`);
-  console.log(`POST /api/scrape with { url, format?, options? }`);
-});

package/scrape/errors.ts

@@ -1,132 +0,0 @@
-export const ErrorCode = {
-  VALIDATION_FAILED: 'VALIDATION_FAILED',
-  INVALID_URL: 'INVALID_URL',
-  TIMEOUT: 'TIMEOUT',
-  BLOCKED: 'BLOCKED',
-  RATE_LIMITED: 'RATE_LIMITED',
-  PROXY_ERROR: 'PROXY_ERROR',
-  PARSE_ERROR: 'PARSE_ERROR',
-  AUTH_REQUIRED: 'AUTH_REQUIRED',
-  NETWORK_ERROR: 'NETWORK_ERROR',
-  BROWSER_NOT_AVAILABLE: 'BROWSER_NOT_AVAILABLE',
-  CRAWL_DEPTH_EXCEEDED: 'CRAWL_DEPTH_EXCEEDED',
-  MAX_RETRIES_EXCEEDED: 'MAX_RETRIES_EXCEEDED',
-} as const;
-
-export type ErrorCode = typeof ErrorCode[keyof typeof ErrorCode];
-
-export class WebScrapeError extends Error {
-  public readonly code: ErrorCode;
-  public readonly statusCode: number;
-  public readonly isRetryable: boolean;
-  public readonly timestamp: string;
-  public readonly details?: Record<string, unknown>;
-  public readonly cause?: Error;
-
-  constructor(
-    message: string,
-    code: ErrorCode,
-    statusCode: number,
-    isRetryable: boolean,
-    details?: Record<string, unknown>,
-    cause?: Error
-  ) {
-    super(message);
-    this.name = 'WebScrapeError';
-    this.code = code;
-    this.statusCode = statusCode;
-    this.isRetryable = isRetryable;
-    this.timestamp = new Date().toISOString();
-    this.details = details;
-    this.cause = cause;
-    Object.setPrototypeOf(this, new.target.prototype);
-    Error.captureStackTrace?.(this, this.constructor);
-  }
-
-  toJSON() {
-    return {
-      error: {
-        name: this.name,
-        code: this.code,
-        message: this.message,
-        statusCode: this.statusCode,
-        isRetryable: this.isRetryable,
-        timestamp: this.timestamp,
-        details: this.details,
-      },
-    };
-  }
-}
-
-export class ValidationError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.VALIDATION_FAILED, 400, false, details, cause);
-    this.name = 'ValidationError';
-  }
-}
-
-export class TimeoutError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.TIMEOUT, 504, true, details, cause);
-    this.name = 'TimeoutError';
-  }
-}
-
-export class BlockedError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.BLOCKED, 403, false, details, cause);
-    this.name = 'BlockedError';
-  }
-}
-
-export class RateLimitError extends WebScrapeError {
-  public readonly retryAfterMs?: number;
-  constructor(
-    message: string,
-    retryAfterMs?: number,
-    details?: Record<string, unknown>,
-    cause?: Error
-  ) {
-    super(message, ErrorCode.RATE_LIMITED, 429, true, { ...details, retryAfterMs }, cause);
-    this.name = 'RateLimitError';
-    this.retryAfterMs = retryAfterMs;
-  }
-}
-
-export class ProxyError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.PROXY_ERROR, 502, true, details, cause);
-    this.name = 'ProxyError';
-  }
-}
-
-export class ParseError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.PARSE_ERROR, 422, false, details, cause);
-    this.name = 'ParseError';
-  }
-}
-
-export class AuthError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.AUTH_REQUIRED, 401, false, details, cause);
-    this.name = 'AuthError';
-  }
-}
-
-export class NetworkError extends WebScrapeError {
-  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
-    super(message, ErrorCode.NETWORK_ERROR, 503, true, details, cause);
-    this.name = 'NetworkError';
-  }
-}
-
-export class BrowserNotAvailableError extends WebScrapeError {
-  constructor(
-    message: string = 'Playwright is not installed. Install it with: npm install playwright',
-    details?: Record<string, unknown>
-  ) {
-    super(message, ErrorCode.BROWSER_NOT_AVAILABLE, 500, false, details);
-    this.name = 'BrowserNotAvailableError';
-  }
-}

package/scrape/events.ts

@@ -1,26 +0,0 @@
-import Emittery from 'emittery';
-import type { ScrapeOptions } from './types.js';
-
-export interface WebScrapeEvents {
-  'request:start': { url: string; options: ScrapeOptions; timestamp: number };
-  'request:response': { url: string; status: number; attempt: number; duration: number };
-  'request:error': { url: string; error: Error; attempt: number };
-  'request:retry': { url: string; attempt: number; maxRetries: number; delayMs: number };
-  'request:complete': { url: string; duration: number; success: boolean };
-  'cache:hit': { key: string };
-  'cache:miss': { key: string };
-  'cache:set': { key: string; ttlMs?: number };
-  'proxy:rotate': { proxy: string; reason: string };
-  'proxy:error': { proxy: string; error: Error };
-  'proxy:disabled': { proxy: string; reason: string };
-  'crawl:discover': { url: string; depth: number; parentUrl?: string };
-  'crawl:complete': { totalUrls: number; successful: number; failed: number; duration: number };
-}
-
-export type EventEmitter = Emittery<WebScrapeEvents>;
-
-export function createEventEmitter(): EventEmitter {
-  return new Emittery<WebScrapeEvents>();
-}
-
-export const globalEmitter = createEventEmitter();

package/scrape/extract.js

@@ -1,14 +0,0 @@
-/**
- * structured first extraction uses small built in heuristics and returns serializable data.
- */
-export function extracthtml(html, url) {
-  const title = match(html, /<title[^>]*>([\s\S]*?)<\/title>/i);
-  const description = match(html, /<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']/i) ?? match(html, /<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["']/i);
-  const links = [...html.matchAll(/<a[^>]+href=["']([^"']+)["'][^>]*>/gi)].map((item) => resolveurl(item[1], url)).filter(Boolean);
-  const text = html.replace(/<script[\s\S]*?<\/script>/gi, " ").replace(/<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
-  return { url, title: decode(title ?? ""), description: decode(description ?? ""), text, links: [...new Set(links)] };
-}
-
-function match(value, expression) { return value.match(expression)?.[1]?.trim(); }
-function decode(value) { return value.replaceAll("&amp;", "&").replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&quot;", '"'); }
-function resolveurl(value, base) { try { return base ? new URL(value, base).href : new URL(value).href; } catch { return null; } }