@wenathlan/saddle 1.8.4 → 1.8.6

package/scrape/chunking.ts

@@ -0,0 +1,193 @@
+import { estimateTokens } from './tokens.js';
+
+export interface ChunkOptions {
+  maxTokens?: number;
+  overlapTokens?: number;
+  preserveCodeBlocks?: boolean;
+  includeHeadingPath?: boolean;
+}
+
+export interface Chunk {
+  content: string;
+  headingPath: string[];
+  tokenCount: number;
+  index: number;
+}
+
+const DEFAULT_CHUNK_OPTIONS: Required<ChunkOptions> = {
+  maxTokens: 512,
+  overlapTokens: 50,
+  preserveCodeBlocks: true,
+  includeHeadingPath: true,
+};
+
+function splitByHeaders(markdown: string): { heading: string; level: number; content: string }[] {
+  const sections: { heading: string; level: number; content: string }[] = [];
+  const lines = markdown.split('\n');
+  let currentHeading = '';
+  let currentLevel = 0;
+  let currentContent: string[] = [];
+
+  for (const line of lines) {
+    const headerMatch = line.match(/^(#{1,6})\s+(.+)/);
+    if (headerMatch) {
+      if (currentContent.length > 0 || currentHeading) {
+        sections.push({
+          heading: currentHeading,
+          level: currentLevel,
+          content: currentContent.join('\n').trim(),
+        });
+      }
+      currentLevel = headerMatch[1].length;
+      currentHeading = headerMatch[2];
+      currentContent = [];
+    } else {
+      currentContent.push(line);
+    }
+  }
+
+  if (currentContent.length > 0 || currentHeading) {
+    sections.push({
+      heading: currentHeading,
+      level: currentLevel,
+      content: currentContent.join('\n').trim(),
+    });
+  }
+
+  return sections;
+}
+
+function splitByParagraphs(text: string): string[] {
+  return text.split(/\n\n+/).filter(p => p.trim().length > 0);
+}
+
+function splitBySentences(text: string): string[] {
+  return text.match(/[^.!?]+[.!?]+/g) || [text];
+}
+
+export function chunkMarkdown(markdown: string, options: ChunkOptions = {}): Chunk[] {
+  const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
+  const chunks: Chunk[] = [];
+  const sections = splitByHeaders(markdown);
+  const headingPath: string[] = [];
+  let headingDepth = 0;
+
+  for (const section of sections) {
+    while (headingDepth > 0 && headingDepth >= section.level) {
+      headingDepth--;
+      headingPath.pop();
+    }
+    if (section.heading) {
+      headingDepth = section.level;
+      headingPath.push(section.heading);
+    }
+
+    const sectionText = section.heading ? `#${'#'.repeat(section.level - 1)} ${section.heading}\n\n${section.content}` : section.content;
+
+    if (estimateTokens(sectionText) <= opts.maxTokens) {
+      chunks.push({
+        content: sectionText,
+        headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+        tokenCount: estimateTokens(sectionText),
+        index: chunks.length,
+      });
+    } else {
+      const paragraphs = splitByParagraphs(sectionText);
+      let currentChunk = '';
+
+      for (const paragraph of paragraphs) {
+        const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
+
+        if (estimateTokens(testChunk) <= opts.maxTokens) {
+          currentChunk = testChunk;
+        } else {
+          if (currentChunk) {
+            chunks.push({
+              content: currentChunk,
+              headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+              tokenCount: estimateTokens(currentChunk),
+              index: chunks.length,
+            });
+          }
+
+          if (estimateTokens(paragraph) > opts.maxTokens) {
+            const sentences = splitBySentences(paragraph);
+            let sentenceChunk = '';
+            for (const sentence of sentences) {
+              const testSentence = sentenceChunk ? `${sentenceChunk} ${sentence}` : sentence;
+              if (estimateTokens(testSentence) <= opts.maxTokens) {
+                sentenceChunk = testSentence;
+              } else {
+                if (sentenceChunk) {
+                  chunks.push({
+                    content: sentenceChunk,
+                    headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+                    tokenCount: estimateTokens(sentenceChunk),
+                    index: chunks.length,
+                  });
+                }
+                sentenceChunk = sentence;
+              }
+            }
+            currentChunk = sentenceChunk;
+          } else {
+            currentChunk = paragraph;
+          }
+        }
+      }
+
+      if (currentChunk) {
+        chunks.push({
+          content: currentChunk,
+          headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+          tokenCount: estimateTokens(currentChunk),
+          index: chunks.length,
+        });
+      }
+    }
+  }
+
+  return chunks;
+}
+
+export function chunkText(text: string, options: ChunkOptions = {}): Chunk[] {
+  const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
+  const chunks: Chunk[] = [];
+  const paragraphs = splitByParagraphs(text);
+  let currentChunk = '';
+
+  for (const paragraph of paragraphs) {
+    const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
+    if (estimateTokens(testChunk) <= opts.maxTokens) {
+      currentChunk = testChunk;
+    } else {
+      if (currentChunk) {
+        chunks.push({
+          content: currentChunk,
+          headingPath: [],
+          tokenCount: estimateTokens(currentChunk),
+          index: chunks.length,
+        });
+      }
+      currentChunk = paragraph;
+    }
+  }
+
+  if (currentChunk) {
+    chunks.push({
+      content: currentChunk,
+      headingPath: [],
+      tokenCount: estimateTokens(currentChunk),
+      index: chunks.length,
+    });
+  }
+
+  return chunks;
+}
+
+export function formatChunksForRAG(chunks: Chunk[]): string {
+  return chunks.map(chunk => {
+    const heading = chunk.headingPath.length > 0 ? `## ${chunk.headingPath.join(' > ')}\n\n` : '';
+    return `${heading}${chunk.content}`;
+  }).join('\n\n---\n\n');
+}

package/scrape/cli.ts

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { readFileSync, existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { scrapeUrl, scrapeHtml } from './scrape.js';
+import { serializeResult } from './serialize.js';
+import { resolveFormat, extensionForFormat, buildSerializeOptions } from './formats.js';
+import { formatForAgent } from './agent.js';
+import { writeOutput } from './utils.js';
+import type { ScrapeOptions, ScrapeResult } from './types.js';
+
+const program = new Command();
+
+program
+  .name('webscrape')
+  .description('DevThink WebScrape — Universal AI-powered web scraping toolkit')
+  .version('2.0.0');
+
+program
+  .argument('[url]', 'URL to scrape')
+  .option('-f, --format <type>', 'Output format: markdown, html, text, json, xml, redis', 'markdown')
+  .option('-o, --output <file>', 'Write output to file')
+  .option('--file <path>', 'Scrape HTML from local file')
+  .option('--mode <mode>', 'Scraping mode: auto, fetch, browser', 'auto')
+  .option('--no-headless', 'Show browser window')
+  .option('--timeout <ms>', 'Navigation timeout', '30000')
+  .option('--scroll', 'Scroll page to load dynamic content')
+  .option('--readable', 'Extract readable content only')
+  .option('--agent', 'Format output for AI agent consumption')
+  .option('--pretty', 'Pretty-print JSON output')
+  .option('--proxy <url>', 'Proxy server URL')
+  .option('--retries <n>', 'Number of retries', '3')
+  .option('--user-agent <ua>', 'Custom User-Agent string')
+  .action(async (url, options) => {
+    if (!url && !options.file) {
+      program.help();
+    }
+
+    const format = resolveFormat(options.format);
+    const scrapeOpts: ScrapeOptions = {
+      format,
+      timeout: parseInt(options.timeout),
+      scroll: options.scroll,
+      readable: options.readable,
+      mode: options.mode,
+      proxy: options.proxy,
+      retries: parseInt(options.retries),
+      userAgent: options.userAgent,
+      extractLinks: true,
+      extractImages: true,
+      extractTables: true,
+    };
+
+    if (options.readable) {
+      scrapeOpts.removeSelectors = [
+        'script', 'style', 'nav', 'footer', 'header',
+        '.sidebar', '.advertisement', '.ads', '.menu',
+        '.comments', '.comment', '#comments',
+      ];
+    }
+
+    console.error(`Scraping: ${url || options.file}`);
+
+    let result: ScrapeResult;
+
+    if (options.file) {
+      const filePath = resolve(options.file);
+      if (!existsSync(filePath)) {
+        console.error(`File not found: ${filePath}`);
+        process.exit(1);
+      }
+      const html = readFileSync(filePath, 'utf-8');
+      result = await scrapeHtml(html, scrapeOpts);
+    } else {
+      result = await scrapeUrl(url, scrapeOpts);
+    }
+
+    console.error(`Done in ${result.duration}ms (${(result.size / 1024).toFixed(1)} KB)`);
+
+    if (options.agent) {
+      const agentOutput = formatForAgent(result);
+      const output = JSON.stringify(agentOutput, null, 2);
+
+      if (options.output) {
+        writeOutput(options.output, output);
+        console.error(`Saved to ${options.output}`);
+      } else {
+        console.log(output);
+      }
+    } else {
+      const serializeOpts = buildSerializeOptions(format, options.pretty);
+      const serialized = serializeResult(result, serializeOpts);
+
+      if (options.output) {
+        const ext = extensionForFormat(format);
+        const finalPath = options.output.endsWith(ext) ? options.output : options.output + ext;
+        writeOutput(finalPath, serialized.content);
+        console.error(`Saved to ${finalPath}`);
+      } else {
+        console.log(serialized.content);
+      }
+    }
+  });
+
+program.parse();

package/scrape/crawler.ts

@@ -0,0 +1,115 @@
+import type { ScrapeOptions, ScrapeResult, CrawlStats } from './types.js';
+import { scrapeUrl } from './scrape.js';
+
+export interface CrawlOptions extends ScrapeOptions {
+  maxDepth?: number;
+  maxPages?: number;
+  maxConcurrent?: number;
+  sameDomain?: boolean;
+  delayMs?: number;
+  onDiscover?: (url: string, depth: number) => void;
+  onResult?: (result: ScrapeResult) => void;
+  onError?: (url: string, error: Error) => void;
+}
+
+interface CrawlEntry {
+  url: string;
+  depth: number;
+}
+
+function isSameDomain(url1: string, url2: string): boolean {
+  try {
+    return new URL(url1).hostname === new URL(url2).hostname;
+  } catch {
+    return false;
+  }
+}
+
+function normalizeForDedup(url: string): string {
+  try {
+    const u = new URL(url);
+    u.hash = '';
+    u.search = '';
+    return u.href.replace(/\/+$/, '') || u.href;
+  } catch {
+    return url;
+  }
+}
+
+export async function crawl(
+  startUrl: string,
+  options: CrawlOptions = {}
+): Promise<{ results: ScrapeResult[]; stats: CrawlStats }> {
+  const {
+    maxDepth = 2,
+    maxPages = 50,
+    maxConcurrent = 3,
+    sameDomain = true,
+    delayMs = 1000,
+    onDiscover,
+    onResult,
+    onError,
+    ...scrapeOpts
+  } = options;
+
+  const startTime = Date.now();
+  const visited = new Set<string>();
+  const queue: CrawlEntry[] = [{ url: startUrl, depth: 0 }];
+  const results: ScrapeResult[] = [];
+  let failed = 0;
+
+  visited.add(normalizeForDedup(startUrl));
+
+  while (queue.length > 0 && results.length < maxPages) {
+    const batch = queue.splice(0, maxConcurrent);
+    const promises = batch.map(async (entry) => {
+      if (entry.depth > maxDepth) return;
+
+      try {
+        const result = await scrapeUrl(entry.url, scrapeOpts);
+        results.push(result);
+        onResult?.(result);
+
+        // Extract links for further crawling
+        if (entry.depth < maxDepth) {
+          for (const link of result.links) {
+            const normalized = normalizeForDedup(link.href);
+            if (visited.has(normalized)) continue;
+            if (sameDomain && !isSameDomain(startUrl, link.href)) continue;
+            try {
+              new URL(link.href);
+            } catch {
+              continue;
+            }
+            visited.add(normalized);
+            queue.push({ url: link.href, depth: entry.depth + 1 });
+            onDiscover?.(link.href, entry.depth + 1);
+          }
+        }
+      } catch (error) {
+        failed++;
+        onError?.(entry.url, error as Error);
+      }
+    });
+
+    await Promise.all(promises);
+
+    if (delayMs > 0 && queue.length > 0) {
+      await new Promise(r => setTimeout(r, delayMs));
+    }
+  }
+
+  const duration = Date.now() - startTime;
+
+  return {
+    results,
+    stats: {
+      totalUrls: visited.size,
+      successful: results.length,
+      failed,
+      skipped: visited.size - results.length - failed,
+      duration,
+      avgResponseTime: results.length > 0 ? results.reduce((sum, r) => sum + r.duration, 0) / results.length : 0,
+    },
+  };
+}

package/scrape/dev-server.ts

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+import { createServer } from 'node:http';
+
+import { scrapeUrl } from './scrape.js';
+import { serializeResult } from './serialize.js';
+import { resolveFormat, buildSerializeOptions } from './formats.js';
+import { formatForAgent } from './agent.js';
+import { randomPort } from './utils/port.js';
+import type { ScrapeOptions } from './types.js';
+
+const PORT = randomPort();
+
+const server = createServer(async (req, res) => {
+  res.setHeader('Access-Control-Allow-Origin', '*');
+  res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
+  res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+  if (req.method === 'OPTIONS') {
+    res.writeHead(200);
+    res.end();
+    return;
+  }
+
+  if (req.method !== 'POST' || req.url !== '/api/scrape') {
+    res.writeHead(404);
+    res.end(JSON.stringify({ error: 'Not found' }));
+    return;
+  }
+
+  let body = '';
+  req.on('data', (chunk: Buffer) => { body += chunk.toString(); });
+  req.on('end', async () => {
+    try {
+      const { url, format: fmt, options = {} }: {
+        url: string;
+        format?: string;
+        options?: Record<string, unknown>;
+      } = JSON.parse(body);
+      if (!url) throw new Error('URL is required');
+
+      const format = resolveFormat(fmt || 'markdown');
+      const result = await scrapeUrl(url, {
+        timeout: 30000,
+        scroll: options.scroll as boolean,
+        extractLinks: (options.extractLinks as boolean) ?? true,
+        extractImages: (options.extractImages as boolean) ?? true,
+        extractTables: (options.extractTables as boolean) ?? true,
+      } as ScrapeOptions);
+
+      let response: Record<string, unknown>;
+
+      if (options.agent) {
+        const agentOutput = formatForAgent(result);
+        response = {
+          success: true,
+          title: result.title,
+          url: result.url,
+          duration: result.duration,
+          size: result.size,
+          agentOutput,
+        };
+      } else {
+        const serializeOpts = buildSerializeOptions(format);
+        const serialized = serializeResult(result, serializeOpts);
+        response = {
+          success: true,
+          title: result.title,
+          url: result.url,
+          format,
+          content: serialized.content,
+          duration: result.duration,
+          size: serialized.size,
+          metadata: result.metadata,
+          links: result.links,
+          images: result.images,
+          tables: result.tables,
+          result,
+        };
+      }
+
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify(response));
+    } catch (err) {
+      const error = err as Error;
+      res.writeHead(500, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ error: error.message }));
+    }
+  });
+});
+
+server.listen(PORT, () => {
+  console.log(`API server running at http://localhost:${PORT}`);
+  console.log(`POST /api/scrape with { url, format?, options? }`);
+});

package/scrape/errors.ts

@@ -0,0 +1,132 @@
+export const ErrorCode = {
+  VALIDATION_FAILED: 'VALIDATION_FAILED',
+  INVALID_URL: 'INVALID_URL',
+  TIMEOUT: 'TIMEOUT',
+  BLOCKED: 'BLOCKED',
+  RATE_LIMITED: 'RATE_LIMITED',
+  PROXY_ERROR: 'PROXY_ERROR',
+  PARSE_ERROR: 'PARSE_ERROR',
+  AUTH_REQUIRED: 'AUTH_REQUIRED',
+  NETWORK_ERROR: 'NETWORK_ERROR',
+  BROWSER_NOT_AVAILABLE: 'BROWSER_NOT_AVAILABLE',
+  CRAWL_DEPTH_EXCEEDED: 'CRAWL_DEPTH_EXCEEDED',
+  MAX_RETRIES_EXCEEDED: 'MAX_RETRIES_EXCEEDED',
+} as const;
+
+export type ErrorCode = typeof ErrorCode[keyof typeof ErrorCode];
+
+export class WebScrapeError extends Error {
+  public readonly code: ErrorCode;
+  public readonly statusCode: number;
+  public readonly isRetryable: boolean;
+  public readonly timestamp: string;
+  public readonly details?: Record<string, unknown>;
+  public readonly cause?: Error;
+
+  constructor(
+    message: string,
+    code: ErrorCode,
+    statusCode: number,
+    isRetryable: boolean,
+    details?: Record<string, unknown>,
+    cause?: Error
+  ) {
+    super(message);
+    this.name = 'WebScrapeError';
+    this.code = code;
+    this.statusCode = statusCode;
+    this.isRetryable = isRetryable;
+    this.timestamp = new Date().toISOString();
+    this.details = details;
+    this.cause = cause;
+    Object.setPrototypeOf(this, new.target.prototype);
+    Error.captureStackTrace?.(this, this.constructor);
+  }
+
+  toJSON() {
+    return {
+      error: {
+        name: this.name,
+        code: this.code,
+        message: this.message,
+        statusCode: this.statusCode,
+        isRetryable: this.isRetryable,
+        timestamp: this.timestamp,
+        details: this.details,
+      },
+    };
+  }
+}
+
+export class ValidationError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.VALIDATION_FAILED, 400, false, details, cause);
+    this.name = 'ValidationError';
+  }
+}
+
+export class TimeoutError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.TIMEOUT, 504, true, details, cause);
+    this.name = 'TimeoutError';
+  }
+}
+
+export class BlockedError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.BLOCKED, 403, false, details, cause);
+    this.name = 'BlockedError';
+  }
+}
+
+export class RateLimitError extends WebScrapeError {
+  public readonly retryAfterMs?: number;
+  constructor(
+    message: string,
+    retryAfterMs?: number,
+    details?: Record<string, unknown>,
+    cause?: Error
+  ) {
+    super(message, ErrorCode.RATE_LIMITED, 429, true, { ...details, retryAfterMs }, cause);
+    this.name = 'RateLimitError';
+    this.retryAfterMs = retryAfterMs;
+  }
+}
+
+export class ProxyError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.PROXY_ERROR, 502, true, details, cause);
+    this.name = 'ProxyError';
+  }
+}
+
+export class ParseError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.PARSE_ERROR, 422, false, details, cause);
+    this.name = 'ParseError';
+  }
+}
+
+export class AuthError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.AUTH_REQUIRED, 401, false, details, cause);
+    this.name = 'AuthError';
+  }
+}
+
+export class NetworkError extends WebScrapeError {
+  constructor(message: string, details?: Record<string, unknown>, cause?: Error) {
+    super(message, ErrorCode.NETWORK_ERROR, 503, true, details, cause);
+    this.name = 'NetworkError';
+  }
+}
+
+export class BrowserNotAvailableError extends WebScrapeError {
+  constructor(
+    message: string = 'Playwright is not installed. Install it with: npm install playwright',
+    details?: Record<string, unknown>
+  ) {
+    super(message, ErrorCode.BROWSER_NOT_AVAILABLE, 500, false, details);
+    this.name = 'BrowserNotAvailableError';
+  }
+}

package/scrape/events.ts

@@ -0,0 +1,26 @@
+import Emittery from 'emittery';
+import type { ScrapeOptions } from './types.js';
+
+export interface WebScrapeEvents {
+  'request:start': { url: string; options: ScrapeOptions; timestamp: number };
+  'request:response': { url: string; status: number; attempt: number; duration: number };
+  'request:error': { url: string; error: Error; attempt: number };
+  'request:retry': { url: string; attempt: number; maxRetries: number; delayMs: number };
+  'request:complete': { url: string; duration: number; success: boolean };
+  'cache:hit': { key: string };
+  'cache:miss': { key: string };
+  'cache:set': { key: string; ttlMs?: number };
+  'proxy:rotate': { proxy: string; reason: string };
+  'proxy:error': { proxy: string; error: Error };
+  'proxy:disabled': { proxy: string; reason: string };
+  'crawl:discover': { url: string; depth: number; parentUrl?: string };
+  'crawl:complete': { totalUrls: number; successful: number; failed: number; duration: number };
+}
+
+export type EventEmitter = Emittery<WebScrapeEvents>;
+
+export function createEventEmitter(): EventEmitter {
+  return new Emittery<WebScrapeEvents>();
+}
+
+export const globalEmitter = createEventEmitter();