@wenathlan/saddle 1.8.2 → 1.8.5

package/scrape/browser.ts

@@ -0,0 +1,222 @@
+import { chromium, firefox, webkit, type Browser, type BrowserContext, type Page } from 'playwright';
+import type { BrowserAgentConfig, ChromeCommand, RenderOptions } from './types.js';
+import { WebScrapeError, ErrorCode, BrowserNotAvailableError } from './errors.js';
+
+export class AgentBrowser {
+  private browser: Browser | null = null;
+  private context: BrowserContext | null = null;
+  private page: Page | null = null;
+  private config: Required<Omit<BrowserAgentConfig, 'userAgent' | 'geolocation'>> & Pick<BrowserAgentConfig, 'userAgent' | 'geolocation'>;
+
+  private static DEFAULTS = {
+    headless: true,
+    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    viewport: { width: 1280, height: 720 },
+    locale: 'en-US',
+    timeout: 30000,
+    recordVideo: false,
+    proxy: '',
+    storageState: '',
+    blockAds: true,
+    stealth: true,
+  };
+
+  constructor(config: Partial<BrowserAgentConfig> = {}) {
+    this.config = { ...AgentBrowser.DEFAULTS, ...config };
+  }
+
+  async launch(browserType: 'chromium' | 'firefox' | 'webkit' = 'chromium'): Promise<void> {
+    try {
+      const launcher = { chromium, firefox, webkit }[browserType];
+      const launchOpts: Parameters<typeof chromium.launch>[0] = {
+        headless: this.config.headless,
+        args: [
+          '--no-sandbox',
+          '--disable-setuid-sandbox',
+          '--disable-dev-shm-usage',
+          '--disable-web-security',
+          '--disable-features=IsolateOrigins,site-per-process',
+        ],
+      };
+      if (this.config.proxy) {
+        launchOpts.proxy = { server: this.config.proxy };
+      }
+      this.browser = await launcher.launch(launchOpts);
+
+      const ctxOpts: Parameters<Browser['newContext']>[0] = {
+        viewport: this.config.viewport,
+        locale: this.config.locale,
+        userAgent: this.config.userAgent,
+      };
+      if (this.config.recordVideo) {
+        ctxOpts.recordVideo = { dir: './videos' };
+      }
+      if (this.config.geolocation) {
+        ctxOpts.geolocation = this.config.geolocation;
+        ctxOpts.permissions = ['geolocation'];
+      }
+      if (this.config.storageState) {
+        ctxOpts.storageState = this.config.storageState;
+      }
+      this.context = await this.browser.newContext(ctxOpts);
+
+      if (this.config.blockAds) {
+        await this.context.route('**/*.{png,jpg,jpeg,gif,svg,ico,css,woff,woff2,mp4,webm}', r => r.abort());
+      }
+
+      this.page = await this.context.newPage();
+      this.page.setDefaultTimeout(this.config.timeout);
+    } catch (error) {
+      if ((error as Error).message?.includes('playwright') || (error as Error).message?.includes('chromium')) {
+        throw new BrowserNotAvailableError();
+      }
+      throw error;
+    }
+  }
+
+  async navigate(url: string, waitUntil: 'load' | 'domcontentloaded' | 'networkidle' = 'networkidle'): Promise<void> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    await this.page.goto(url, { waitUntil, timeout: this.config.timeout });
+  }
+
+  async screenshot(opts: RenderOptions = {}): Promise<Buffer> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    return this.page.screenshot({
+      fullPage: opts.fullPage ?? false,
+      type: opts.type ?? 'png',
+      quality: opts.quality ?? 80,
+    }) as Promise<Buffer>;
+  }
+
+  async evaluate<T = unknown>(fn: string | (() => T), args?: unknown[]): Promise<T> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    if (typeof fn === 'string') {
+      return this.page.evaluate(new Function(fn) as () => T);
+    }
+    return this.page.evaluate(fn, args);
+  }
+
+  async click(selector: string): Promise<void> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    await this.page.click(selector);
+  }
+
+  async type(selector: string, text: string, delayMs = 0): Promise<void> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    await this.page.fill(selector, '');
+    await this.page.type(selector, text, { delay: delayMs });
+  }
+
+  async scrollToBottom(step = 800, delayMs = 300, maxScrolls = 50): Promise<number> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    let scrolls = 0;
+    let prevHeight = 0;
+    while (scrolls < maxScrolls) {
+      const height = await this.page.evaluate(() => document.body.scrollHeight);
+      if (height === prevHeight && scrolls > 2) break;
+      prevHeight = height;
+      await this.page.evaluate(y => window.scrollBy(0, y), step);
+      await new Promise(r => setTimeout(r, delayMs));
+      scrolls++;
+    }
+    return scrolls;
+  }
+
+  async scrollTo(x: number, y: number): Promise<void> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    await this.page.evaluate(([px, py]) => window.scrollTo(px, py), [x, y]);
+  }
+
+  async waitFor(selector: string, timeout?: number): Promise<void> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    await this.page.waitForSelector(selector, { timeout: timeout ?? this.config.timeout });
+  }
+
+  async waitForLoad(): Promise<void> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    await this.page.waitForLoadState('networkidle');
+  }
+
+  async html(): Promise<string> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    return this.page.content();
+  }
+
+  async text(): Promise<string> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    return this.page.evaluate(() => document.body.innerText);
+  }
+
+  async title(): Promise<string> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    return this.page.title();
+  }
+
+  async url(): Promise<string> {
+    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
+    return this.page.url();
+  }
+
+  async executeCommands(commands: ChromeCommand[]): Promise<unknown[]> {
+    const results: unknown[] = [];
+    for (const cmd of commands) {
+      switch (cmd.action) {
+        case 'goto':
+          await this.navigate(cmd.args?.url as string);
+          results.push({ ok: true });
+          break;
+        case 'click':
+          await this.click(cmd.args?.selector as string);
+          results.push({ ok: true });
+          break;
+        case 'type':
+          await this.type(cmd.args?.selector as string, cmd.args?.text as string);
+          results.push({ ok: true });
+          break;
+        case 'screenshot':
+          results.push(await this.screenshot(cmd.args as RenderOptions));
+          break;
+        case 'evaluate':
+          results.push(await this.evaluate(cmd.args?.fn as string));
+          break;
+        case 'wait':
+          await this.waitFor(cmd.args?.selector as string);
+          results.push({ ok: true });
+          break;
+        case 'scroll':
+          if (cmd.args?.to) {
+            await this.scrollTo(0, cmd.args.to as number);
+          } else {
+            await this.scrollToBottom();
+          }
+          results.push({ ok: true });
+          break;
+        case 'extract':
+          results.push({
+            html: await this.html(),
+            text: await this.text(),
+            title: await this.title(),
+          });
+          break;
+      }
+    }
+    return results;
+  }
+
+  async close(): Promise<void> {
+    if (this.page) await this.page.close().catch(() => {});
+    if (this.context) await this.context.close().catch(() => {});
+    if (this.browser) await this.browser.close().catch(() => {});
+    this.page = null;
+    this.context = null;
+    this.browser = null;
+  }
+
+  isConnected(): boolean {
+    return this.browser?.isConnected() ?? false;
+  }
+}
+
+export function createBrowser(config?: BrowserAgentConfig): AgentBrowser {
+  return new AgentBrowser(config);
+}

package/scrape/cache.ts

@@ -0,0 +1,84 @@
+import { LRUCache } from 'lru-cache';
+
+export interface CacheEntry<T = unknown> {
+  value: T;
+  timestamp: number;
+  expires?: number;
+}
+
+export interface CacheConfig {
+  maxSize?: number;
+  defaultTtlMs?: number;
+  checkPeriodMs?: number;
+}
+
+const DEFAULT_CONFIG: Required<CacheConfig> = {
+  maxSize: 1000,
+  defaultTtlMs: 5 * 60 * 1000,
+  checkPeriodMs: 60 * 1000,
+};
+
+export class WebScrapeCache {
+  private store: LRUCache<string, CacheEntry>;
+
+  constructor(config: Partial<CacheConfig> = {}) {
+    const cfg = { ...DEFAULT_CONFIG, ...config };
+    this.store = new LRUCache<string, CacheEntry>({
+      max: cfg.maxSize,
+      ttl: cfg.defaultTtlMs,
+      updateAgeOnGet: true,
+    });
+  }
+
+  get<T = unknown>(key: string): T | undefined {
+    const entry = this.store.get(key);
+    if (!entry) return undefined;
+    if (entry.expires && Date.now() > entry.expires) {
+      this.store.delete(key);
+      return undefined;
+    }
+    return entry.value as T;
+  }
+
+  set<T = unknown>(key: string, value: T, ttlMs?: number): void {
+    const entry: CacheEntry<T> = {
+      value,
+      timestamp: Date.now(),
+      expires: ttlMs ? Date.now() + ttlMs : undefined,
+    };
+    this.store.set(key, entry, { ttl: ttlMs });
+  }
+
+  has(key: string): boolean {
+    return this.get(key) !== undefined;
+  }
+
+  delete(key: string): boolean {
+    return this.store.delete(key);
+  }
+
+  clear(): void {
+    this.store.clear();
+  }
+
+  getOrSet<T = unknown>(key: string, factory: () => Promise<T>, ttlMs?: number): Promise<T> {
+    const cached = this.get<T>(key);
+    if (cached !== undefined) return Promise.resolve(cached);
+    return factory().then(value => {
+      this.set(key, value, ttlMs);
+      return value;
+    });
+  }
+
+  size(): number {
+    return this.store.size;
+  }
+
+  keys(): string[] {
+    return [...this.store.keys()];
+  }
+}
+
+export function createCache(config?: Partial<CacheConfig>): WebScrapeCache {
+  return new WebScrapeCache(config);
+}

package/scrape/chunking.ts

@@ -0,0 +1,193 @@
+import { estimateTokens } from './tokens.js';
+
+export interface ChunkOptions {
+  maxTokens?: number;
+  overlapTokens?: number;
+  preserveCodeBlocks?: boolean;
+  includeHeadingPath?: boolean;
+}
+
+export interface Chunk {
+  content: string;
+  headingPath: string[];
+  tokenCount: number;
+  index: number;
+}
+
+const DEFAULT_CHUNK_OPTIONS: Required<ChunkOptions> = {
+  maxTokens: 512,
+  overlapTokens: 50,
+  preserveCodeBlocks: true,
+  includeHeadingPath: true,
+};
+
+function splitByHeaders(markdown: string): { heading: string; level: number; content: string }[] {
+  const sections: { heading: string; level: number; content: string }[] = [];
+  const lines = markdown.split('\n');
+  let currentHeading = '';
+  let currentLevel = 0;
+  let currentContent: string[] = [];
+
+  for (const line of lines) {
+    const headerMatch = line.match(/^(#{1,6})\s+(.+)/);
+    if (headerMatch) {
+      if (currentContent.length > 0 || currentHeading) {
+        sections.push({
+          heading: currentHeading,
+          level: currentLevel,
+          content: currentContent.join('\n').trim(),
+        });
+      }
+      currentLevel = headerMatch[1].length;
+      currentHeading = headerMatch[2];
+      currentContent = [];
+    } else {
+      currentContent.push(line);
+    }
+  }
+
+  if (currentContent.length > 0 || currentHeading) {
+    sections.push({
+      heading: currentHeading,
+      level: currentLevel,
+      content: currentContent.join('\n').trim(),
+    });
+  }
+
+  return sections;
+}
+
+function splitByParagraphs(text: string): string[] {
+  return text.split(/\n\n+/).filter(p => p.trim().length > 0);
+}
+
+function splitBySentences(text: string): string[] {
+  return text.match(/[^.!?]+[.!?]+/g) || [text];
+}
+
+export function chunkMarkdown(markdown: string, options: ChunkOptions = {}): Chunk[] {
+  const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
+  const chunks: Chunk[] = [];
+  const sections = splitByHeaders(markdown);
+  const headingPath: string[] = [];
+  let headingDepth = 0;
+
+  for (const section of sections) {
+    while (headingDepth > 0 && headingDepth >= section.level) {
+      headingDepth--;
+      headingPath.pop();
+    }
+    if (section.heading) {
+      headingDepth = section.level;
+      headingPath.push(section.heading);
+    }
+
+    const sectionText = section.heading ? `#${'#'.repeat(section.level - 1)} ${section.heading}\n\n${section.content}` : section.content;
+
+    if (estimateTokens(sectionText) <= opts.maxTokens) {
+      chunks.push({
+        content: sectionText,
+        headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+        tokenCount: estimateTokens(sectionText),
+        index: chunks.length,
+      });
+    } else {
+      const paragraphs = splitByParagraphs(sectionText);
+      let currentChunk = '';
+
+      for (const paragraph of paragraphs) {
+        const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
+
+        if (estimateTokens(testChunk) <= opts.maxTokens) {
+          currentChunk = testChunk;
+        } else {
+          if (currentChunk) {
+            chunks.push({
+              content: currentChunk,
+              headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+              tokenCount: estimateTokens(currentChunk),
+              index: chunks.length,
+            });
+          }
+
+          if (estimateTokens(paragraph) > opts.maxTokens) {
+            const sentences = splitBySentences(paragraph);
+            let sentenceChunk = '';
+            for (const sentence of sentences) {
+              const testSentence = sentenceChunk ? `${sentenceChunk} ${sentence}` : sentence;
+              if (estimateTokens(testSentence) <= opts.maxTokens) {
+                sentenceChunk = testSentence;
+              } else {
+                if (sentenceChunk) {
+                  chunks.push({
+                    content: sentenceChunk,
+                    headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+                    tokenCount: estimateTokens(sentenceChunk),
+                    index: chunks.length,
+                  });
+                }
+                sentenceChunk = sentence;
+              }
+            }
+            currentChunk = sentenceChunk;
+          } else {
+            currentChunk = paragraph;
+          }
+        }
+      }
+
+      if (currentChunk) {
+        chunks.push({
+          content: currentChunk,
+          headingPath: opts.includeHeadingPath ? [...headingPath] : [],
+          tokenCount: estimateTokens(currentChunk),
+          index: chunks.length,
+        });
+      }
+    }
+  }
+
+  return chunks;
+}
+
+export function chunkText(text: string, options: ChunkOptions = {}): Chunk[] {
+  const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
+  const chunks: Chunk[] = [];
+  const paragraphs = splitByParagraphs(text);
+  let currentChunk = '';
+
+  for (const paragraph of paragraphs) {
+    const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
+    if (estimateTokens(testChunk) <= opts.maxTokens) {
+      currentChunk = testChunk;
+    } else {
+      if (currentChunk) {
+        chunks.push({
+          content: currentChunk,
+          headingPath: [],
+          tokenCount: estimateTokens(currentChunk),
+          index: chunks.length,
+        });
+      }
+      currentChunk = paragraph;
+    }
+  }
+
+  if (currentChunk) {
+    chunks.push({
+      content: currentChunk,
+      headingPath: [],
+      tokenCount: estimateTokens(currentChunk),
+      index: chunks.length,
+    });
+  }
+
+  return chunks;
+}
+
+export function formatChunksForRAG(chunks: Chunk[]): string {
+  return chunks.map(chunk => {
+    const heading = chunk.headingPath.length > 0 ? `## ${chunk.headingPath.join(' > ')}\n\n` : '';
+    return `${heading}${chunk.content}`;
+  }).join('\n\n---\n\n');
+}

package/scrape/cli.ts

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { readFileSync, existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { scrapeUrl, scrapeHtml } from './scrape.js';
+import { serializeResult } from './serialize.js';
+import { resolveFormat, extensionForFormat, buildSerializeOptions } from './formats.js';
+import { formatForAgent } from './agent.js';
+import { writeOutput } from './utils.js';
+import type { ScrapeOptions, ScrapeResult } from './types.js';
+
+const program = new Command();
+
+program
+  .name('webscrape')
+  .description('DevThink WebScrape — Universal AI-powered web scraping toolkit')
+  .version('2.0.0');
+
+program
+  .argument('[url]', 'URL to scrape')
+  .option('-f, --format <type>', 'Output format: markdown, html, text, json, xml, redis', 'markdown')
+  .option('-o, --output <file>', 'Write output to file')
+  .option('--file <path>', 'Scrape HTML from local file')
+  .option('--mode <mode>', 'Scraping mode: auto, fetch, browser', 'auto')
+  .option('--no-headless', 'Show browser window')
+  .option('--timeout <ms>', 'Navigation timeout', '30000')
+  .option('--scroll', 'Scroll page to load dynamic content')
+  .option('--readable', 'Extract readable content only')
+  .option('--agent', 'Format output for AI agent consumption')
+  .option('--pretty', 'Pretty-print JSON output')
+  .option('--proxy <url>', 'Proxy server URL')
+  .option('--retries <n>', 'Number of retries', '3')
+  .option('--user-agent <ua>', 'Custom User-Agent string')
+  .action(async (url, options) => {
+    if (!url && !options.file) {
+      program.help();
+    }
+
+    const format = resolveFormat(options.format);
+    const scrapeOpts: ScrapeOptions = {
+      format,
+      timeout: parseInt(options.timeout),
+      scroll: options.scroll,
+      readable: options.readable,
+      mode: options.mode,
+      proxy: options.proxy,
+      retries: parseInt(options.retries),
+      userAgent: options.userAgent,
+      extractLinks: true,
+      extractImages: true,
+      extractTables: true,
+    };
+
+    if (options.readable) {
+      scrapeOpts.removeSelectors = [
+        'script', 'style', 'nav', 'footer', 'header',
+        '.sidebar', '.advertisement', '.ads', '.menu',
+        '.comments', '.comment', '#comments',
+      ];
+    }
+
+    console.error(`Scraping: ${url || options.file}`);
+
+    let result: ScrapeResult;
+
+    if (options.file) {
+      const filePath = resolve(options.file);
+      if (!existsSync(filePath)) {
+        console.error(`File not found: ${filePath}`);
+        process.exit(1);
+      }
+      const html = readFileSync(filePath, 'utf-8');
+      result = await scrapeHtml(html, scrapeOpts);
+    } else {
+      result = await scrapeUrl(url, scrapeOpts);
+    }
+
+    console.error(`Done in ${result.duration}ms (${(result.size / 1024).toFixed(1)} KB)`);
+
+    if (options.agent) {
+      const agentOutput = formatForAgent(result);
+      const output = JSON.stringify(agentOutput, null, 2);
+
+      if (options.output) {
+        writeOutput(options.output, output);
+        console.error(`Saved to ${options.output}`);
+      } else {
+        console.log(output);
+      }
+    } else {
+      const serializeOpts = buildSerializeOptions(format, options.pretty);
+      const serialized = serializeResult(result, serializeOpts);
+
+      if (options.output) {
+        const ext = extensionForFormat(format);
+        const finalPath = options.output.endsWith(ext) ? options.output : options.output + ext;
+        writeOutput(finalPath, serialized.content);
+        console.error(`Saved to ${finalPath}`);
+      } else {
+        console.log(serialized.content);
+      }
+    }
+  });
+
+program.parse();