@wenathlan/saddle 1.8.8 → 1.8.9

package/scrape/chunking.ts

@@ -1,193 +0,0 @@
-import { estimateTokens } from './tokens.js';
-
-export interface ChunkOptions {
-  maxTokens?: number;
-  overlapTokens?: number;
-  preserveCodeBlocks?: boolean;
-  includeHeadingPath?: boolean;
-}
-
-export interface Chunk {
-  content: string;
-  headingPath: string[];
-  tokenCount: number;
-  index: number;
-}
-
-const DEFAULT_CHUNK_OPTIONS: Required<ChunkOptions> = {
-  maxTokens: 512,
-  overlapTokens: 50,
-  preserveCodeBlocks: true,
-  includeHeadingPath: true,
-};
-
-function splitByHeaders(markdown: string): { heading: string; level: number; content: string }[] {
-  const sections: { heading: string; level: number; content: string }[] = [];
-  const lines = markdown.split('\n');
-  let currentHeading = '';
-  let currentLevel = 0;
-  let currentContent: string[] = [];
-
-  for (const line of lines) {
-    const headerMatch = line.match(/^(#{1,6})\s+(.+)/);
-    if (headerMatch) {
-      if (currentContent.length > 0 || currentHeading) {
-        sections.push({
-          heading: currentHeading,
-          level: currentLevel,
-          content: currentContent.join('\n').trim(),
-        });
-      }
-      currentLevel = headerMatch[1].length;
-      currentHeading = headerMatch[2];
-      currentContent = [];
-    } else {
-      currentContent.push(line);
-    }
-  }
-
-  if (currentContent.length > 0 || currentHeading) {
-    sections.push({
-      heading: currentHeading,
-      level: currentLevel,
-      content: currentContent.join('\n').trim(),
-    });
-  }
-
-  return sections;
-}
-
-function splitByParagraphs(text: string): string[] {
-  return text.split(/\n\n+/).filter(p => p.trim().length > 0);
-}
-
-function splitBySentences(text: string): string[] {
-  return text.match(/[^.!?]+[.!?]+/g) || [text];
-}
-
-export function chunkMarkdown(markdown: string, options: ChunkOptions = {}): Chunk[] {
-  const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
-  const chunks: Chunk[] = [];
-  const sections = splitByHeaders(markdown);
-  const headingPath: string[] = [];
-  let headingDepth = 0;
-
-  for (const section of sections) {
-    while (headingDepth > 0 && headingDepth >= section.level) {
-      headingDepth--;
-      headingPath.pop();
-    }
-    if (section.heading) {
-      headingDepth = section.level;
-      headingPath.push(section.heading);
-    }
-
-    const sectionText = section.heading ? `#${'#'.repeat(section.level - 1)} ${section.heading}\n\n${section.content}` : section.content;
-
-    if (estimateTokens(sectionText) <= opts.maxTokens) {
-      chunks.push({
-        content: sectionText,
-        headingPath: opts.includeHeadingPath ? [...headingPath] : [],
-        tokenCount: estimateTokens(sectionText),
-        index: chunks.length,
-      });
-    } else {
-      const paragraphs = splitByParagraphs(sectionText);
-      let currentChunk = '';
-
-      for (const paragraph of paragraphs) {
-        const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
-
-        if (estimateTokens(testChunk) <= opts.maxTokens) {
-          currentChunk = testChunk;
-        } else {
-          if (currentChunk) {
-            chunks.push({
-              content: currentChunk,
-              headingPath: opts.includeHeadingPath ? [...headingPath] : [],
-              tokenCount: estimateTokens(currentChunk),
-              index: chunks.length,
-            });
-          }
-
-          if (estimateTokens(paragraph) > opts.maxTokens) {
-            const sentences = splitBySentences(paragraph);
-            let sentenceChunk = '';
-            for (const sentence of sentences) {
-              const testSentence = sentenceChunk ? `${sentenceChunk} ${sentence}` : sentence;
-              if (estimateTokens(testSentence) <= opts.maxTokens) {
-                sentenceChunk = testSentence;
-              } else {
-                if (sentenceChunk) {
-                  chunks.push({
-                    content: sentenceChunk,
-                    headingPath: opts.includeHeadingPath ? [...headingPath] : [],
-                    tokenCount: estimateTokens(sentenceChunk),
-                    index: chunks.length,
-                  });
-                }
-                sentenceChunk = sentence;
-              }
-            }
-            currentChunk = sentenceChunk;
-          } else {
-            currentChunk = paragraph;
-          }
-        }
-      }
-
-      if (currentChunk) {
-        chunks.push({
-          content: currentChunk,
-          headingPath: opts.includeHeadingPath ? [...headingPath] : [],
-          tokenCount: estimateTokens(currentChunk),
-          index: chunks.length,
-        });
-      }
-    }
-  }
-
-  return chunks;
-}
-
-export function chunkText(text: string, options: ChunkOptions = {}): Chunk[] {
-  const opts = { ...DEFAULT_CHUNK_OPTIONS, ...options };
-  const chunks: Chunk[] = [];
-  const paragraphs = splitByParagraphs(text);
-  let currentChunk = '';
-
-  for (const paragraph of paragraphs) {
-    const testChunk = currentChunk ? `${currentChunk}\n\n${paragraph}` : paragraph;
-    if (estimateTokens(testChunk) <= opts.maxTokens) {
-      currentChunk = testChunk;
-    } else {
-      if (currentChunk) {
-        chunks.push({
-          content: currentChunk,
-          headingPath: [],
-          tokenCount: estimateTokens(currentChunk),
-          index: chunks.length,
-        });
-      }
-      currentChunk = paragraph;
-    }
-  }
-
-  if (currentChunk) {
-    chunks.push({
-      content: currentChunk,
-      headingPath: [],
-      tokenCount: estimateTokens(currentChunk),
-      index: chunks.length,
-    });
-  }
-
-  return chunks;
-}
-
-export function formatChunksForRAG(chunks: Chunk[]): string {
-  return chunks.map(chunk => {
-    const heading = chunk.headingPath.length > 0 ? `## ${chunk.headingPath.join(' > ')}\n\n` : '';
-    return `${heading}${chunk.content}`;
-  }).join('\n\n---\n\n');
-}

package/scrape/cli.ts

@@ -1,105 +0,0 @@
-#!/usr/bin/env node
-import { Command } from 'commander';
-import { readFileSync, existsSync } from 'node:fs';
-import { resolve } from 'node:path';
-import { scrapeUrl, scrapeHtml } from './scrape.js';
-import { serializeResult } from './serialize.js';
-import { resolveFormat, extensionForFormat, buildSerializeOptions } from './formats.js';
-import { formatForAgent } from './agent.js';
-import { writeOutput } from './utils.js';
-import type { ScrapeOptions, ScrapeResult } from './types.js';
-
-const program = new Command();
-
-program
-  .name('webscrape')
-  .description('DevThink WebScrape — Universal AI-powered web scraping toolkit')
-  .version('2.0.0');
-
-program
-  .argument('[url]', 'URL to scrape')
-  .option('-f, --format <type>', 'Output format: markdown, html, text, json, xml, redis', 'markdown')
-  .option('-o, --output <file>', 'Write output to file')
-  .option('--file <path>', 'Scrape HTML from local file')
-  .option('--mode <mode>', 'Scraping mode: auto, fetch, browser', 'auto')
-  .option('--no-headless', 'Show browser window')
-  .option('--timeout <ms>', 'Navigation timeout', '30000')
-  .option('--scroll', 'Scroll page to load dynamic content')
-  .option('--readable', 'Extract readable content only')
-  .option('--agent', 'Format output for AI agent consumption')
-  .option('--pretty', 'Pretty-print JSON output')
-  .option('--proxy <url>', 'Proxy server URL')
-  .option('--retries <n>', 'Number of retries', '3')
-  .option('--user-agent <ua>', 'Custom User-Agent string')
-  .action(async (url, options) => {
-    if (!url && !options.file) {
-      program.help();
-    }
-
-    const format = resolveFormat(options.format);
-    const scrapeOpts: ScrapeOptions = {
-      format,
-      timeout: parseInt(options.timeout),
-      scroll: options.scroll,
-      readable: options.readable,
-      mode: options.mode,
-      proxy: options.proxy,
-      retries: parseInt(options.retries),
-      userAgent: options.userAgent,
-      extractLinks: true,
-      extractImages: true,
-      extractTables: true,
-    };
-
-    if (options.readable) {
-      scrapeOpts.removeSelectors = [
-        'script', 'style', 'nav', 'footer', 'header',
-        '.sidebar', '.advertisement', '.ads', '.menu',
-        '.comments', '.comment', '#comments',
-      ];
-    }
-
-    console.error(`Scraping: ${url || options.file}`);
-
-    let result: ScrapeResult;
-
-    if (options.file) {
-      const filePath = resolve(options.file);
-      if (!existsSync(filePath)) {
-        console.error(`File not found: ${filePath}`);
-        process.exit(1);
-      }
-      const html = readFileSync(filePath, 'utf-8');
-      result = await scrapeHtml(html, scrapeOpts);
-    } else {
-      result = await scrapeUrl(url, scrapeOpts);
-    }
-
-    console.error(`Done in ${result.duration}ms (${(result.size / 1024).toFixed(1)} KB)`);
-
-    if (options.agent) {
-      const agentOutput = formatForAgent(result);
-      const output = JSON.stringify(agentOutput, null, 2);
-
-      if (options.output) {
-        writeOutput(options.output, output);
-        console.error(`Saved to ${options.output}`);
-      } else {
-        console.log(output);
-      }
-    } else {
-      const serializeOpts = buildSerializeOptions(format, options.pretty);
-      const serialized = serializeResult(result, serializeOpts);
-
-      if (options.output) {
-        const ext = extensionForFormat(format);
-        const finalPath = options.output.endsWith(ext) ? options.output : options.output + ext;
-        writeOutput(finalPath, serialized.content);
-        console.error(`Saved to ${finalPath}`);
-      } else {
-        console.log(serialized.content);
-      }
-    }
-  });
-
-program.parse();

package/scrape/crawl.js

@@ -1,104 +0,0 @@
-/**
- * scrape crawl context owns URL normalization traversal frontier and durable crawl state.
- *
- * The context keeps single page acquisition injectable while grouping the correlated
- * crawl responsibilities that previously lived in four separate top level files.
- */
-
-/** Removes fragments and tracking parameters before crawl deduplication. */
-export function normalizeurl(value) {
-  const url = new URL(value);
-  url.hash = "";
-  for (const key of [...url.searchParams.keys()]) if (/^(utm_|fbclid$|msclkid$|gclid$|gclsrc$|dclid$|gbraid$|wbraid$|twclid$|campaign$|content$|term$|source$|medium$|ref$|share_id$)/i.test(key)) url.searchParams.delete(key);
-  if (url.pathname.length > 1) url.pathname = url.pathname.replace(/\/+$/, "");
-  return url.href;
-}
-
-/** Compares two crawl targets by origin. */
-export function sameorigin(left, right) { return new URL(left).origin === new URL(right).origin; }
-
-/** Creates a bounded priority frontier for crawler and queue adapters. */
-export function crawlfrontier(options = {}) {
-  const maxpages = Number(options.maxpages ?? 20);
-  const maxperdomain = Number(options.maxperdomain ?? maxpages);
-  const queue = [];
-  const seen = new Set();
-  const completed = new Set();
-  const domains = new Map();
-
-  /** Adds a URL once while respecting the global page budget. */
-  function add(input = {}) {
-    const url = String(input.url ?? "");
-    if (!url || seen.has(url) || seen.size >= maxpages) return false;
-    seen.add(url);
-    queue.push({ url, depth: Number(input.depth ?? 0), priority: Number(input.priority ?? 0), discoveredat: Number(input.discoveredat ?? Date.now()) });
-    queue.sort((left, right) => right.priority - left.priority || left.discoveredat - right.discoveredat);
-    return true;
-  }
-
-  /** Removes the next URL that still fits its per-domain budget. */
-  function next() {
-    while (queue.length) {
-      const item = queue.shift();
-      const domain = new URL(item.url).hostname;
-      if ((domains.get(domain) ?? 0) >= maxperdomain) continue;
-      domains.set(domain, (domains.get(domain) ?? 0) + 1);
-      return { ...item };
-    }
-    return null;
-  }
-
-  /** Records a completed URL for diagnostics and persistence-friendly state. */
-  function complete(url) { completed.add(String(url)); }
-
-  /** Returns stable frontier diagnostics without exposing mutable collections. */
-  function state() { return { maxpages, maxperdomain, queued: queue.length, discovered: seen.size, completed: completed.size, domains: Object.fromEntries(domains) }; }
-
-  return { add, next, complete, state, list() { return queue.map((item) => ({ ...item })); } };
-}
-
-/** Creates a durable crawl queue around a caller-owned store. */
-export function persistentqueue(options = {}) {
-  const store = options.store;
-  const values = [];
-  const seen = new Set();
-
-  /** Restores unfinished crawl records from the injected store. */
-  async function restore() { if (typeof store?.list !== "function") return; for (const item of await store.list()) if (!seen.has(item.url) && item.status !== "done") { seen.add(item.url); values.push(item); } }
-
-  /** Adds a crawl record and persists it when the store supports writes. */
-  async function add(item) { if (!item?.url || seen.has(item.url)) return false; const value = { url: item.url, depth: item.depth ?? 0, status: "queued", createdat: Date.now(), metadata: item.metadata ?? {} }; seen.add(value.url); values.push(value); if (typeof store?.save === "function") await store.save(value); return true; }
-
-  /** Claims the next queued crawl record. */
-  async function next() { const item = values.find((value) => value.status === "queued"); if (!item) return null; item.status = "running"; if (typeof store?.update === "function") await store.update(item.url, item); return item; }
-
-  /** Completes a crawl record and persists the resulting status. */
-  async function complete(url, patch = {}) { const item = values.find((value) => value.url === url); if (!item) return null; Object.assign(item, patch, { status: patch.status ?? "done", processedat: Date.now() }); if (typeof store?.update === "function") await store.update(url, item); return item; }
-
-  return { restore, add, next, complete, list() { return values.map((value) => ({ ...value })); } };
-}
-
-/** Runs bounded breadth first traversal through the injected single page scrape contract. */
-export async function crawl(start, options = {}) {
-  const maxdepth = options.maxdepth ?? 1;
-  const maxpages = options.maxpages ?? 20;
-  const sameDomain = options.samedomain ?? true;
-  const frontier = crawlfrontier({ maxpages, maxperdomain: options.maxperdomain ?? maxpages });
-  frontier.add({ url: normalizeurl(start), depth: 0, priority: options.startpriority ?? 0 });
-  const results = [];
-  while (frontier.state().queued && results.length < maxpages) {
-    const current = frontier.next();
-    if (!current || current.depth > maxdepth) continue;
-    const result = await options.scrape(current.url);
-    results.push({ ...result, depth: current.depth });
-    frontier.complete(current.url);
-    if (current.depth >= maxdepth) continue;
-    for (const link of result.links ?? []) {
-      let url;
-      try { url = normalizeurl(link); } catch { continue; }
-      if (sameDomain && !sameorigin(start, url)) continue;
-      frontier.add({ url, depth: current.depth + 1, priority: Number(options.priority?.(url, result) ?? 0) });
-    }
-  }
-  return { results, stats: { ...frontier.state(), completed: results.length, maxdepth, maxpages } };
-}

package/scrape/crawler.ts

@@ -1,115 +0,0 @@
-import type { ScrapeOptions, ScrapeResult, CrawlStats } from './types.js';
-import { scrapeUrl } from './scrape.js';
-
-export interface CrawlOptions extends ScrapeOptions {
-  maxDepth?: number;
-  maxPages?: number;
-  maxConcurrent?: number;
-  sameDomain?: boolean;
-  delayMs?: number;
-  onDiscover?: (url: string, depth: number) => void;
-  onResult?: (result: ScrapeResult) => void;
-  onError?: (url: string, error: Error) => void;
-}
-
-interface CrawlEntry {
-  url: string;
-  depth: number;
-}
-
-function isSameDomain(url1: string, url2: string): boolean {
-  try {
-    return new URL(url1).hostname === new URL(url2).hostname;
-  } catch {
-    return false;
-  }
-}
-
-function normalizeForDedup(url: string): string {
-  try {
-    const u = new URL(url);
-    u.hash = '';
-    u.search = '';
-    return u.href.replace(/\/+$/, '') || u.href;
-  } catch {
-    return url;
-  }
-}
-
-export async function crawl(
-  startUrl: string,
-  options: CrawlOptions = {}
-): Promise<{ results: ScrapeResult[]; stats: CrawlStats }> {
-  const {
-    maxDepth = 2,
-    maxPages = 50,
-    maxConcurrent = 3,
-    sameDomain = true,
-    delayMs = 1000,
-    onDiscover,
-    onResult,
-    onError,
-    ...scrapeOpts
-  } = options;
-
-  const startTime = Date.now();
-  const visited = new Set<string>();
-  const queue: CrawlEntry[] = [{ url: startUrl, depth: 0 }];
-  const results: ScrapeResult[] = [];
-  let failed = 0;
-
-  visited.add(normalizeForDedup(startUrl));
-
-  while (queue.length > 0 && results.length < maxPages) {
-    const batch = queue.splice(0, maxConcurrent);
-    const promises = batch.map(async (entry) => {
-      if (entry.depth > maxDepth) return;
-
-      try {
-        const result = await scrapeUrl(entry.url, scrapeOpts);
-        results.push(result);
-        onResult?.(result);
-
-        // Extract links for further crawling
-        if (entry.depth < maxDepth) {
-          for (const link of result.links) {
-            const normalized = normalizeForDedup(link.href);
-            if (visited.has(normalized)) continue;
-            if (sameDomain && !isSameDomain(startUrl, link.href)) continue;
-            try {
-              new URL(link.href);
-            } catch {
-              continue;
-            }
-            visited.add(normalized);
-            queue.push({ url: link.href, depth: entry.depth + 1 });
-            onDiscover?.(link.href, entry.depth + 1);
-          }
-        }
-      } catch (error) {
-        failed++;
-        onError?.(entry.url, error as Error);
-      }
-    });
-
-    await Promise.all(promises);
-
-    if (delayMs > 0 && queue.length > 0) {
-      await new Promise(r => setTimeout(r, delayMs));
-    }
-  }
-
-  const duration = Date.now() - startTime;
-
-  return {
-    results,
-    stats: {
-      totalUrls: visited.size,
-      successful: results.length,
-      failed,
-      skipped: visited.size - results.length - failed,
-      duration,
-      avgResponseTime: results.length > 0 ? results.reduce((sum, r) => sum + r.duration, 0) / results.length : 0,
-    },
-  };
-}

package/scrape/dev-server.ts

@@ -1,94 +0,0 @@
-#!/usr/bin/env node
-import { createServer } from 'node:http';
-
-import { scrapeUrl } from './scrape.js';
-import { serializeResult } from './serialize.js';
-import { resolveFormat, buildSerializeOptions } from './formats.js';
-import { formatForAgent } from './agent.js';
-import { randomPort } from './utils/port.js';
-import type { ScrapeOptions } from './types.js';
-
-const PORT = randomPort();
-
-const server = createServer(async (req, res) => {
-  res.setHeader('Access-Control-Allow-Origin', '*');
-  res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
-  res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
-
-  if (req.method === 'OPTIONS') {
-    res.writeHead(200);
-    res.end();
-    return;
-  }
-
-  if (req.method !== 'POST' || req.url !== '/api/scrape') {
-    res.writeHead(404);
-    res.end(JSON.stringify({ error: 'Not found' }));
-    return;
-  }
-
-  let body = '';
-  req.on('data', (chunk: Buffer) => { body += chunk.toString(); });
-  req.on('end', async () => {
-    try {
-      const { url, format: fmt, options = {} }: {
-        url: string;
-        format?: string;
-        options?: Record<string, unknown>;
-      } = JSON.parse(body);
-      if (!url) throw new Error('URL is required');
-
-      const format = resolveFormat(fmt || 'markdown');
-      const result = await scrapeUrl(url, {
-        timeout: 30000,
-        scroll: options.scroll as boolean,
-        extractLinks: (options.extractLinks as boolean) ?? true,
-        extractImages: (options.extractImages as boolean) ?? true,
-        extractTables: (options.extractTables as boolean) ?? true,
-      } as ScrapeOptions);
-
-      let response: Record<string, unknown>;
-
-      if (options.agent) {
-        const agentOutput = formatForAgent(result);
-        response = {
-          success: true,
-          title: result.title,
-          url: result.url,
-          duration: result.duration,
-          size: result.size,
-          agentOutput,
-        };
-      } else {
-        const serializeOpts = buildSerializeOptions(format);
-        const serialized = serializeResult(result, serializeOpts);
-        response = {
-          success: true,
-          title: result.title,
-          url: result.url,
-          format,
-          content: serialized.content,
-          duration: result.duration,
-          size: serialized.size,
-          metadata: result.metadata,
-          links: result.links,
-          images: result.images,
-          tables: result.tables,
-          result,
-        };
-      }
-
-      res.writeHead(200, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify(response));
-    } catch (err) {
-      const error = err as Error;
-      res.writeHead(500, { 'Content-Type': 'application/json' });
-      res.end(JSON.stringify({ error: error.message }));
-    }
-  });
-});
-
-server.listen(PORT, () => {
-  console.log(`API server running at http://localhost:${PORT}`);
-  console.log(`POST /api/scrape with { url, format?, options? }`);
-});