@wenathlan/saddle 1.8.8 → 1.8.9

package/scrape/agent.ts

@@ -1,122 +0,0 @@
-import type { ScrapeResult, AgentOutput } from './types.js';
-import { estimateTokens } from './tokens.js';
-import { chunkMarkdown } from './chunking.js';
-
-export interface AgentFormatOptions {
-  includeSummary?: boolean;
-  includeKeyPoints?: boolean;
-  includeUrls?: boolean;
-  includeRaw?: boolean;
-  maxTokens?: number;
-  chunkSize?: number;
-  model?: 'gpt-4o' | 'gpt-4' | 'gpt-3.5-turbo' | 'claude' | 'gemini' | 'default';
-}
-
-const DEFAULT_OPTIONS: Required<AgentFormatOptions> = {
-  includeSummary: true,
-  includeKeyPoints: true,
-  includeUrls: true,
-  includeRaw: true,
-  maxTokens: 8000,
-  chunkSize: 512,
-  model: 'default',
-};
-
-function generateSummary(result: ScrapeResult): string {
-  const wordCount = result.text.split(/\s+/).length;
-  return `This page titled "${result.title}" contains approximately ${wordCount} words with ${result.links.length} links and ${result.images.length} images.`;
-}
-
-function generateKeyPoints(result: ScrapeResult): string[] {
-  const points: string[] = [];
-  const text = result.text.slice(0, 5000);
-
-  if (result.metadata.author) points.push(`Author: ${result.metadata.author}`);
-  if (result.metadata.publishedDate) points.push(`Published: ${result.metadata.publishedDate}`);
-  if (result.metadata.description) points.push(`Description: ${result.metadata.description.slice(0, 200)}`);
-
-  const sentences = text.match(/[^.!?]+[.!?]+/g) || [];
-  for (const s of sentences.slice(0, 5)) {
-    const trimmed = s.trim();
-    if (trimmed.length > 30) points.push(trimmed);
-  }
-
-  return points.slice(0, 8);
-}
-
-function buildSystemPrompt(result: ScrapeResult): string {
-  let prompt = `# Web Page Content\n\n`;
-  prompt += `## Title\n${result.title}\n\n`;
-  prompt += `## URL\n${result.url}\n\n`;
-
-  if (result.metadata.description) {
-    prompt += `## Description\n${result.metadata.description}\n\n`;
-  }
-
-  if (result.tables.length > 0) {
-    prompt += `## Tables (${result.tables.length} found)\n`;
-    for (const t of result.tables.slice(0, 3)) {
-      prompt += `- ${t.caption || 'Unnamed table'}: ${t.headers.join(', ')}\n`;
-    }
-    prompt += '\n';
-  }
-
-  prompt += `## Content\n${result.content}\n\n`;
-  return prompt;
-}
-
-export function formatForAgent(result: ScrapeResult, options?: AgentFormatOptions): AgentOutput {
-  const opts = { ...DEFAULT_OPTIONS, ...options };
-  let content = '';
-
-  if (opts.includeSummary) {
-    content += `## Summary\n${generateSummary(result)}\n\n`;
-  }
-
-  if (opts.includeKeyPoints) {
-    const points = generateKeyPoints(result);
-    content += `## Key Points\n${points.map(p => `- ${p}`).join('\n')}\n\n`;
-  }
-
-  if (opts.includeUrls) {
-    const externalUrls = result.links.filter(l => l.isExternal).slice(0, 15);
-    if (externalUrls.length) {
-      content += `## External Links\n${externalUrls.map(l => `- [${l.text || l.href}](${l.href})`).join('\n')}\n\n`;
-    }
-    const internalUrls = result.links.filter(l => l.isInternal).slice(0, 10);
-    if (internalUrls.length) {
-      content += `## Internal Links\n${internalUrls.map(l => `- [${l.text || l.href}](${l.href})`).join('\n')}\n\n`;
-    }
-  }
-
-  if (opts.includeRaw) {
-    content += `## Full Content\n${result.content}\n\n`;
-  }
-
-  const totalTokens = estimateTokens(content, opts.model);
-
-  if (totalTokens > opts.maxTokens) {
-    const charLimit = Math.floor(opts.maxTokens * 4);
-    content = content.slice(0, charLimit) + '\n\n... (truncated)';
-  }
-
-  const chunks = chunkMarkdown(content, { maxTokens: opts.chunkSize });
-
-  const output: AgentOutput = {
-    summary: generateSummary(result),
-    content,
-    keyPoints: generateKeyPoints(result),
-    relevantUrls: result.links.filter(l => l.isExternal).map(l => l.href).slice(0, 10),
-    tokens: estimateTokens(content, opts.model),
-  };
-
-  if (chunks.length > 1) {
-    (output as unknown as Record<string, unknown>).chunks = chunks.map(c => c.content);
-  }
-
-  return output;
-}
-
-export function buildContext(result: ScrapeResult): string {
-  return buildSystemPrompt(result);
-}

package/scrape/batch.ts

@@ -1,79 +0,0 @@
-import pLimit from 'p-limit';
-import type { ScrapeOptions, ScrapeResult, BatchOptions } from './types.js';
-import { scrapeUrl } from './scrape.js';
-
-export interface BatchResult {
-  results: ScrapeResult[];
-  errors: { url: string; error: Error }[];
-  successful: number;
-  failed: number;
-  duration: number;
-}
-
-export async function batchScrape(options: BatchOptions, scrapeOpts: ScrapeOptions = {}): Promise<BatchResult> {
-  const startTime = Date.now();
-  const limit = pLimit(options.concurrency || 5);
-  const results: ScrapeResult[] = [];
-  const errors: { url: string; error: Error }[] = [];
-
-  const tasks = options.urls.map((url) =>
-    limit(async () => {
-      try {
-        const result = await scrapeUrl(url, { ...scrapeOpts, mode: scrapeOpts.mode || 'auto' });
-        results.push(result);
-        options.onProgress?.(results.length + errors.length, options.urls.length, url);
-      } catch (error) {
-        errors.push({ url, error: error as Error });
-        options.onError?.(url, error as Error);
-        options.onProgress?.(results.length + errors.length, options.urls.length, url);
-      }
-    })
-  );
-
-  await Promise.allSettled(tasks);
-
-  return {
-    results,
-    errors,
-    successful: results.length,
-    failed: errors.length,
-    duration: Date.now() - startTime,
-  };
-}
-
-export async function batchScrapeSequential(
-  urls: string[],
-  options: ScrapeOptions = {},
-  callbacks?: {
-    onResult?: (result: ScrapeResult, index: number) => void;
-    onError?: (url: string, error: Error, index: number) => void;
-    delayMs?: number;
-  }
-): Promise<BatchResult> {
-  const startTime = Date.now();
-  const results: ScrapeResult[] = [];
-  const errors: { url: string; error: Error }[] = [];
-
-  for (let i = 0; i < urls.length; i++) {
-    try {
-      const result = await scrapeUrl(urls[i], options);
-      results.push(result);
-      callbacks?.onResult?.(result, i);
-    } catch (error) {
-      errors.push({ url: urls[i], error: error as Error });
-      callbacks?.onError?.(urls[i], error as Error, i);
-    }
-
-    if (callbacks?.delayMs && i < urls.length - 1) {
-      await new Promise(r => setTimeout(r, callbacks.delayMs));
-    }
-  }
-
-  return {
-    results,
-    errors,
-    successful: results.length,
-    failed: errors.length,
-    duration: Date.now() - startTime,
-  };
-}

package/scrape/biome.json

@@ -1,76 +0,0 @@
-{
-	"$schema": "https://biomejs.dev/schemas/2.4.15/schema.json",
-	"formatter": {
-		"enabled": true,
-		"indentStyle": "tab",
-		"indentWidth": 2,
-		"lineWidth": 100,
-		"lineEnding": "lf",
-		"formatWithErrors": false
-	},
-	"linter": {
-		"enabled": true,
-		"rules": {
-			"recommended": true,
-			"a11y": {
-				"noStaticElementInteractions": "off",
-				"useKeyWithClickEvents": "off",
-				"useSemanticElements": "off"
-			},
-			"correctness": {
-				"useExhaustiveDependencies": "off"
-			},
-			"style": {
-				"noNonNullAssertion": "off",
-				"useNodejsImportProtocol": "off",
-				"useConst": "error"
-			},
-			"suspicious": {
-				"noExplicitAny": "off",
-				"noEmptyBlock": "warn"
-			},
-			"complexity": {
-				"noUselessFragments": "warn",
-				"noForEach": "off"
-			},
-			"performance": {
-				"noDelete": "off"
-			}
-		}
-	},
-	"assist": {
-		"enabled": true,
-		"actions": {
-			"source": {
-				"organizeImports": "on"
-			}
-		}
-	},
-	"javascript": {
-		"formatter": {
-			"quoteStyle": "double",
-			"semicolons": "always",
-			"trailingCommas": "es5",
-			"arrowParentheses": "always"
-		}
-	},
-	"css": {
-		"parser": {
-			"cssModules": true,
-			"allowWrongLineComments": true
-		},
-		"formatter": {
-			"enabled": true
-		}
-	},
-	"files": {
-		"includes": ["*.ts", "web/**", "tests/**"],
-		"ignore": ["node_modules", "dist", "*.min.*"],
-		"ignoreUnknown": true
-	},
-	"vcs": {
-		"enabled": true,
-		"clientKind": "git",
-		"useIgnoreFile": true
-	}
-}

package/scrape/browser.ts

@@ -1,222 +0,0 @@
-import { chromium, firefox, webkit, type Browser, type BrowserContext, type Page } from 'playwright';
-import type { BrowserAgentConfig, ChromeCommand, RenderOptions } from './types.js';
-import { WebScrapeError, ErrorCode, BrowserNotAvailableError } from './errors.js';
-
-export class AgentBrowser {
-  private browser: Browser | null = null;
-  private context: BrowserContext | null = null;
-  private page: Page | null = null;
-  private config: Required<Omit<BrowserAgentConfig, 'userAgent' | 'geolocation'>> & Pick<BrowserAgentConfig, 'userAgent' | 'geolocation'>;
-
-  private static DEFAULTS = {
-    headless: true,
-    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
-    viewport: { width: 1280, height: 720 },
-    locale: 'en-US',
-    timeout: 30000,
-    recordVideo: false,
-    proxy: '',
-    storageState: '',
-    blockAds: true,
-    stealth: true,
-  };
-
-  constructor(config: Partial<BrowserAgentConfig> = {}) {
-    this.config = { ...AgentBrowser.DEFAULTS, ...config };
-  }
-
-  async launch(browserType: 'chromium' | 'firefox' | 'webkit' = 'chromium'): Promise<void> {
-    try {
-      const launcher = { chromium, firefox, webkit }[browserType];
-      const launchOpts: Parameters<typeof chromium.launch>[0] = {
-        headless: this.config.headless,
-        args: [
-          '--no-sandbox',
-          '--disable-setuid-sandbox',
-          '--disable-dev-shm-usage',
-          '--disable-web-security',
-          '--disable-features=IsolateOrigins,site-per-process',
-        ],
-      };
-      if (this.config.proxy) {
-        launchOpts.proxy = { server: this.config.proxy };
-      }
-      this.browser = await launcher.launch(launchOpts);
-
-      const ctxOpts: Parameters<Browser['newContext']>[0] = {
-        viewport: this.config.viewport,
-        locale: this.config.locale,
-        userAgent: this.config.userAgent,
-      };
-      if (this.config.recordVideo) {
-        ctxOpts.recordVideo = { dir: './videos' };
-      }
-      if (this.config.geolocation) {
-        ctxOpts.geolocation = this.config.geolocation;
-        ctxOpts.permissions = ['geolocation'];
-      }
-      if (this.config.storageState) {
-        ctxOpts.storageState = this.config.storageState;
-      }
-      this.context = await this.browser.newContext(ctxOpts);
-
-      if (this.config.blockAds) {
-        await this.context.route('**/*.{png,jpg,jpeg,gif,svg,ico,css,woff,woff2,mp4,webm}', r => r.abort());
-      }
-
-      this.page = await this.context.newPage();
-      this.page.setDefaultTimeout(this.config.timeout);
-    } catch (error) {
-      if ((error as Error).message?.includes('playwright') || (error as Error).message?.includes('chromium')) {
-        throw new BrowserNotAvailableError();
-      }
-      throw error;
-    }
-  }
-
-  async navigate(url: string, waitUntil: 'load' | 'domcontentloaded' | 'networkidle' = 'networkidle'): Promise<void> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    await this.page.goto(url, { waitUntil, timeout: this.config.timeout });
-  }
-
-  async screenshot(opts: RenderOptions = {}): Promise<Buffer> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    return this.page.screenshot({
-      fullPage: opts.fullPage ?? false,
-      type: opts.type ?? 'png',
-      quality: opts.quality ?? 80,
-    }) as Promise<Buffer>;
-  }
-
-  async evaluate<T = unknown>(fn: string | (() => T), args?: unknown[]): Promise<T> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    if (typeof fn === 'string') {
-      return this.page.evaluate(new Function(fn) as () => T);
-    }
-    return this.page.evaluate(fn, args);
-  }
-
-  async click(selector: string): Promise<void> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    await this.page.click(selector);
-  }
-
-  async type(selector: string, text: string, delayMs = 0): Promise<void> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    await this.page.fill(selector, '');
-    await this.page.type(selector, text, { delay: delayMs });
-  }
-
-  async scrollToBottom(step = 800, delayMs = 300, maxScrolls = 50): Promise<number> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    let scrolls = 0;
-    let prevHeight = 0;
-    while (scrolls < maxScrolls) {
-      const height = await this.page.evaluate(() => document.body.scrollHeight);
-      if (height === prevHeight && scrolls > 2) break;
-      prevHeight = height;
-      await this.page.evaluate(y => window.scrollBy(0, y), step);
-      await new Promise(r => setTimeout(r, delayMs));
-      scrolls++;
-    }
-    return scrolls;
-  }
-
-  async scrollTo(x: number, y: number): Promise<void> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    await this.page.evaluate(([px, py]) => window.scrollTo(px, py), [x, y]);
-  }
-
-  async waitFor(selector: string, timeout?: number): Promise<void> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    await this.page.waitForSelector(selector, { timeout: timeout ?? this.config.timeout });
-  }
-
-  async waitForLoad(): Promise<void> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    await this.page.waitForLoadState('networkidle');
-  }
-
-  async html(): Promise<string> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    return this.page.content();
-  }
-
-  async text(): Promise<string> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    return this.page.evaluate(() => document.body.innerText);
-  }
-
-  async title(): Promise<string> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    return this.page.title();
-  }
-
-  async url(): Promise<string> {
-    if (!this.page) throw new WebScrapeError('Browser not launched', ErrorCode.BROWSER_NOT_AVAILABLE, 500, false);
-    return this.page.url();
-  }
-
-  async executeCommands(commands: ChromeCommand[]): Promise<unknown[]> {
-    const results: unknown[] = [];
-    for (const cmd of commands) {
-      switch (cmd.action) {
-        case 'goto':
-          await this.navigate(cmd.args?.url as string);
-          results.push({ ok: true });
-          break;
-        case 'click':
-          await this.click(cmd.args?.selector as string);
-          results.push({ ok: true });
-          break;
-        case 'type':
-          await this.type(cmd.args?.selector as string, cmd.args?.text as string);
-          results.push({ ok: true });
-          break;
-        case 'screenshot':
-          results.push(await this.screenshot(cmd.args as RenderOptions));
-          break;
-        case 'evaluate':
-          results.push(await this.evaluate(cmd.args?.fn as string));
-          break;
-        case 'wait':
-          await this.waitFor(cmd.args?.selector as string);
-          results.push({ ok: true });
-          break;
-        case 'scroll':
-          if (cmd.args?.to) {
-            await this.scrollTo(0, cmd.args.to as number);
-          } else {
-            await this.scrollToBottom();
-          }
-          results.push({ ok: true });
-          break;
-        case 'extract':
-          results.push({
-            html: await this.html(),
-            text: await this.text(),
-            title: await this.title(),
-          });
-          break;
-      }
-    }
-    return results;
-  }
-
-  async close(): Promise<void> {
-    if (this.page) await this.page.close().catch(() => {});
-    if (this.context) await this.context.close().catch(() => {});
-    if (this.browser) await this.browser.close().catch(() => {});
-    this.page = null;
-    this.context = null;
-    this.browser = null;
-  }
-
-  isConnected(): boolean {
-    return this.browser?.isConnected() ?? false;
-  }
-}
-
-export function createBrowser(config?: BrowserAgentConfig): AgentBrowser {
-  return new AgentBrowser(config);
-}

package/scrape/cache.js

@@ -1,14 +0,0 @@
-/**
- * ttl cache keeps fetch policy separate from extraction and transport.
- */
-export function ttlcache(options = {}) {
-  const values = new Map();
-  const ttl = options.ttl ?? 300000;
-  return {
-    get(key) { const item = values.get(key); if (!item) return null; if (Date.now() > item.expires) { if (Date.now() > item.stale) values.delete(key); return options.stale ? item.value : null; } return item.value; },
-    set(key, value, valueoptions = {}) { const current = Date.now(); values.set(key, { value, expires: current + (valueoptions.ttl ?? ttl), stale: current + (valueoptions.stale ?? ttl * 2) }); return value; },
-    delete(key) { values.delete(key); },
-    clear() { values.clear(); },
-    size() { return values.size; }
-  };
-}

package/scrape/cache.ts

@@ -1,84 +0,0 @@
-import { LRUCache } from 'lru-cache';
-
-export interface CacheEntry<T = unknown> {
-  value: T;
-  timestamp: number;
-  expires?: number;
-}
-
-export interface CacheConfig {
-  maxSize?: number;
-  defaultTtlMs?: number;
-  checkPeriodMs?: number;
-}
-
-const DEFAULT_CONFIG: Required<CacheConfig> = {
-  maxSize: 1000,
-  defaultTtlMs: 5 * 60 * 1000,
-  checkPeriodMs: 60 * 1000,
-};
-
-export class WebScrapeCache {
-  private store: LRUCache<string, CacheEntry>;
-
-  constructor(config: Partial<CacheConfig> = {}) {
-    const cfg = { ...DEFAULT_CONFIG, ...config };
-    this.store = new LRUCache<string, CacheEntry>({
-      max: cfg.maxSize,
-      ttl: cfg.defaultTtlMs,
-      updateAgeOnGet: true,
-    });
-  }
-
-  get<T = unknown>(key: string): T | undefined {
-    const entry = this.store.get(key);
-    if (!entry) return undefined;
-    if (entry.expires && Date.now() > entry.expires) {
-      this.store.delete(key);
-      return undefined;
-    }
-    return entry.value as T;
-  }
-
-  set<T = unknown>(key: string, value: T, ttlMs?: number): void {
-    const entry: CacheEntry<T> = {
-      value,
-      timestamp: Date.now(),
-      expires: ttlMs ? Date.now() + ttlMs : undefined,
-    };
-    this.store.set(key, entry, { ttl: ttlMs });
-  }
-
-  has(key: string): boolean {
-    return this.get(key) !== undefined;
-  }
-
-  delete(key: string): boolean {
-    return this.store.delete(key);
-  }
-
-  clear(): void {
-    this.store.clear();
-  }
-
-  getOrSet<T = unknown>(key: string, factory: () => Promise<T>, ttlMs?: number): Promise<T> {
-    const cached = this.get<T>(key);
-    if (cached !== undefined) return Promise.resolve(cached);
-    return factory().then(value => {
-      this.set(key, value, ttlMs);
-      return value;
-    });
-  }
-
-  size(): number {
-    return this.store.size;
-  }
-
-  keys(): string[] {
-    return [...this.store.keys()];
-  }
-}
-
-export function createCache(config?: Partial<CacheConfig>): WebScrapeCache {
-  return new WebScrapeCache(config);
-}