@wenathlan/saddle 1.8.2 → 1.8.5

package/scrape/pool.ts

@@ -0,0 +1,95 @@
+import { AgentBrowser } from './browser.js';
+import type { BrowserAgentConfig } from './types.js';
+
+export interface PoolConfig {
+  maxSize?: number;
+  acquireTimeoutMs?: number;
+}
+
+interface PoolEntry {
+  browser: AgentBrowser;
+  inUse: boolean;
+  createdAt: number;
+}
+
+export class BrowserPool {
+  private pool: PoolEntry[] = [];
+  private waitQueue: Array<(browser: AgentBrowser) => void> = [];
+  private config: Required<PoolConfig>;
+  private browserConfig: BrowserAgentConfig;
+
+  constructor(config: PoolConfig = {}, browserConfig: BrowserAgentConfig = {}) {
+    this.config = {
+      maxSize: config.maxSize || 3,
+      acquireTimeoutMs: config.acquireTimeoutMs || 30000,
+    };
+    this.browserConfig = browserConfig;
+  }
+
+  async acquire(): Promise<AgentBrowser> {
+    const idle = this.pool.find(e => !e.inUse && e.browser.isConnected());
+    if (idle) {
+      idle.inUse = true;
+      return idle.browser;
+    }
+
+    if (this.pool.length < this.config.maxSize) {
+      const browser = new AgentBrowser(this.browserConfig);
+      await browser.launch();
+      const entry: PoolEntry = { browser, inUse: true, createdAt: Date.now() };
+      this.pool.push(entry);
+      return browser;
+    }
+
+    return new Promise<AgentBrowser>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        const idx = this.waitQueue.indexOf(waiter);
+        if (idx !== -1) this.waitQueue.splice(idx, 1);
+        reject(new Error('Timeout waiting for browser'));
+      }, this.config.acquireTimeoutMs);
+
+      const waiter = (browser: AgentBrowser) => {
+        clearTimeout(timeout);
+        resolve(browser);
+      };
+      this.waitQueue.push(waiter);
+    });
+  }
+
+  release(browser: AgentBrowser): void {
+    const entry = this.pool.find(e => e.browser === browser);
+    if (entry) {
+      entry.inUse = false;
+
+      if (this.waitQueue.length > 0) {
+        const waiter = this.waitQueue.shift()!;
+        entry.inUse = true;
+        waiter(browser);
+      }
+    }
+  }
+
+  async destroy(): Promise<void> {
+    for (const entry of this.pool) {
+      await entry.browser.close().catch(() => {});
+    }
+    this.pool = [];
+    for (const waiter of this.waitQueue) {
+      waiter(new AgentBrowser());
+    }
+    this.waitQueue = [];
+  }
+
+  getStatus(): { total: number; idle: number; inUse: number; waiting: number } {
+    return {
+      total: this.pool.length,
+      idle: this.pool.filter(e => !e.inUse).length,
+      inUse: this.pool.filter(e => e.inUse).length,
+      waiting: this.waitQueue.length,
+    };
+  }
+}
+
+export function createPool(config?: PoolConfig, browserConfig?: BrowserAgentConfig): BrowserPool {
+  return new BrowserPool(config, browserConfig);
+}

package/scrape/port.ts

@@ -0,0 +1,18 @@
+/**
+ * Generate a random port number in the ephemeral port range (1024-65535).
+ * The port is calculated once and then fixed for the lifetime of the process.
+ */
+let _cachedPort: number | null = null;
+
+export function randomPort(): number {
+  if (_cachedPort !== null) return _cachedPort;
+  _cachedPort = Math.floor(Math.random() * 64511) + 1024;
+  return _cachedPort;
+}
+
+/**
+ * Reset the cached port (for testing).
+ */
+export function resetPort(): void {
+  _cachedPort = null;
+}

package/scrape/proxy.ts

@@ -0,0 +1,103 @@
+import type { ProxyConfig } from './types.js';
+
+interface ProxyStats {
+  failures: number;
+  successes: number;
+  consecutiveFailures: number;
+  lastFailure?: number;
+  lastSuccess?: number;
+  disabled: boolean;
+  disabledAt?: number;
+}
+
+export type ProxyRotationStrategy = 'round-robin' | 'random';
+
+export interface ProxyPoolConfig {
+  maxFailures?: number;
+  reviveAfterMs?: number;
+  strategy?: ProxyRotationStrategy;
+}
+
+const DEFAULT_POOL_CONFIG: Required<ProxyPoolConfig> = {
+  maxFailures: 3,
+  reviveAfterMs: 30 * 60 * 1000,
+  strategy: 'round-robin',
+};
+
+export class ProxyPool {
+  private proxies: ProxyConfig[];
+  private stats = new Map<string, ProxyStats>();
+  private currentIndex = 0;
+  private config: Required<ProxyPoolConfig>;
+
+  constructor(proxies: ProxyConfig[], config: Partial<ProxyPoolConfig> = {}) {
+    this.proxies = proxies;
+    this.config = { ...DEFAULT_POOL_CONFIG, ...config };
+    for (const p of proxies) {
+      this.stats.set(p.url, { failures: 0, successes: 0, consecutiveFailures: 0, disabled: false });
+    }
+  }
+
+  private getActiveProxies(): ProxyConfig[] {
+    const now = Date.now();
+    return this.proxies.filter(p => {
+      const s = this.stats.get(p.url);
+      if (!s) return true;
+      if (s.disabled && s.disabledAt && (now - s.disabledAt) >= this.config.reviveAfterMs) {
+        s.disabled = false;
+        s.failures = 0;
+        s.consecutiveFailures = 0;
+        return true;
+      }
+      return !s.disabled;
+    });
+  }
+
+  next(): ProxyConfig | null {
+    const active = this.getActiveProxies();
+    if (active.length === 0) return null;
+
+    if (this.config.strategy === 'random') {
+      return active[Math.floor(Math.random() * active.length)];
+    }
+
+    const proxy = active[this.currentIndex % active.length];
+    this.currentIndex = (this.currentIndex + 1) % active.length;
+    return proxy;
+  }
+
+  reportSuccess(proxy: ProxyConfig): void {
+    const s = this.stats.get(proxy.url);
+    if (s) {
+      s.successes++;
+      s.consecutiveFailures = 0;
+      s.lastSuccess = Date.now();
+    }
+  }
+
+  reportFailure(proxy: ProxyConfig): void {
+    const s = this.stats.get(proxy.url);
+    if (s) {
+      s.failures++;
+      s.consecutiveFailures++;
+      s.lastFailure = Date.now();
+      if (s.consecutiveFailures >= this.config.maxFailures) {
+        s.disabled = true;
+        s.disabledAt = Date.now();
+      }
+    }
+  }
+
+  getStatus(): { total: number; active: number; disabled: number } {
+    const active = this.getActiveProxies();
+    return {
+      total: this.proxies.length,
+      active: active.length,
+      disabled: this.proxies.length - active.length,
+    };
+  }
+}
+
+export function createProxyPool(proxies: ProxyConfig[], config?: Partial<ProxyPoolConfig>): ProxyPool {
+  return new ProxyPool(proxies, config);
+}

package/scrape/rate-limiter.ts

@@ -0,0 +1,95 @@
+interface TokenBucket {
+	tokens: number;
+	lastRefill: number;
+	maxTokens: number;
+	refillRate: number;
+}
+
+export interface RateLimiterConfig {
+	maxTokensPerInterval: number;
+	intervalMs: number;
+	maxConcurrent?: number;
+}
+
+const DEFAULT_CONFIG: Required<RateLimiterConfig> = {
+	maxTokensPerInterval: 10,
+	intervalMs: 1000,
+	maxConcurrent: 5,
+};
+
+export class RateLimiter {
+	private buckets = new Map<string, TokenBucket>();
+	private active = new Map<string, number>();
+	private config: Required<RateLimiterConfig>;
+
+	constructor(config: Partial<RateLimiterConfig> = {}) {
+		this.config = { ...DEFAULT_CONFIG, ...config };
+	}
+
+	private getBucket(key: string): TokenBucket {
+		if (!this.buckets.has(key)) {
+			this.buckets.set(key, {
+				tokens: this.config.maxTokensPerInterval,
+				lastRefill: Date.now(),
+				maxTokens: this.config.maxTokensPerInterval,
+				refillRate: this.config.maxTokensPerInterval / (this.config.intervalMs / 1000),
+			});
+		}
+		return this.buckets.get(key)!;
+	}
+
+	private refill(bucket: TokenBucket): void {
+		const now = Date.now();
+		const elapsed = (now - bucket.lastRefill) / 1000;
+		bucket.tokens = Math.min(bucket.maxTokens, bucket.tokens + elapsed * bucket.refillRate);
+		bucket.lastRefill = now;
+	}
+
+	async acquire(key: string): Promise<void> {
+		const bucket = this.getBucket(key);
+		const currentActive = this.active.get(key) || 0;
+
+		if (currentActive >= this.config.maxConcurrent) {
+			await new Promise((resolve) => setTimeout(resolve, 100));
+			return this.acquire(key);
+		}
+
+		this.refill(bucket);
+
+		if (bucket.tokens < 1) {
+			const waitMs = ((1 - bucket.tokens) / bucket.refillRate) * 1000;
+			await new Promise((resolve) => setTimeout(resolve, Math.ceil(waitMs)));
+			this.refill(bucket);
+		}
+
+		bucket.tokens -= 1;
+		this.active.set(key, currentActive + 1);
+	}
+
+	release(key: string): void {
+		const current = this.active.get(key) || 0;
+		if (current <= 1) {
+			this.active.delete(key);
+		} else {
+			this.active.set(key, current - 1);
+		}
+	}
+
+	getStats(key: string): { tokens: number; active: number } {
+		const bucket = this.buckets.get(key);
+		if (bucket) this.refill(bucket);
+		return {
+			tokens: bucket?.tokens ?? this.config.maxTokensPerInterval,
+			active: this.active.get(key) || 0,
+		};
+	}
+
+	clear(): void {
+		this.buckets.clear();
+		this.active.clear();
+	}
+}
+
+export function createRateLimiter(config?: Partial<RateLimiterConfig>): RateLimiter {
+	return new RateLimiter(config);
+}

package/scrape/renderer.ts

@@ -0,0 +1,194 @@
+import { spawn, type ChildProcess } from 'node:child_process';
+import { existsSync, writeFileSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+interface PygameConfig {
+  pythonPath?: string;
+  width?: number;
+  height?: number;
+  fps?: number;
+  fullscreen?: boolean;
+}
+
+const DEFAULT_PYGAME_CONFIG: Required<PygameConfig> = {
+  pythonPath: 'python',
+  width: 1280,
+  height: 720,
+  fps: 30,
+  fullscreen: false,
+};
+
+function generatePygameScript(
+  imagePath: string,
+  config: Required<PygameConfig>,
+  interactive = false
+): string {
+  const lines = [
+    'import sys, json',
+    'import pygame',
+    'pygame.init()',
+    `screen = pygame.display.set_mode((${config.width}, ${config.height})${config.fullscreen ? ', pygame.FULLSCREEN' : ''})`,
+    'pygame.display.set_caption("DevThink WebScrape Renderer")',
+    `clock = pygame.time.Clock()`,
+    '',
+    'try:',
+    `  img = pygame.image.load(${JSON.stringify(imagePath)})`,
+    '  img_rect = img.get_rect(center=(screen.get_width()//2, screen.get_height()//2))',
+    'except:',
+    '  img = None',
+    '',
+    'font = pygame.font.Font(None, 36)',
+    '',
+    'running = True',
+    'while running:',
+    '  for event in pygame.event.get():',
+    '    if event.type == pygame.QUIT:',
+    '      running = False',
+    '    elif event.type == pygame.KEYDOWN:',
+    '      if event.key == pygame.K_ESCAPE:',
+    '        running = False',
+    ...(interactive ? [
+    '      elif event.key == pygame.K_SPACE:',
+    '        print(json.dumps({"action": "next"}))',
+    '        sys.stdout.flush()',
+    ] : []),
+    '',
+    '  screen.fill((30, 30, 40))',
+    '  if img:',
+    '    screen.blit(img, img_rect)',
+    '  else:',
+    '    text = font.render("No screenshot available", True, (200, 200, 200))',
+    '    screen.blit(text, (config.width//2 - text.get_width()//2, config.height//2))',
+    '',
+    '  pygame.display.flip()',
+    '  clock.tick(30)',
+    '',
+    'pygame.quit()',
+  ];
+  return lines.join('\n');
+}
+
+function generateDoomScript(
+  config: Required<PygameConfig>
+): string {
+  const lines = [
+    'import sys, json, asyncio, subprocess',
+    'import pygame',
+    'pygame.init()',
+    `screen = pygame.display.set_mode((${config.width}, ${config.height})${config.fullscreen ? ', pygame.FULLSCREEN' : ''})`,
+    'pygame.display.set_caption("DevThink WebScrape - DOOM")',
+    '',
+    'font = pygame.font.Font(None, 24)',
+    'small_font = pygame.font.Font(None, 18)',
+    '',
+    'frames = []',
+    'current_frame = 0',
+    'running = True',
+    '',
+    'def load_frames(path):',
+    '  import os',
+    '  frames_dir = os.path.expanduser(path)',
+    '  if os.path.isdir(frames_dir):',
+    '    files = sorted(os.listdir(frames_dir))',
+    '    for f in files:',
+    '      if f.endswith((".png", ".jpg", ".jpeg")):',
+    '        try:',
+    '          img = pygame.image.load(os.path.join(frames_dir, f))',
+    '          frames.append(img)',
+    '        except: pass',
+    '',
+    'load_frames("~/devthink/webscrape/doom_frames")',
+    '',
+    'while running:',
+    '  for event in pygame.event.get():',
+    '    if event.type == pygame.QUIT:',
+    '      running = False',
+    '    elif event.type == pygame.KEYDOWN:',
+    '      if event.key == pygame.K_ESCAPE: running = False',
+    '      elif event.key == pygame.K_LEFT: current_frame = max(0, current_frame - 1)',
+    '      elif event.key == pygame.K_RIGHT: current_frame = min(len(frames)-1, current_frame + 1)',
+    '      elif event.key == pygame.K_SPACE:',
+    '        out = {"action": "frame", "index": current_frame, "total": len(frames)}',
+    '        print(json.dumps(out))',
+    '        sys.stdout.flush()',
+    '',
+    '  screen.fill((20, 20, 30))',
+    '  if frames and current_frame < len(frames):',
+    '    img = pygame.transform.scale(frames[current_frame], (config.width, config.height - 60))',
+    '    screen.blit(img, (0, 0))',
+    '  else:',
+    '    text = font.render("No DOOM frames loaded", True, (200, 80, 80))',
+    '    screen.blit(text, (config.width//2 - 150, config.height//2))',
+    '',
+    '  info = f"Frame {current_frame+1}/{len(frames)} | ESC: quit | <- ->: navigate | SPACE: capture"',
+    '  info_surf = small_font.render(info, True, (180, 180, 180))',
+    '  screen.blit(info_surf, (10, config.height - 40))',
+    '',
+    '  pygame.display.flip()',
+    '  clock.tick(30)',
+    '',
+    'pygame.quit()',
+  ];
+  return lines.join('\n');
+}
+
+export class PygameRenderer {
+  private config: Required<PygameConfig>;
+  private process: ChildProcess | null = null;
+  private tempDir: string;
+  private screenDir: string;
+
+  constructor(config: PygameConfig = {}) {
+    this.config = { ...DEFAULT_PYGAME_CONFIG, ...config };
+    this.tempDir = join(process.cwd(), '.webscrape');
+    this.screenDir = join(this.tempDir, 'screenshots');
+    if (!existsSync(this.tempDir)) mkdirSync(this.tempDir, { recursive: true });
+    if (!existsSync(this.screenDir)) mkdirSync(this.screenDir, { recursive: true });
+  }
+
+  async renderScreenshot(imageBuffer: Buffer): Promise<void> {
+    const imagePath = join(this.screenDir, 'current.png');
+    writeFileSync(imagePath, imageBuffer);
+
+    const script = generatePygameScript(imagePath, this.config);
+    const scriptPath = join(this.tempDir, 'render_screenshot.py');
+    writeFileSync(scriptPath, script);
+
+    await this.runScript(scriptPath);
+  }
+
+  async renderDoom(): Promise<void> {
+    const script = generateDoomScript(this.config);
+    const scriptPath = join(this.tempDir, 'render_doom.py');
+    writeFileSync(scriptPath, script);
+
+    await this.runScript(scriptPath);
+  }
+
+  private runScript(scriptPath: string): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.process = spawn(this.config.pythonPath, [scriptPath], {
+        stdio: ['inherit', 'inherit', 'pipe'],
+      });
+
+      this.process.on('close', code => {
+        this.process = null;
+        if (code === 0) resolve();
+        else reject(new Error(`Pygame exited with code ${code}`));
+      });
+
+      this.process.on('error', err => reject(err));
+    });
+  }
+
+  close(): void {
+    if (this.process && !this.process.killed) {
+      this.process.kill();
+      this.process = null;
+    }
+  }
+}
+
+export function createRenderer(config?: PygameConfig): PygameRenderer {
+  return new PygameRenderer(config);
+}

package/scrape/retry.ts

@@ -0,0 +1,64 @@
+import pRetry, { AbortError } from "p-retry";
+import type { RetryConfig } from "./types.js";
+
+export const DEFAULT_RETRY_CONFIG: Required<RetryConfig> = {
+	retries: 3,
+	delay: 1000,
+	backoff: "exponential",
+	maxDelay: 30000,
+	statusCodes: [408, 429, 500, 502, 503, 504],
+};
+
+const RETRYABLE_NETWORK_CODES = ["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EPIPE", "ENOTFOUND"];
+
+export function isRetryableError(error: Error): boolean {
+	const code = (error as any).code;
+	if (code && RETRYABLE_NETWORK_CODES.includes(code)) return true;
+	const status = (error as any).statusCode || (error as any).status;
+	if (status && [408, 429, 500, 502, 503, 504].includes(status)) return true;
+	if (error.name === "AbortError") return false;
+	return false;
+}
+
+function calculateDelay(attempt: number, config: Required<RetryConfig>): number {
+	switch (config.backoff) {
+		case "exponential":
+			return Math.min(config.delay * 2 ** attempt, config.maxDelay);
+		case "linear":
+			return Math.min(config.delay * (attempt + 1), config.maxDelay);
+		case "constant":
+			return config.delay;
+	}
+}
+
+export async function withRetry<T>(
+	fn: (attemptNumber: number) => Promise<T>,
+	config: Partial<RetryConfig> = {},
+	onRetry?: (error: Error, attempt: number) => void,
+): Promise<T> {
+	const cfg = { ...DEFAULT_RETRY_CONFIG, ...config };
+
+	return pRetry(
+		async (attemptNumber) => {
+			try {
+				return await fn(attemptNumber);
+			} catch (error) {
+				if (!isRetryableError(error as Error)) {
+					throw new AbortError((error as Error).message);
+				}
+				throw error;
+			}
+		},
+		{
+			retries: cfg.retries,
+			minTimeout: calculateDelay(0, cfg),
+			maxTimeout: cfg.maxDelay,
+			randomize: cfg.backoff === "exponential",
+			onFailedAttempt: (context) => {
+				onRetry?.(context.error, context.attemptNumber);
+			},
+		},
+	);
+}
+
+export { AbortError };

package/scrape/robots.ts

@@ -0,0 +1,137 @@
+export interface RobotsDirective {
+  userAgent: string;
+  allow: string[];
+  disallow: string[];
+  crawlDelay?: number;
+}
+
+export interface RobotsTxt {
+  directives: RobotsDirective[];
+  sitemaps: string[];
+  crawlDelay?: number;
+}
+
+const cache = new Map<string, { robots: RobotsTxt; fetchedAt: number }>();
+const CACHE_TTL = 24 * 60 * 60 * 1000;
+
+function parseRobotsTxt(content: string): RobotsTxt {
+  const lines = content.split('\n').map(l => l.trim());
+  const directives: RobotsDirective[] = [];
+  const sitemaps: string[] = [];
+  let current: RobotsDirective | null = null;
+  let globalCrawlDelay: number | undefined;
+
+  for (const line of lines) {
+    if (!line || line.startsWith('#')) continue;
+
+    const colonIndex = line.indexOf(':');
+    if (colonIndex === -1) continue;
+
+    const key = line.slice(0, colonIndex).trim().toLowerCase();
+    const value = line.slice(colonIndex + 1).trim();
+
+    if (key === 'user-agent') {
+      current = { userAgent: value, allow: [], disallow: [] };
+      directives.push(current);
+    } else if (key === 'allow' && current) {
+      current.allow.push(value);
+    } else if (key === 'disallow' && current) {
+      if (value) current.disallow.push(value);
+    } else if (key === 'crawl-delay' && current) {
+      const delay = parseInt(value);
+      if (!isNaN(delay)) {
+        current.crawlDelay = delay;
+        globalCrawlDelay = delay;
+      }
+    } else if (key === 'sitemap') {
+      sitemaps.push(value);
+    }
+  }
+
+  return { directives, sitemaps, crawlDelay: globalCrawlDelay };
+}
+
+function matchUserAgent(pattern: string, userAgent: string): boolean {
+  const lowerPattern = pattern.toLowerCase();
+  const lowerUA = userAgent.toLowerCase();
+  if (lowerPattern === '*') return true;
+  return lowerUA.includes(lowerPattern.replace(/\*/g, ''));
+}
+
+function isPathAllowed(path: string, directive: RobotsDirective): boolean {
+  // Check disallow rules first
+  for (const pattern of directive.disallow) {
+    if (pattern === '/') return false;
+    if (pattern && path.startsWith(pattern)) return false;
+  }
+  // Then check allow rules
+  for (const pattern of directive.allow) {
+    if (pattern === '/') return true;
+    if (pattern && path.startsWith(pattern)) return true;
+  }
+  // Default: allowed
+  return true;
+}
+
+export async function fetchRobotsTxt(siteUrl: string, options: { timeout?: number; userAgent?: string } = {}): Promise<RobotsTxt> {
+  const base = siteUrl.replace(/\/$/, '');
+  const robotsUrl = `${base}/robots.txt`;
+  const userAgent = options.userAgent || 'DevThink-WebScrape/2.0';
+
+  const cached = cache.get(robotsUrl);
+  if (cached && (Date.now() - cached.fetchedAt) < CACHE_TTL) {
+    return cached.robots;
+  }
+
+  try {
+    const response = await fetch(robotsUrl, {
+      headers: { 'User-Agent': userAgent },
+      signal: AbortSignal.timeout(options.timeout || 5000),
+    });
+
+    if (!response.ok) {
+      const robots: RobotsTxt = { directives: [], sitemaps: [], crawlDelay: undefined };
+      cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
+      return robots;
+    }
+
+    const text = await response.text();
+    const robots = parseRobotsTxt(text);
+    cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
+    return robots;
+  } catch {
+    const robots: RobotsTxt = { directives: [], sitemaps: [], crawlDelay: undefined };
+    cache.set(robotsUrl, { robots, fetchedAt: Date.now() });
+    return robots;
+  }
+}
+
+export function isAllowed(url: string, robots: RobotsTxt, userAgent: string = '*'): boolean {
+  try {
+    const parsed = new URL(url);
+    const path = parsed.pathname;
+
+    for (const directive of robots.directives) {
+      if (matchUserAgent(directive.userAgent, userAgent)) {
+        return isPathAllowed(path, directive);
+      }
+    }
+
+    return true;
+  } catch {
+    return true;
+  }
+}
+
+export function getCrawlDelay(robots: RobotsTxt, userAgent: string = '*'): number | undefined {
+  for (const directive of robots.directives) {
+    if (matchUserAgent(directive.userAgent, userAgent)) {
+      return directive.crawlDelay;
+    }
+  }
+  return robots.crawlDelay;
+}
+
+export function getSitemaps(robots: RobotsTxt): string[] {
+  return robots.sitemaps;
+}