@wenathlan/saddle 1.8.2 → 1.8.5

package/scrape/scrape.ts

@@ -0,0 +1,123 @@
+import { AgentBrowser } from './browser.js';
+import { extractContent } from './extract.js';
+import { fetchHtml, detectRenderingMode } from './fetch.js';
+import type { ScrapeOptions, ScrapeResult } from './types.js';
+import { WebScrapeError, ErrorCode } from './errors.js';
+
+export async function scrapeUrl(url: string, options: ScrapeOptions = {}): Promise<ScrapeResult> {
+  if (!url || !url.startsWith('http')) {
+    throw new WebScrapeError(`Invalid URL: ${url}`, ErrorCode.VALIDATION_FAILED, 400, false);
+  }
+
+  const startTime = Date.now();
+  const mode = options.mode || 'auto';
+
+  if (mode === 'fetch' || mode === 'auto') {
+    try {
+      const fetchResult = await fetchHtml(url, {
+        timeout: options.timeout,
+        headers: options.headers,
+        userAgent: options.userAgent,
+      });
+
+      if (mode === 'auto') {
+        const renderMode = detectRenderingMode(fetchResult.html);
+        if (renderMode === 'static') {
+          return buildResult(fetchResult.html, fetchResult.url, options, startTime);
+        }
+      } else {
+        return buildResult(fetchResult.html, fetchResult.url, options, startTime);
+      }
+    } catch (error) {
+      if (mode === 'fetch') throw error;
+      if (!(error instanceof WebScrapeError) || error.isRetryable) {
+        // Fall through to browser mode
+      } else {
+        throw error;
+      }
+    }
+  }
+
+  const browser = new AgentBrowser({
+    headless: true,
+    timeout: options.timeout ?? 30000,
+    proxy: options.proxy ?? '',
+    userAgent: options.userAgent,
+  });
+
+  try {
+    await browser.launch();
+    await browser.navigate(url, (options.waitUntil as 'load' | 'domcontentloaded' | 'networkidle') ?? 'networkidle');
+
+    if (options.scroll) {
+      await browser.scrollToBottom(800, options.scrollDelay ?? 300, options.maxScrolls ?? 50);
+    }
+
+    const html = await browser.html();
+    const pageTitle = await browser.title();
+    const pageUrl = await browser.url();
+
+    return buildResult(html, pageUrl, options, startTime, pageTitle);
+  } finally {
+    await browser.close();
+  }
+}
+
+export async function scrapeHtml(html: string, options: ScrapeOptions = {}): Promise<ScrapeResult> {
+  const startTime = Date.now();
+  return buildResult(html, 'about:blank', options, startTime);
+}
+
+export async function scrapeWithBrowser(
+  browser: AgentBrowser,
+  url: string,
+  options: ScrapeOptions = {}
+): Promise<ScrapeResult> {
+  const startTime = Date.now();
+
+  await browser.navigate(url, (options.waitUntil as 'load' | 'domcontentloaded' | 'networkidle') ?? 'networkidle');
+
+  if (options.scroll) {
+    await browser.scrollToBottom(800, options.scrollDelay ?? 300, options.maxScrolls ?? 50);
+  }
+
+  const html = await browser.html();
+  const pageTitle = await browser.title();
+  const pageUrl = await browser.url();
+
+  return buildResult(html, pageUrl, options, startTime, pageTitle);
+}
+
+async function buildResult(
+  html: string,
+  url: string,
+  options: ScrapeOptions,
+  startTime: number,
+  title?: string,
+): Promise<ScrapeResult> {
+  const extracted = await extractContent(html, {
+    readable: true,
+    preserveLinks: options.extractLinks ?? true,
+    preserveImages: options.extractImages ?? true,
+    preserveTables: options.extractTables ?? true,
+    maxLength: options.maxContentLength,
+    removeSelectors: options.removeSelectors,
+  });
+
+  const duration = Date.now() - startTime;
+
+  return {
+    url,
+    title: title || extracted.metadata.title || '',
+    content: extracted.content,
+    format: options.format ?? 'markdown',
+    text: extracted.text,
+    links: extracted.links,
+    images: extracted.images,
+    tables: extracted.tables,
+    metadata: extracted.metadata,
+    extractedAt: new Date().toISOString(),
+    duration,
+    size: extracted.content.length + extracted.text.length,
+  };
+}

package/scrape/serialize.ts

@@ -0,0 +1,310 @@
+import { JSDOM } from 'jsdom';
+import type { SerializeOptions, SerializedOutput, ScrapeResult } from './types.js';
+import { chunkText } from './utils.js';
+
+function htmlToMarkdown(html: string): string {
+  const dom = new JSDOM(html);
+  const doc = dom.window.document;
+  let md = '';
+
+  function processNode(node: globalThis.Node, depth = 0): void {
+    if (node.nodeType === 3) {
+      const text = (node as globalThis.Text).textContent?.trim();
+      if (text) md += text + ' ';
+      return;
+    }
+    if (node.nodeType !== 1) return;
+    const el = node as globalThis.Element;
+    const tag = el.tagName.toLowerCase();
+
+    switch (tag) {
+      case 'h1': case 'h2': case 'h3': case 'h4': case 'h5': case 'h6': {
+        const level = parseInt(tag[1]);
+        md += '\n' + '#'.repeat(level) + ' ' + el.textContent?.trim() + '\n\n';
+        break;
+      }
+      case 'p':
+        md += '\n' + el.textContent?.trim() + '\n\n';
+        break;
+      case 'br':
+        md += '\n';
+        break;
+      case 'hr':
+        md += '\n---\n\n';
+        break;
+      case 'ul': case 'ol': {
+        el.childNodes.forEach(li => {
+          if (li.nodeType === 1) {
+            const liEl = li as globalThis.Element;
+            if (liEl.tagName.toLowerCase() === 'li') {
+              md += '  '.repeat(depth) + '- ' + liEl.textContent?.trim() + '\n';
+            }
+          }
+        });
+        md += '\n';
+        break;
+      }
+      case 'a': {
+        const href = el.getAttribute('href') || '';
+        const text = el.textContent?.trim() || href;
+        if (href && href !== '#') md += `[${text}](${href}) `;
+        else md += text + ' ';
+        break;
+      }
+      case 'img': {
+        const src = el.getAttribute('src') || '';
+        const alt = el.getAttribute('alt') || '';
+        if (src) md += `![${alt}](${src}) `;
+        break;
+      }
+      case 'strong': case 'b':
+        md += '**' + el.textContent?.trim() + '** ';
+        break;
+      case 'em': case 'i':
+        md += '*' + el.textContent?.trim() + '* ';
+        break;
+      case 'code':
+        md += '`' + el.textContent?.trim() + '` ';
+        break;
+      case 'pre': {
+        const code = el.querySelector('code');
+        const lang = code?.getAttribute('class')?.replace(/^language-/, '') || '';
+        md += '\n```' + lang + '\n' + (code || el).textContent + '\n```\n\n';
+        break;
+      }
+      case 'blockquote':
+        md += '\n> ' + el.textContent?.trim().replace(/\n/g, '\n> ') + '\n\n';
+        break;
+      case 'table': {
+        const rows = el.querySelectorAll('tr');
+        rows.forEach((row, i) => {
+          const cells = row.querySelectorAll('td, th');
+          md += '| ' + Array.from(cells).map(c => c.textContent?.trim()).join(' | ') + ' |\n';
+          if (i === 0 && row.parentElement?.tagName === 'THEAD') {
+            md += '|' + Array.from(cells).map(() => ' --- ').join('|') + '|\n';
+          }
+        });
+        md += '\n';
+        break;
+      }
+      default:
+        el.childNodes.forEach(child => processNode(child, depth));
+    }
+  }
+
+  doc.body?.childNodes.forEach((child: globalThis.Node) => processNode(child));
+  return md.replace(/\n{3,}/g, '\n\n').trim();
+}
+
+function htmlToXml(html: string, root = 'document'): string {
+  const dom = new JSDOM(html);
+  const doc = dom.window.document;
+
+  function serialize(el: globalThis.Element, depth = 0): string {
+    const indent = '  '.repeat(depth);
+    const tag = el.tagName.toLowerCase();
+    if (el.childNodes.length === 0) {
+      return `${indent}<${tag} />\n`;
+    }
+    const text = el.textContent?.trim();
+    if (el.childNodes.length === 1 && el.firstChild?.nodeType === 3 && text) {
+      return `${indent}<${tag}>${escapeXml(text)}</${tag}>\n`;
+    }
+    let xml = `${indent}<${tag}>\n`;
+    el.childNodes.forEach(child => {
+      if (child.nodeType === 1) xml += serialize(child as globalThis.Element, depth + 1);
+      else if (child.nodeType === 3 && (child as globalThis.Text).textContent?.trim()) {
+        xml += `${indent}  ${escapeXml((child as globalThis.Text).textContent!.trim())}\n`;
+      }
+    });
+    xml += `${indent}</${tag}>\n`;
+    return xml;
+  }
+
+  function escapeXml(s: string): string {
+    return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
+  }
+
+  let xml = `<?xml version="1.0" encoding="UTF-8"?>\n<${root}>\n`;
+  doc.body?.childNodes.forEach((child: globalThis.Node) => {
+    if (child.nodeType === 1) xml += serialize(child as globalThis.Element, 1);
+  });
+  xml += `</${root}>\n`;
+  return xml;
+}
+
+function htmlToRedisJson(html: string, key = 'page'): string {
+  const dom = new JSDOM(html);
+  const doc = dom.window.document;
+
+  function extract(el: globalThis.Element): Record<string, unknown> {
+    const tag = el.tagName.toLowerCase();
+    const children = Array.from(el.children);
+    if (children.length === 0) return { _tag: tag, _text: el.textContent?.trim() || '' };
+    const result: Record<string, unknown> = { _tag: tag };
+    if (el.getAttribute('href')) result.href = el.getAttribute('href')!;
+    if (el.getAttribute('src')) result.src = el.getAttribute('src')!;
+    if (el.getAttribute('alt')) result.alt = el.getAttribute('alt')!;
+    if (el.getAttribute('class')) result.class = el.getAttribute('class')!;
+    children.forEach(child => {
+      const childTag = child.tagName.toLowerCase();
+      if (result[childTag]) {
+        if (!Array.isArray(result[childTag])) result[childTag] = [result[childTag]];
+        (result[childTag] as unknown[]).push(extract(child));
+      } else {
+        result[childTag] = extract(child);
+      }
+    });
+    return result;
+  }
+
+  const title = doc.title;
+  const bodyArr: Record<string, unknown>[] = [];
+  doc.body?.childNodes.forEach((child: globalThis.Node) => {
+    if (child.nodeType === 1) bodyArr.push(extract(child as globalThis.Element));
+  });
+
+  const json: Record<string, unknown> = {
+    title,
+    body: bodyArr,
+    links: Array.from(doc.querySelectorAll('a[href]')).map((a) => ({
+      href: (a as globalThis.Element).getAttribute('href'),
+      text: (a as globalThis.Element).textContent?.trim(),
+    })),
+    images: Array.from(doc.querySelectorAll('img[src]')).map((img) => ({
+      src: (img as globalThis.Element).getAttribute('src'),
+      alt: (img as globalThis.Element).getAttribute('alt'),
+    })),
+  };
+
+  return `JSON.SET ${key} $ ${JSON.stringify(json, null, 2)}`;
+}
+
+function plainText(html: string): string {
+  return html
+    .replace(/<style[^>]*>.*?<\/style>/gs, '')
+    .replace(/<script[^>]*>.*?<\/script>/gs, '')
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/p>/gi, '\n\n')
+    .replace(/<\/h[1-6]>/gi, '\n')
+    .replace(/<\/li>/gi, '\n')
+    .replace(/<\/tr>/gi, '\n')
+    .replace(/<\/td>/gi, ' | ')
+    .replace(/<\/th>/gi, ' | ')
+    .replace(/<[^>]+>/g, '')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+}
+
+function tableToMarkdown(headers: string[], rows: string[][], caption?: string): string {
+  let md = caption ? `### ${caption}\n\n` : '';
+  md += '| ' + headers.join(' | ') + ' |\n';
+  md += '|' + headers.map(() => ' --- ').join('|') + '|\n';
+  for (const row of rows) {
+    md += '| ' + row.join(' | ') + ' |\n';
+  }
+  return md + '\n';
+}
+
+export function serializeResult(result: ScrapeResult, options: SerializeOptions): SerializedOutput {
+  let content = '';
+
+  if (options.includeMetadata !== false) {
+    content += `# ${result.title}\n\n`;
+    content += `- **URL:** ${result.url}\n`;
+    content += `- **Extracted:** ${result.extractedAt}\n`;
+    if (result.metadata.author) content += `- **Author:** ${result.metadata.author}\n`;
+    if (result.metadata.publishedDate) content += `- **Published:** ${result.metadata.publishedDate}\n`;
+    content += `- **Size:** ${result.size} bytes\n\n---\n\n`;
+  }
+
+  switch (options.format) {
+    case 'markdown':
+      content += result.content;
+      if (result.tables.length) {
+        content += '\n\n## Tables\n\n';
+        for (const t of result.tables) {
+          content += tableToMarkdown(t.headers, t.rows, t.caption);
+        }
+      }
+      break;
+
+    case 'text':
+      content += result.text;
+      break;
+
+    case 'xml':
+      content = htmlToXml(`<html><head><title>${result.title}</title></head><body>${result.content}</body></html>`, options.xmlRoot || 'document');
+      break;
+
+    case 'json':
+      content = JSON.stringify({
+        title: result.title,
+        url: result.url,
+        metadata: result.metadata,
+        content: result.content,
+        links: result.links,
+        images: result.images,
+        tables: result.tables,
+        extractedAt: result.extractedAt,
+        duration: result.duration,
+      }, null, options.pretty ? 2 : undefined);
+      break;
+
+    case 'redis':
+      content = htmlToRedisJson(`<html><head><title>${result.title}</title></head><body>${result.content}</body></html>`, options.redisKey || `page:${encodeURIComponent(result.url)}`);
+      break;
+  }
+
+  const chunks = options.maxChunkSize ? chunkText(content, options.maxChunkSize) : [];
+
+  const output: SerializedOutput = {
+    format: options.format,
+    content,
+    chunks: chunks.length ? chunks : undefined,
+    size: content.length,
+  };
+
+  if (options.includeMetadata) {
+    output.metadata = {
+      title: result.title,
+      url: result.url,
+      format: result.format,
+      extractedAt: result.extractedAt,
+      duration: result.duration,
+      linkCount: result.links.length,
+      imageCount: result.images.length,
+      tableCount: result.tables.length,
+    };
+  }
+
+  return output;
+}
+
+export function serializeHtml(html: string, options: SerializeOptions): SerializedOutput {
+  const content = (() => {
+    switch (options.format) {
+      case 'markdown': return htmlToMarkdown(html);
+      case 'xml': return htmlToXml(html, options.xmlRoot || 'document');
+      case 'json': return JSON.stringify({ html }, null, options.pretty ? 2 : undefined);
+      case 'redis': return htmlToRedisJson(html, options.redisKey || 'page:1');
+      case 'text': return plainText(html);
+      default: return plainText(html);
+    }
+  })();
+
+  const chunks = options.maxChunkSize ? chunkText(content, options.maxChunkSize) : [];
+
+  return {
+    format: options.format,
+    content,
+    chunks: chunks.length ? chunks : undefined,
+    size: content.length,
+  };
+}

package/scrape/server.ts

@@ -0,0 +1,137 @@
+import { Hono } from 'hono';
+import { cors } from 'hono/cors';
+import { scrapeUrl } from './scrape.js';
+import { batchScrape } from './batch.js';
+import { crawl } from './crawler.js';
+import { formatForAgent } from './agent.js';
+import { randomPort } from './utils/port.js';
+import type { ScrapeOptions } from './types.js';
+
+export interface ServerConfig {
+  port?: number;
+  host?: string;
+  apiKey?: string;
+}
+
+const app = new Hono();
+
+app.use('*', cors());
+
+const api = new Hono();
+
+api.post('/scrape', async (c) => {
+  const body = await c.req.json();
+  const { url, options = {} } = body;
+
+  if (!url) {
+    return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
+  }
+
+  try {
+    const result = await scrapeUrl(url, options as ScrapeOptions);
+    return c.json({ status: 'ok', data: result });
+  } catch (error) {
+    return c.json({
+      error: {
+        code: (error as any).code || 'SCRAPE_ERROR',
+        message: (error as Error).message,
+      }
+    }, (error as any).statusCode || 500);
+  }
+});
+
+api.post('/scrape/agent', async (c) => {
+  const body = await c.req.json();
+  const { url, options = {} } = body;
+
+  if (!url) {
+    return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
+  }
+
+  try {
+    const result = await scrapeUrl(url, options as ScrapeOptions);
+    const agentOutput = formatForAgent(result);
+    return c.json({ status: 'ok', data: agentOutput });
+  } catch (error) {
+    return c.json({
+      error: {
+        code: (error as any).code || 'SCRAPE_ERROR',
+        message: (error as Error).message,
+      }
+    }, (error as any).statusCode || 500);
+  }
+});
+
+api.post('/batch', async (c) => {
+  const body = await c.req.json();
+  const { urls, options = {}, scrapeOptions = {} } = body;
+
+  if (!urls || !Array.isArray(urls) || urls.length === 0) {
+    return c.json({ error: { code: 'MISSING_URLS', message: 'urls array is required' } }, 400);
+  }
+
+  try {
+    const result = await batchScrape({ urls, ...options }, scrapeOptions as ScrapeOptions);
+    return c.json({
+      status: 'ok',
+      data: {
+        successful: result.successful,
+        failed: result.failed,
+        duration: result.duration,
+        results: result.results,
+        errors: result.errors.map(e => ({ url: e.url, error: e.error.message })),
+      },
+    });
+  } catch (error) {
+    return c.json({
+      error: {
+        code: 'BATCH_ERROR',
+        message: (error as Error).message,
+      }
+    }, 500);
+  }
+});
+
+api.post('/crawl', async (c) => {
+  const body = await c.req.json();
+  const { url, options = {} } = body;
+
+  if (!url) {
+    return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
+  }
+
+  try {
+    const result = await crawl(url, options);
+    return c.json({
+      status: 'ok',
+      data: {
+        stats: result.stats,
+        results: result.results.slice(0, 100),
+      },
+    });
+  } catch (error) {
+    return c.json({
+      error: {
+        code: 'CRAWL_ERROR',
+        message: (error as Error).message,
+      }
+    }, 500);
+  }
+});
+
+api.get('/health', (c) => {
+  return c.json({ status: 'ok', version: '2.0.0', uptime: process.uptime() });
+});
+
+app.route('/v1', api);
+
+export function createServer(config: ServerConfig = {}) {
+  return {
+    fetch: app.fetch,
+    port: config.port || randomPort(),
+    host: config.host || '0.0.0.0',
+  };
+}
+
+export { app };
+export default app;

package/scrape/session.ts

@@ -0,0 +1,109 @@
+export interface Cookie {
+  name: string;
+  value: string;
+  domain?: string;
+  path?: string;
+  expires?: number;
+  httpOnly?: boolean;
+  secure?: boolean;
+}
+
+export class CookieJar {
+  private cookies = new Map<string, Cookie>();
+
+  set(cookie: Cookie): void {
+    const key = `${cookie.domain || ''}:${cookie.path || '/'}:${cookie.name}`;
+    this.cookies.set(key, cookie);
+  }
+
+  get(name: string, domain?: string, _path?: string): Cookie | undefined {
+    for (const [key, cookie] of this.cookies) {
+      if (cookie.name !== name) continue;
+      if (domain && cookie.domain && !domain.includes(cookie.domain)) continue;
+      if (cookie.expires && cookie.expires < Date.now()) {
+        this.cookies.delete(key);
+        continue;
+      }
+      return cookie;
+    }
+    return undefined;
+  }
+
+  getAll(domain?: string): Cookie[] {
+    const result: Cookie[] = [];
+    const now = Date.now();
+    for (const [key, cookie] of this.cookies) {
+      if (cookie.expires && cookie.expires < now) {
+        this.cookies.delete(key);
+        continue;
+      }
+      if (domain && cookie.domain && !domain.includes(cookie.domain)) continue;
+      result.push(cookie);
+    }
+    return result;
+  }
+
+  toString(domain?: string): string {
+    return this.getAll(domain)
+      .map(c => `${c.name}=${c.value}`)
+      .join('; ');
+  }
+
+  parseSetCookie(header: string, defaultDomain?: string): void {
+    const parts = header.split(';').map(s => s.trim());
+    const [nameValue, ...attrs] = parts;
+    const [name, ...valueParts] = nameValue.split('=');
+    const value = valueParts.join('=');
+
+    const cookie: Cookie = {
+      name: name.trim(),
+      value: value.trim(),
+      domain: defaultDomain,
+      path: '/',
+    };
+
+    for (const attr of attrs) {
+      const [key, val] = attr.split('=').map(s => s.trim());
+      const lower = key.toLowerCase();
+      if (lower === 'domain') cookie.domain = val;
+      else if (lower === 'path') cookie.path = val;
+      else if (lower === 'expires') cookie.expires = new Date(val).getTime();
+      else if (lower === 'httponly') cookie.httpOnly = true;
+      else if (lower === 'secure') cookie.secure = true;
+    }
+
+    this.set(cookie);
+  }
+
+  clear(): void {
+    this.cookies.clear();
+  }
+}
+
+export class ScrapingSession {
+  public readonly cookieJar: CookieJar;
+  public readonly id: string;
+  private data = new Map<string, unknown>();
+
+  constructor(id?: string) {
+    this.id = id || crypto.randomUUID();
+    this.cookieJar = new CookieJar();
+  }
+
+  get<T = unknown>(key: string): T | undefined {
+    return this.data.get(key) as T;
+  }
+
+  set(key: string, value: unknown): void {
+    this.data.set(key, value);
+  }
+
+  clear(): void {
+    this.cookieJar.clear();
+    this.data.clear();
+  }
+}
+
+export function createSession(id?: string): ScrapingSession {
+  return new ScrapingSession(id);
+}