@wenathlan/saddle 1.8.8 → 1.8.9

package/scrape/serialize.ts

@@ -1,310 +0,0 @@
-import { JSDOM } from 'jsdom';
-import type { SerializeOptions, SerializedOutput, ScrapeResult } from './types.js';
-import { chunkText } from './utils.js';
-
-function htmlToMarkdown(html: string): string {
-  const dom = new JSDOM(html);
-  const doc = dom.window.document;
-  let md = '';
-
-  function processNode(node: globalThis.Node, depth = 0): void {
-    if (node.nodeType === 3) {
-      const text = (node as globalThis.Text).textContent?.trim();
-      if (text) md += text + ' ';
-      return;
-    }
-    if (node.nodeType !== 1) return;
-    const el = node as globalThis.Element;
-    const tag = el.tagName.toLowerCase();
-
-    switch (tag) {
-      case 'h1': case 'h2': case 'h3': case 'h4': case 'h5': case 'h6': {
-        const level = parseInt(tag[1]);
-        md += '\n' + '#'.repeat(level) + ' ' + el.textContent?.trim() + '\n\n';
-        break;
-      }
-      case 'p':
-        md += '\n' + el.textContent?.trim() + '\n\n';
-        break;
-      case 'br':
-        md += '\n';
-        break;
-      case 'hr':
-        md += '\n---\n\n';
-        break;
-      case 'ul': case 'ol': {
-        el.childNodes.forEach(li => {
-          if (li.nodeType === 1) {
-            const liEl = li as globalThis.Element;
-            if (liEl.tagName.toLowerCase() === 'li') {
-              md += '  '.repeat(depth) + '- ' + liEl.textContent?.trim() + '\n';
-            }
-          }
-        });
-        md += '\n';
-        break;
-      }
-      case 'a': {
-        const href = el.getAttribute('href') || '';
-        const text = el.textContent?.trim() || href;
-        if (href && href !== '#') md += `[${text}](${href}) `;
-        else md += text + ' ';
-        break;
-      }
-      case 'img': {
-        const src = el.getAttribute('src') || '';
-        const alt = el.getAttribute('alt') || '';
-        if (src) md += `![${alt}](${src}) `;
-        break;
-      }
-      case 'strong': case 'b':
-        md += '**' + el.textContent?.trim() + '** ';
-        break;
-      case 'em': case 'i':
-        md += '*' + el.textContent?.trim() + '* ';
-        break;
-      case 'code':
-        md += '`' + el.textContent?.trim() + '` ';
-        break;
-      case 'pre': {
-        const code = el.querySelector('code');
-        const lang = code?.getAttribute('class')?.replace(/^language-/, '') || '';
-        md += '\n```' + lang + '\n' + (code || el).textContent + '\n```\n\n';
-        break;
-      }
-      case 'blockquote':
-        md += '\n> ' + el.textContent?.trim().replace(/\n/g, '\n> ') + '\n\n';
-        break;
-      case 'table': {
-        const rows = el.querySelectorAll('tr');
-        rows.forEach((row, i) => {
-          const cells = row.querySelectorAll('td, th');
-          md += '| ' + Array.from(cells).map(c => c.textContent?.trim()).join(' | ') + ' |\n';
-          if (i === 0 && row.parentElement?.tagName === 'THEAD') {
-            md += '|' + Array.from(cells).map(() => ' --- ').join('|') + '|\n';
-          }
-        });
-        md += '\n';
-        break;
-      }
-      default:
-        el.childNodes.forEach(child => processNode(child, depth));
-    }
-  }
-
-  doc.body?.childNodes.forEach((child: globalThis.Node) => processNode(child));
-  return md.replace(/\n{3,}/g, '\n\n').trim();
-}
-
-function htmlToXml(html: string, root = 'document'): string {
-  const dom = new JSDOM(html);
-  const doc = dom.window.document;
-
-  function serialize(el: globalThis.Element, depth = 0): string {
-    const indent = '  '.repeat(depth);
-    const tag = el.tagName.toLowerCase();
-    if (el.childNodes.length === 0) {
-      return `${indent}<${tag} />\n`;
-    }
-    const text = el.textContent?.trim();
-    if (el.childNodes.length === 1 && el.firstChild?.nodeType === 3 && text) {
-      return `${indent}<${tag}>${escapeXml(text)}</${tag}>\n`;
-    }
-    let xml = `${indent}<${tag}>\n`;
-    el.childNodes.forEach(child => {
-      if (child.nodeType === 1) xml += serialize(child as globalThis.Element, depth + 1);
-      else if (child.nodeType === 3 && (child as globalThis.Text).textContent?.trim()) {
-        xml += `${indent}  ${escapeXml((child as globalThis.Text).textContent!.trim())}\n`;
-      }
-    });
-    xml += `${indent}</${tag}>\n`;
-    return xml;
-  }
-
-  function escapeXml(s: string): string {
-    return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
-  }
-
-  let xml = `<?xml version="1.0" encoding="UTF-8"?>\n<${root}>\n`;
-  doc.body?.childNodes.forEach((child: globalThis.Node) => {
-    if (child.nodeType === 1) xml += serialize(child as globalThis.Element, 1);
-  });
-  xml += `</${root}>\n`;
-  return xml;
-}
-
-function htmlToRedisJson(html: string, key = 'page'): string {
-  const dom = new JSDOM(html);
-  const doc = dom.window.document;
-
-  function extract(el: globalThis.Element): Record<string, unknown> {
-    const tag = el.tagName.toLowerCase();
-    const children = Array.from(el.children);
-    if (children.length === 0) return { _tag: tag, _text: el.textContent?.trim() || '' };
-    const result: Record<string, unknown> = { _tag: tag };
-    if (el.getAttribute('href')) result.href = el.getAttribute('href')!;
-    if (el.getAttribute('src')) result.src = el.getAttribute('src')!;
-    if (el.getAttribute('alt')) result.alt = el.getAttribute('alt')!;
-    if (el.getAttribute('class')) result.class = el.getAttribute('class')!;
-    children.forEach(child => {
-      const childTag = child.tagName.toLowerCase();
-      if (result[childTag]) {
-        if (!Array.isArray(result[childTag])) result[childTag] = [result[childTag]];
-        (result[childTag] as unknown[]).push(extract(child));
-      } else {
-        result[childTag] = extract(child);
-      }
-    });
-    return result;
-  }
-
-  const title = doc.title;
-  const bodyArr: Record<string, unknown>[] = [];
-  doc.body?.childNodes.forEach((child: globalThis.Node) => {
-    if (child.nodeType === 1) bodyArr.push(extract(child as globalThis.Element));
-  });
-
-  const json: Record<string, unknown> = {
-    title,
-    body: bodyArr,
-    links: Array.from(doc.querySelectorAll('a[href]')).map((a) => ({
-      href: (a as globalThis.Element).getAttribute('href'),
-      text: (a as globalThis.Element).textContent?.trim(),
-    })),
-    images: Array.from(doc.querySelectorAll('img[src]')).map((img) => ({
-      src: (img as globalThis.Element).getAttribute('src'),
-      alt: (img as globalThis.Element).getAttribute('alt'),
-    })),
-  };
-
-  return `JSON.SET ${key} $ ${JSON.stringify(json, null, 2)}`;
-}
-
-function plainText(html: string): string {
-  return html
-    .replace(/<style[^>]*>.*?<\/style>/gs, '')
-    .replace(/<script[^>]*>.*?<\/script>/gs, '')
-    .replace(/<br\s*\/?>/gi, '\n')
-    .replace(/<\/p>/gi, '\n\n')
-    .replace(/<\/h[1-6]>/gi, '\n')
-    .replace(/<\/li>/gi, '\n')
-    .replace(/<\/tr>/gi, '\n')
-    .replace(/<\/td>/gi, ' | ')
-    .replace(/<\/th>/gi, ' | ')
-    .replace(/<[^>]+>/g, '')
-    .replace(/&nbsp;/g, ' ')
-    .replace(/&amp;/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/\n{3,}/g, '\n\n')
-    .trim();
-}
-
-function tableToMarkdown(headers: string[], rows: string[][], caption?: string): string {
-  let md = caption ? `### ${caption}\n\n` : '';
-  md += '| ' + headers.join(' | ') + ' |\n';
-  md += '|' + headers.map(() => ' --- ').join('|') + '|\n';
-  for (const row of rows) {
-    md += '| ' + row.join(' | ') + ' |\n';
-  }
-  return md + '\n';
-}
-
-export function serializeResult(result: ScrapeResult, options: SerializeOptions): SerializedOutput {
-  let content = '';
-
-  if (options.includeMetadata !== false) {
-    content += `# ${result.title}\n\n`;
-    content += `- **URL:** ${result.url}\n`;
-    content += `- **Extracted:** ${result.extractedAt}\n`;
-    if (result.metadata.author) content += `- **Author:** ${result.metadata.author}\n`;
-    if (result.metadata.publishedDate) content += `- **Published:** ${result.metadata.publishedDate}\n`;
-    content += `- **Size:** ${result.size} bytes\n\n---\n\n`;
-  }
-
-  switch (options.format) {
-    case 'markdown':
-      content += result.content;
-      if (result.tables.length) {
-        content += '\n\n## Tables\n\n';
-        for (const t of result.tables) {
-          content += tableToMarkdown(t.headers, t.rows, t.caption);
-        }
-      }
-      break;
-
-    case 'text':
-      content += result.text;
-      break;
-
-    case 'xml':
-      content = htmlToXml(`<html><head><title>${result.title}</title></head><body>${result.content}</body></html>`, options.xmlRoot || 'document');
-      break;
-
-    case 'json':
-      content = JSON.stringify({
-        title: result.title,
-        url: result.url,
-        metadata: result.metadata,
-        content: result.content,
-        links: result.links,
-        images: result.images,
-        tables: result.tables,
-        extractedAt: result.extractedAt,
-        duration: result.duration,
-      }, null, options.pretty ? 2 : undefined);
-      break;
-
-    case 'redis':
-      content = htmlToRedisJson(`<html><head><title>${result.title}</title></head><body>${result.content}</body></html>`, options.redisKey || `page:${encodeURIComponent(result.url)}`);
-      break;
-  }
-
-  const chunks = options.maxChunkSize ? chunkText(content, options.maxChunkSize) : [];
-
-  const output: SerializedOutput = {
-    format: options.format,
-    content,
-    chunks: chunks.length ? chunks : undefined,
-    size: content.length,
-  };
-
-  if (options.includeMetadata) {
-    output.metadata = {
-      title: result.title,
-      url: result.url,
-      format: result.format,
-      extractedAt: result.extractedAt,
-      duration: result.duration,
-      linkCount: result.links.length,
-      imageCount: result.images.length,
-      tableCount: result.tables.length,
-    };
-  }
-
-  return output;
-}
-
-export function serializeHtml(html: string, options: SerializeOptions): SerializedOutput {
-  const content = (() => {
-    switch (options.format) {
-      case 'markdown': return htmlToMarkdown(html);
-      case 'xml': return htmlToXml(html, options.xmlRoot || 'document');
-      case 'json': return JSON.stringify({ html }, null, options.pretty ? 2 : undefined);
-      case 'redis': return htmlToRedisJson(html, options.redisKey || 'page:1');
-      case 'text': return plainText(html);
-      default: return plainText(html);
-    }
-  })();
-
-  const chunks = options.maxChunkSize ? chunkText(content, options.maxChunkSize) : [];
-
-  return {
-    format: options.format,
-    content,
-    chunks: chunks.length ? chunks : undefined,
-    size: content.length,
-  };
-}

package/scrape/server.ts

@@ -1,137 +0,0 @@
-import { Hono } from 'hono';
-import { cors } from 'hono/cors';
-import { scrapeUrl } from './scrape.js';
-import { batchScrape } from './batch.js';
-import { crawl } from './crawler.js';
-import { formatForAgent } from './agent.js';
-import { randomPort } from './utils/port.js';
-import type { ScrapeOptions } from './types.js';
-
-export interface ServerConfig {
-  port?: number;
-  host?: string;
-  apiKey?: string;
-}
-
-const app = new Hono();
-
-app.use('*', cors());
-
-const api = new Hono();
-
-api.post('/scrape', async (c) => {
-  const body = await c.req.json();
-  const { url, options = {} } = body;
-
-  if (!url) {
-    return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
-  }
-
-  try {
-    const result = await scrapeUrl(url, options as ScrapeOptions);
-    return c.json({ status: 'ok', data: result });
-  } catch (error) {
-    return c.json({
-      error: {
-        code: (error as any).code || 'SCRAPE_ERROR',
-        message: (error as Error).message,
-      }
-    }, (error as any).statusCode || 500);
-  }
-});
-
-api.post('/scrape/agent', async (c) => {
-  const body = await c.req.json();
-  const { url, options = {} } = body;
-
-  if (!url) {
-    return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
-  }
-
-  try {
-    const result = await scrapeUrl(url, options as ScrapeOptions);
-    const agentOutput = formatForAgent(result);
-    return c.json({ status: 'ok', data: agentOutput });
-  } catch (error) {
-    return c.json({
-      error: {
-        code: (error as any).code || 'SCRAPE_ERROR',
-        message: (error as Error).message,
-      }
-    }, (error as any).statusCode || 500);
-  }
-});
-
-api.post('/batch', async (c) => {
-  const body = await c.req.json();
-  const { urls, options = {}, scrapeOptions = {} } = body;
-
-  if (!urls || !Array.isArray(urls) || urls.length === 0) {
-    return c.json({ error: { code: 'MISSING_URLS', message: 'urls array is required' } }, 400);
-  }
-
-  try {
-    const result = await batchScrape({ urls, ...options }, scrapeOptions as ScrapeOptions);
-    return c.json({
-      status: 'ok',
-      data: {
-        successful: result.successful,
-        failed: result.failed,
-        duration: result.duration,
-        results: result.results,
-        errors: result.errors.map(e => ({ url: e.url, error: e.error.message })),
-      },
-    });
-  } catch (error) {
-    return c.json({
-      error: {
-        code: 'BATCH_ERROR',
-        message: (error as Error).message,
-      }
-    }, 500);
-  }
-});
-
-api.post('/crawl', async (c) => {
-  const body = await c.req.json();
-  const { url, options = {} } = body;
-
-  if (!url) {
-    return c.json({ error: { code: 'MISSING_URL', message: 'URL is required' } }, 400);
-  }
-
-  try {
-    const result = await crawl(url, options);
-    return c.json({
-      status: 'ok',
-      data: {
-        stats: result.stats,
-        results: result.results.slice(0, 100),
-      },
-    });
-  } catch (error) {
-    return c.json({
-      error: {
-        code: 'CRAWL_ERROR',
-        message: (error as Error).message,
-      }
-    }, 500);
-  }
-});
-
-api.get('/health', (c) => {
-  return c.json({ status: 'ok', version: '2.0.0', uptime: process.uptime() });
-});
-
-app.route('/v1', api);
-
-export function createServer(config: ServerConfig = {}) {
-  return {
-    fetch: app.fetch,
-    port: config.port || randomPort(),
-    host: config.host || '0.0.0.0',
-  };
-}
-
-export { app };
-export default app;

package/scrape/session.ts

@@ -1,109 +0,0 @@
-export interface Cookie {
-  name: string;
-  value: string;
-  domain?: string;
-  path?: string;
-  expires?: number;
-  httpOnly?: boolean;
-  secure?: boolean;
-}
-
-export class CookieJar {
-  private cookies = new Map<string, Cookie>();
-
-  set(cookie: Cookie): void {
-    const key = `${cookie.domain || ''}:${cookie.path || '/'}:${cookie.name}`;
-    this.cookies.set(key, cookie);
-  }
-
-  get(name: string, domain?: string, _path?: string): Cookie | undefined {
-    for (const [key, cookie] of this.cookies) {
-      if (cookie.name !== name) continue;
-      if (domain && cookie.domain && !domain.includes(cookie.domain)) continue;
-      if (cookie.expires && cookie.expires < Date.now()) {
-        this.cookies.delete(key);
-        continue;
-      }
-      return cookie;
-    }
-    return undefined;
-  }
-
-  getAll(domain?: string): Cookie[] {
-    const result: Cookie[] = [];
-    const now = Date.now();
-    for (const [key, cookie] of this.cookies) {
-      if (cookie.expires && cookie.expires < now) {
-        this.cookies.delete(key);
-        continue;
-      }
-      if (domain && cookie.domain && !domain.includes(cookie.domain)) continue;
-      result.push(cookie);
-    }
-    return result;
-  }
-
-  toString(domain?: string): string {
-    return this.getAll(domain)
-      .map(c => `${c.name}=${c.value}`)
-      .join('; ');
-  }
-
-  parseSetCookie(header: string, defaultDomain?: string): void {
-    const parts = header.split(';').map(s => s.trim());
-    const [nameValue, ...attrs] = parts;
-    const [name, ...valueParts] = nameValue.split('=');
-    const value = valueParts.join('=');
-
-    const cookie: Cookie = {
-      name: name.trim(),
-      value: value.trim(),
-      domain: defaultDomain,
-      path: '/',
-    };
-
-    for (const attr of attrs) {
-      const [key, val] = attr.split('=').map(s => s.trim());
-      const lower = key.toLowerCase();
-      if (lower === 'domain') cookie.domain = val;
-      else if (lower === 'path') cookie.path = val;
-      else if (lower === 'expires') cookie.expires = new Date(val).getTime();
-      else if (lower === 'httponly') cookie.httpOnly = true;
-      else if (lower === 'secure') cookie.secure = true;
-    }
-
-    this.set(cookie);
-  }
-
-  clear(): void {
-    this.cookies.clear();
-  }
-}
-
-export class ScrapingSession {
-  public readonly cookieJar: CookieJar;
-  public readonly id: string;
-  private data = new Map<string, unknown>();
-
-  constructor(id?: string) {
-    this.id = id || crypto.randomUUID();
-    this.cookieJar = new CookieJar();
-  }
-
-  get<T = unknown>(key: string): T | undefined {
-    return this.data.get(key) as T;
-  }
-
-  set(key: string, value: unknown): void {
-    this.data.set(key, value);
-  }
-
-  clear(): void {
-    this.cookieJar.clear();
-    this.data.clear();
-  }
-}
-
-export function createSession(id?: string): ScrapingSession {
-  return new ScrapingSession(id);
-}

package/scrape/sitemap.ts

@@ -1,131 +0,0 @@
-import { WebScrapeError, ErrorCode } from './errors.js';
-
-export interface SitemapUrl {
-  loc: string;
-  lastmod?: string;
-  changefreq?: string;
-  priority?: number;
-}
-
-export interface SitemapResult {
-  urls: SitemapUrl[];
-  sitemaps: string[];
-  duration: number;
-}
-
-function parseSitemapXml(xml: string): SitemapUrl[] {
-  const urls: SitemapUrl[] = [];
-  const urlMatches = xml.match(/<url[^>]*>([\s\S]*?)<\/url>/gi) || [];
-
-  for (const urlBlock of urlMatches) {
-    const loc = urlBlock.match(/<loc>([\s\S]*?)<\/loc>/i)?.[1]?.trim();
-    if (!loc) continue;
-
-    const lastmod = urlBlock.match(/<lastmod>([\s\S]*?)<\/lastmod>/i)?.[1]?.trim();
-    const changefreq = urlBlock.match(/<changefreq>([\s\S]*?)<\/changefreq>/i)?.[1]?.trim();
-    const priorityStr = urlBlock.match(/<priority>([\s\S]*?)<\/priority>/i)?.[1]?.trim();
-    const priority = priorityStr ? parseFloat(priorityStr) : undefined;
-
-    urls.push({ loc, lastmod, changefreq, priority });
-  }
-
-  return urls;
-}
-
-function extractSitemapUrls(xml: string): string[] {
-  const urls: string[] = [];
-  const sitemapMatches = xml.match(/<sitemap[^>]*>[\s\S]*?<loc>([\s\S]*?)<\/loc>[\s\S]*?<\/sitemap>/gi) || [];
-  for (const match of sitemapMatches) {
-    const loc = match.match(/<loc>([\s\S]*?)<\/loc>/i)?.[1]?.trim();
-    if (loc) urls.push(loc);
-  }
-  return urls;
-}
-
-export async function fetchSitemap(url: string, options: { timeout?: number; headers?: Record<string, string> } = {}): Promise<SitemapResult> {
-  const startTime = Date.now();
-
-  try {
-    const response = await fetch(url, {
-      headers: {
-        'User-Agent': 'DevThink-WebScrape/2.0 (sitemap parser)',
-        ...options.headers,
-      },
-      signal: AbortSignal.timeout(options.timeout || 10000),
-    });
-
-    if (!response.ok) {
-      throw new WebScrapeError(`Failed to fetch sitemap: ${response.status}`, ErrorCode.NETWORK_ERROR, response.status, false, { url });
-    }
-
-    const text = await response.text();
-    const isIndex = text.includes('<sitemapindex');
-    const urls = isIndex ? [] : parseSitemapXml(text);
-    const sitemaps = isIndex ? extractSitemapUrls(text) : [];
-
-    return {
-      urls,
-      sitemaps,
-      duration: Date.now() - startTime,
-    };
-  } catch (error) {
-    if (error instanceof WebScrapeError) throw error;
-    throw new WebScrapeError(`Failed to parse sitemap: ${(error as Error).message}`, ErrorCode.PARSE_ERROR, 422, false, { url }, error as Error);
-  }
-}
-
-export async function discoverSitemaps(siteUrl: string): Promise<string[]> {
-  const candidates = [
-    '/sitemap.xml',
-    '/sitemap_index.xml',
-    '/sitemap-index.xml',
-    '/sitemaps.xml',
-    '/robots.txt',
-  ];
-
-  const found: string[] = [];
-  const base = siteUrl.replace(/\/$/, '');
-
-  for (const path of candidates) {
-    try {
-      const url = path === '/robots.txt' ? `${base}/robots.txt` : `${base}${path}`;
-      const response = await fetch(url, {
-        headers: { 'User-Agent': 'DevThink-WebScrape/2.0' },
-        signal: AbortSignal.timeout(5000),
-      });
-      if (response.ok) {
-        const text = await response.text();
-        if (path === '/robots.txt') {
-          const sitemapUrls = text.match(/Sitemap:\s*(https?:\/\/[^\s]+)/gi);
-          if (sitemapUrls) {
-            for (const s of sitemapUrls) {
-              found.push(s.replace(/Sitemap:\s*/i, '').trim());
-            }
-          }
-        } else {
-          found.push(url);
-        }
-      }
-    } catch {}
-  }
-
-  return [...new Set(found)];
-}
-
-export async function parseSitemap(url: string, options: { timeout?: number; followIndexes?: boolean; maxUrls?: number } = {}): Promise<SitemapUrl[]> {
-  const { followIndexes = true, maxUrls = 10000 } = options;
-  const result = await fetchSitemap(url, options);
-  let allUrls = [...result.urls];
-
-  if (followIndexes && result.sitemaps.length > 0) {
-    for (const sitemapUrl of result.sitemaps) {
-      if (allUrls.length >= maxUrls) break;
-      try {
-        const subResult = await fetchSitemap(sitemapUrl, options);
-        allUrls.push(...subResult.urls);
-      } catch {}
-    }
-  }
-
-  return allUrls.slice(0, maxUrls);
-}