@wenathlan/saddle 1.8.2 → 1.8.5

package/scrape/sitemap.ts

@@ -0,0 +1,131 @@
+import { WebScrapeError, ErrorCode } from './errors.js';
+
+export interface SitemapUrl {
+  loc: string;
+  lastmod?: string;
+  changefreq?: string;
+  priority?: number;
+}
+
+export interface SitemapResult {
+  urls: SitemapUrl[];
+  sitemaps: string[];
+  duration: number;
+}
+
+function parseSitemapXml(xml: string): SitemapUrl[] {
+  const urls: SitemapUrl[] = [];
+  const urlMatches = xml.match(/<url[^>]*>([\s\S]*?)<\/url>/gi) || [];
+
+  for (const urlBlock of urlMatches) {
+    const loc = urlBlock.match(/<loc>([\s\S]*?)<\/loc>/i)?.[1]?.trim();
+    if (!loc) continue;
+
+    const lastmod = urlBlock.match(/<lastmod>([\s\S]*?)<\/lastmod>/i)?.[1]?.trim();
+    const changefreq = urlBlock.match(/<changefreq>([\s\S]*?)<\/changefreq>/i)?.[1]?.trim();
+    const priorityStr = urlBlock.match(/<priority>([\s\S]*?)<\/priority>/i)?.[1]?.trim();
+    const priority = priorityStr ? parseFloat(priorityStr) : undefined;
+
+    urls.push({ loc, lastmod, changefreq, priority });
+  }
+
+  return urls;
+}
+
+function extractSitemapUrls(xml: string): string[] {
+  const urls: string[] = [];
+  const sitemapMatches = xml.match(/<sitemap[^>]*>[\s\S]*?<loc>([\s\S]*?)<\/loc>[\s\S]*?<\/sitemap>/gi) || [];
+  for (const match of sitemapMatches) {
+    const loc = match.match(/<loc>([\s\S]*?)<\/loc>/i)?.[1]?.trim();
+    if (loc) urls.push(loc);
+  }
+  return urls;
+}
+
+export async function fetchSitemap(url: string, options: { timeout?: number; headers?: Record<string, string> } = {}): Promise<SitemapResult> {
+  const startTime = Date.now();
+
+  try {
+    const response = await fetch(url, {
+      headers: {
+        'User-Agent': 'DevThink-WebScrape/2.0 (sitemap parser)',
+        ...options.headers,
+      },
+      signal: AbortSignal.timeout(options.timeout || 10000),
+    });
+
+    if (!response.ok) {
+      throw new WebScrapeError(`Failed to fetch sitemap: ${response.status}`, ErrorCode.NETWORK_ERROR, response.status, false, { url });
+    }
+
+    const text = await response.text();
+    const isIndex = text.includes('<sitemapindex');
+    const urls = isIndex ? [] : parseSitemapXml(text);
+    const sitemaps = isIndex ? extractSitemapUrls(text) : [];
+
+    return {
+      urls,
+      sitemaps,
+      duration: Date.now() - startTime,
+    };
+  } catch (error) {
+    if (error instanceof WebScrapeError) throw error;
+    throw new WebScrapeError(`Failed to parse sitemap: ${(error as Error).message}`, ErrorCode.PARSE_ERROR, 422, false, { url }, error as Error);
+  }
+}
+
+export async function discoverSitemaps(siteUrl: string): Promise<string[]> {
+  const candidates = [
+    '/sitemap.xml',
+    '/sitemap_index.xml',
+    '/sitemap-index.xml',
+    '/sitemaps.xml',
+    '/robots.txt',
+  ];
+
+  const found: string[] = [];
+  const base = siteUrl.replace(/\/$/, '');
+
+  for (const path of candidates) {
+    try {
+      const url = path === '/robots.txt' ? `${base}/robots.txt` : `${base}${path}`;
+      const response = await fetch(url, {
+        headers: { 'User-Agent': 'DevThink-WebScrape/2.0' },
+        signal: AbortSignal.timeout(5000),
+      });
+      if (response.ok) {
+        const text = await response.text();
+        if (path === '/robots.txt') {
+          const sitemapUrls = text.match(/Sitemap:\s*(https?:\/\/[^\s]+)/gi);
+          if (sitemapUrls) {
+            for (const s of sitemapUrls) {
+              found.push(s.replace(/Sitemap:\s*/i, '').trim());
+            }
+          }
+        } else {
+          found.push(url);
+        }
+      }
+    } catch {}
+  }
+
+  return [...new Set(found)];
+}
+
+export async function parseSitemap(url: string, options: { timeout?: number; followIndexes?: boolean; maxUrls?: number } = {}): Promise<SitemapUrl[]> {
+  const { followIndexes = true, maxUrls = 10000 } = options;
+  const result = await fetchSitemap(url, options);
+  let allUrls = [...result.urls];
+
+  if (followIndexes && result.sitemaps.length > 0) {
+    for (const sitemapUrl of result.sitemaps) {
+      if (allUrls.length >= maxUrls) break;
+      try {
+        const subResult = await fetchSitemap(sitemapUrl, options);
+        allUrls.push(...subResult.urls);
+      } catch {}
+    }
+  }
+
+  return allUrls.slice(0, maxUrls);
+}

package/scrape/tokens.ts

@@ -0,0 +1,45 @@
+export type ModelType = 'gpt-4o' | 'gpt-4' | 'gpt-3.5-turbo' | 'claude' | 'gemini' | 'default';
+
+const MODEL_TOKEN_RATIOS: Record<ModelType, number> = {
+  'gpt-4o': 3.5,
+  'gpt-4': 3.5,
+  'gpt-3.5-turbo': 4,
+  'claude': 3.2,
+  'gemini': 3.5,
+  'default': 4,
+};
+
+export function estimateTokens(text: string, model: ModelType = 'default'): number {
+  const ratio = MODEL_TOKEN_RATIOS[model] || 4;
+  return Math.ceil(text.length / ratio);
+}
+
+export function countTokens(text: string, model: ModelType = 'default'): number {
+  return estimateTokens(text, model);
+}
+
+export function fitsInContext(text: string, maxTokens: number, model: ModelType = 'default'): boolean {
+  return estimateTokens(text, model) <= maxTokens;
+}
+
+export function truncateToTokens(text: string, maxTokens: number, model: ModelType = 'default'): string {
+  const estimatedTokens = estimateTokens(text, model);
+  if (estimatedTokens <= maxTokens) return text;
+  const charLimit = Math.floor(maxTokens * (MODEL_TOKEN_RATIOS[model] || 4));
+  return text.slice(0, charLimit) + '...';
+}
+
+export function tokenCost(text: string, model: ModelType, pricePer1kTokens: number): number {
+  const tokens = estimateTokens(text, model);
+  return (tokens / 1000) * pricePer1kTokens;
+}
+
+export const MODEL_PRICES: Record<string, { input: number; output: number }> = {
+  'gpt-4o': { input: 0.0025, output: 0.01 },
+  'gpt-4o-mini': { input: 0.00015, output: 0.0006 },
+  'gpt-4-turbo': { input: 0.01, output: 0.03 },
+  'claude-3-5-sonnet': { input: 0.003, output: 0.015 },
+  'claude-3-haiku': { input: 0.00025, output: 0.00125 },
+  'gemini-1.5-pro': { input: 0.00125, output: 0.005 },
+  'gemini-1.5-flash': { input: 0.000075, output: 0.0003 },
+};

package/scrape/tsconfig.json

@@ -0,0 +1,28 @@
+{
+	"compilerOptions": {
+		"target": "ES2024",
+		"module": "ESNext",
+		"lib": ["ES2024", "DOM", "DOM.Iterable"],
+		"moduleResolution": "bundler",
+		"strict": true,
+		"declaration": true,
+		"declarationMap": true,
+		"sourceMap": true,
+		"outDir": "./dist",
+		"rootDir": ".",
+		"noEmit": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true,
+		"resolveJsonModule": true,
+		"isolatedModules": true,
+		"verbatimModuleSyntax": true,
+		"noUnusedLocals": true,
+		"noUnusedParameters": true,
+		"noImplicitReturns": true,
+		"noFallthroughCasesInSwitch": true,
+		"types": ["node"]
+	},
+	"include": ["*.ts"],
+	"exclude": ["node_modules", "dist", "web", "vite.config.ts", "vitest.config.ts", "tests"]
+}

package/scrape/types.ts

@@ -0,0 +1,214 @@
+import { z } from 'zod';
+
+export const ScrapeFormatSchema = z.enum(['markdown', 'html', 'text', 'json', 'xml', 'redis']);
+export type ScrapeFormat = z.infer<typeof ScrapeFormatSchema>;
+
+export const SerializeTargetSchema = z.enum(['markdown', 'xml', 'json', 'redis', 'text']);
+export type SerializeTarget = z.infer<typeof SerializeTargetSchema>;
+
+export const ScrapeOptionsSchema = z.strictObject({
+  url: z.string().url().optional(),
+  html: z.string().optional(),
+  mode: z.enum(['auto', 'fetch', 'browser']).prefault('auto').optional(),
+  waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle', 'commit']).prefault('networkidle').optional(),
+  timeout: z.number().positive().prefault(30000).optional(),
+  scroll: z.boolean().prefault(false).optional(),
+  scrollDelay: z.number().nonnegative().prefault(300).optional(),
+  maxScrolls: z.number().positive().prefault(50).optional(),
+  removeSelectors: z.array(z.string()).prefault([]).optional(),
+  extractImages: z.boolean().prefault(true).optional(),
+  extractLinks: z.boolean().prefault(true).optional(),
+  extractTables: z.boolean().prefault(true).optional(),
+  maxContentLength: z.number().positive().optional(),
+  format: ScrapeFormatSchema.prefault('markdown').optional(),
+  language: z.string().optional(),
+  proxy: z.string().optional(),
+  retries: z.number().nonnegative().prefault(3).optional(),
+  retryDelay: z.number().positive().prefault(1000).optional(),
+  userAgent: z.string().optional(),
+  headers: z.record(z.string(), z.string()).prefault({}).optional(),
+  cache: z.boolean().prefault(false).optional(),
+  cacheTtlMs: z.number().positive().prefault(300000).optional(),
+  readable: z.boolean().optional(),
+});
+export type ScrapeOptions = z.infer<typeof ScrapeOptionsSchema>;
+
+export interface ScrapeResult {
+  url: string;
+  title: string;
+  content: string;
+  format: ScrapeFormat;
+  text: string;
+  links: LinkInfo[];
+  images: ImageInfo[];
+  tables: TableInfo[];
+  metadata: PageMetadata;
+  extractedAt: string;
+  duration: number;
+  size: number;
+}
+
+export interface PageMetadata {
+  title: string;
+  description?: string;
+  favicon?: string;
+  charset?: string;
+  language?: string;
+  author?: string;
+  publishedDate?: string;
+  ogImage?: string;
+  ogType?: string;
+  keywords?: string[];
+}
+
+export interface LinkInfo {
+  href: string;
+  text: string;
+  isInternal: boolean;
+  isExternal: boolean;
+}
+
+export interface ImageInfo {
+  src: string;
+  alt: string;
+  width?: number;
+  height?: number;
+}
+
+export interface TableInfo {
+  headers: string[];
+  rows: string[][];
+  caption?: string;
+}
+
+export const BrowserAgentConfigSchema = z.strictObject({
+  headless: z.boolean().prefault(true).optional(),
+  userAgent: z.string().optional(),
+  viewport: z
+    .strictObject({
+      width: z.number().positive().prefault(1280),
+      height: z.number().positive().prefault(720),
+    })
+    .prefault({ width: 1280, height: 720 })
+    .optional(),
+  locale: z.string().prefault('en-US').optional(),
+  geolocation: z
+    .strictObject({
+      latitude: z.number(),
+      longitude: z.number(),
+    })
+    .optional(),
+  timeout: z.number().positive().prefault(30000).optional(),
+  recordVideo: z.boolean().prefault(false).optional(),
+  proxy: z.string().prefault('').optional(),
+  storageState: z.string().prefault('').optional(),
+  blockAds: z.boolean().prefault(true).optional(),
+  stealth: z.boolean().prefault(true).optional(),
+});
+export type BrowserAgentConfig = z.infer<typeof BrowserAgentConfigSchema>;
+
+export interface ExtractOptions {
+  readable?: boolean;
+  stripTags?: string[];
+  preserveLinks?: boolean;
+  preserveImages?: boolean;
+  preserveTables?: boolean;
+  maxLength?: number;
+  removeSelectors?: string[];
+  baseUrl?: string;
+}
+
+export interface SerializeOptions {
+  format: SerializeTarget;
+  pretty?: boolean;
+  maxChunkSize?: number;
+  includeMetadata?: boolean;
+  redisKey?: string;
+  xmlRoot?: string;
+}
+
+export interface SerializedOutput {
+  format: SerializeTarget;
+  content: string;
+  chunks?: string[];
+  size: number;
+  metadata?: Record<string, unknown>;
+}
+
+export interface AgentOutput {
+  summary: string;
+  content: string;
+  keyPoints: string[];
+  relevantUrls: string[];
+  structured?: Record<string, unknown>;
+  tokens: number;
+}
+
+export interface RenderOptions {
+  width?: number;
+  height?: number;
+  fullPage?: boolean;
+  type?: 'png' | 'jpeg';
+  quality?: number;
+  output?: string;
+}
+
+export interface ChromeCommand {
+  action: 'goto' | 'click' | 'type' | 'screenshot' | 'evaluate' | 'wait' | 'scroll' | 'extract';
+  args?: Record<string, unknown>;
+}
+
+export interface CliOptions {
+  url?: string;
+  file?: string;
+  format?: ScrapeFormat;
+  output?: string;
+  'no-headless'?: boolean;
+  timeout?: number;
+  scroll?: boolean;
+  readable?: boolean;
+  agent?: boolean;
+  pretty?: boolean;
+  proxy?: string;
+  retries?: number;
+  mode?: 'auto' | 'fetch' | 'browser';
+}
+
+export interface RetryConfig {
+  retries: number;
+  delay: number;
+  backoff: 'exponential' | 'linear' | 'constant';
+  maxDelay?: number;
+  statusCodes?: number[];
+}
+
+export interface ProxyConfig {
+  url: string;
+  username?: string;
+  password?: string;
+}
+
+export interface CacheConfig {
+  enabled: boolean;
+  ttlMs?: number;
+  maxEntries?: number;
+  storage?: 'memory' | 'redis' | 'sqlite' | 'indexeddb';
+}
+
+export interface CrawlStats {
+  totalUrls: number;
+  successful: number;
+  failed: number;
+  skipped: number;
+  duration: number;
+  avgResponseTime: number;
+}
+
+export interface BatchOptions {
+  urls: string[];
+  concurrency?: number;
+  mode?: 'auto' | 'fetch' | 'browser';
+  format?: ScrapeFormat;
+  onProgress?: (completed: number, total: number, currentUrl: string) => void;
+  onError?: (url: string, error: Error) => void;
+}

package/scrape/utils.ts

@@ -0,0 +1,77 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+
+export function ensureDir(path: string): void {
+  const dir = dirname(path);
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+}
+
+export function writeOutput(path: string, content: string): void {
+  ensureDir(path);
+  writeFileSync(path, content, 'utf-8');
+}
+
+export function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s-]/g, '')
+    .replace(/[\s_]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+export function truncate(text: string, max: number, suffix = '...'): string {
+  if (text.length <= max) return text;
+  return text.slice(0, max - suffix.length) + suffix;
+}
+
+export function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+export function chunkText(text: string, chunkSize: number): string[] {
+  const chunks: string[] = [];
+  let i = 0;
+  while (i < text.length) {
+    let end = i + chunkSize;
+    if (end < text.length) {
+      const breakIdx = text.lastIndexOf('\n', end);
+      if (breakIdx > i) end = breakIdx + 1;
+    }
+    chunks.push(text.slice(i, end).trim());
+    i = end;
+  }
+  return chunks;
+}
+
+export function delay(ms: number): Promise<void> {
+  return new Promise(r => setTimeout(r, ms));
+}
+
+export function isValidUrl(url: string): boolean {
+  try {
+    new URL(url);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function normalizeUrl(url: string, base?: string): string {
+  try {
+    return new URL(url, base).href;
+  } catch {
+    return url;
+  }
+}
+
+export function isInternalUrl(href: string, base: string): boolean {
+  try {
+    const baseUrl = new URL(base);
+    const target = new URL(href, base);
+    return target.hostname === baseUrl.hostname;
+  } catch {
+    return false;
+  }
+}

package/scrape/vite.config.ts

@@ -0,0 +1,55 @@
+import { defineConfig } from "vite";
+import dts from "vite-plugin-dts";
+import { resolve } from "node:path";
+import { randomPort } from './utils/port.js';
+
+const VITE_PORT = randomPort();
+
+export default defineConfig({
+	root: "web",
+	build: {
+		lib: {
+			entry: resolve(__dirname, "index.ts"),
+			name: "DevThinkWebScrape",
+			formats: ["es", "cjs"],
+			fileName: (format) => `index.${format === "es" ? "mjs" : "js"}`,
+		},
+		rollupOptions: {
+			external: [
+				"playwright",
+				"cheerio",
+				"jsdom",
+				"turndown",
+				"turndown-plugin-gfm",
+				"@mozilla/readability",
+				"zod",
+				"p-retry",
+				"p-limit",
+				"p-queue",
+				"robots-parser",
+				"sitemapper",
+				"lru-cache",
+				"keyv",
+				"hono",
+				"hono/cors",
+				"commander",
+				"emittery",
+				"normalize-url",
+				"node:fs",
+				"node:fs/promises",
+				"node:path",
+				"node:child_process",
+				"node:http",
+				"node:crypto",
+			],
+		},
+		sourcemap: true,
+		minify: false,
+	},
+	plugins: [dts()],
+	server: {
+		port: VITE_PORT,
+		host: "127.0.0.1",
+		cors: true,
+	},
+});

package/scrape/vitest.config.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+	test: {
+		environment: "jsdom",
+		include: ["tests/**/*.{test,spec}.{ts,mjs}", "tests/**/*.test.ts"],
+		globals: true,
+		testTimeout: 10000,
+		coverage: {
+			provider: "v8",
+			reporter: ["text", "json", "html"],
+			include: ["*.ts"],
+			exclude: ["tests/**", "scripts/**", "web/**", "ink-components/**", "node_modules/**", "dist/**", "coverage/**"],
+			thresholds: { lines: 60, functions: 50, branches: 50, statements: 60 },
+		},
+	},
+});

package/workflow/templates.js

@@ -5,14 +5,14 @@ import { workflowinputs } from "./manifest.js";
 
 export function githubworkflow(manifest) {
   const input = workflowinputs(manifest);
-  return `name: ${manifest.name}\non:\n  workflow_dispatch:\n    inputs:\n      jobid:\n        required: true\n        type: string\n      command:\n        required: true\n        type: string\n        default: ${manifest.command}\n  repository_dispatch:\n    types: [saddle-job]\npermissions:\n  contents: read\njobs:\n  process:\n    runs-on: ubuntu-latest\n    timeout-minutes: ${input.timeoutminutes}\n    steps:\n      - uses: actions/checkout@v4\n      - uses: actions/setup-node@v4\n        with:\n          node-version: 22\n      - run: npm ci\n      - run: \${{ github.event.inputs.command || '${manifest.command}' }}\n        env:\n          SBOT_JOB_ID: \${{ github.event.inputs.jobid || github.event.client_payload.jobid }}\n      - uses: actions/upload-artifact@v4\n        with:\n          name: saddle-results\n          path: ${manifest.artifacts.join("\n            ")}\n          if-no-files-found: ignore\n`;
+  return `name: ${manifest.name}\non:\n  workflow_dispatch:\n    inputs:\n      jobid:\n        required: true\n        type: string\n      command:\n        required: true\n        type: string\n        default: ${manifest.command}\n  repository_dispatch:\n    types: [saddle-job]\npermissions:\n  contents: read\njobs:\n  process:\n    runs-on: ubuntu-latest\n    timeout-minutes: ${input.timeoutminutes}\n    steps:\n      - uses: actions/checkout@v4\n      - uses: actions/setup-node@v4\n        with:\n          node-version: 26.7.0\n      - run: npm ci\n      - run: \${{ github.event.inputs.command || '${manifest.command}' }}\n        env:\n          SBOT_JOB_ID: \${{ github.event.inputs.jobid || github.event.client_payload.jobid }}\n      - uses: actions/upload-artifact@v4\n        with:\n          name: saddle-results\n          path: ${manifest.artifacts.join("\n            ")}\n          if-no-files-found: ignore\n`;
 }
 
 export function forgejoworkflow(manifest) { return genericworkflow(manifest, "forgejo"); }
 export function giteaworkflow(manifest) { return genericworkflow(manifest, "gitea"); }
-export function woodpeckerworkflow(manifest) { return `when:\n  - event: push\n  - event: manual\nsteps:\n  process:\n    image: node:22\n    commands:\n      - npm ci\n      - ${manifest.command}\n`;
+export function woodpeckerworkflow(manifest) { return `when:\n  - event: push\n  - event: manual\nsteps:\n  process:\n    image: node:26.7.0\n    commands:\n      - npm ci\n      - ${manifest.command}\n`;
 }
-export function gitlabworkflow(manifest) { return `stages:\n  - process\nprocess:\n  stage: process\n  image: node:22\n  script:\n    - npm ci\n    - ${manifest.command}\n  artifacts:\n    when: always\n    paths:\n${manifest.artifacts.map((item) => `      - ${item}`).join("\n")}\n`; }
+export function gitlabworkflow(manifest) { return `stages:\n  - process\nprocess:\n  stage: process\n  image: node:26.7.0\n  script:\n    - npm ci\n    - ${manifest.command}\n  artifacts:\n    when: always\n    paths:\n${manifest.artifacts.map((item) => `      - ${item}`).join("\n")}\n`; }
 
-function genericworkflow(manifest, name) { return `name: ${manifest.name}\non:\n  workflow_dispatch:\njobs:\n  process:\n    runs-on: ubuntu-latest\n    timeout-minutes: ${manifest.timeoutminutes}\n    steps:\n      - uses: actions/checkout@v4\n      - uses: actions/setup-node@v4\n        with:\n          node-version: 22\n      - run: npm ci\n      - run: ${manifest.command}\n`;
+function genericworkflow(manifest, name) { return `name: ${manifest.name}\non:\n  workflow_dispatch:\njobs:\n  process:\n    runs-on: ubuntu-latest\n    timeout-minutes: ${manifest.timeoutminutes}\n    steps:\n      - uses: actions/checkout@v4\n      - uses: actions/setup-node@v4\n        with:\n          node-version: 26.7.0\n      - run: npm ci\n      - run: ${manifest.command}\n`;
 }