@wenathlan/saddle 1.8.8 → 1.8.9

package/scrape/tokens.ts

@@ -1,45 +0,0 @@
-export type ModelType = 'gpt-4o' | 'gpt-4' | 'gpt-3.5-turbo' | 'claude' | 'gemini' | 'default';
-
-const MODEL_TOKEN_RATIOS: Record<ModelType, number> = {
-  'gpt-4o': 3.5,
-  'gpt-4': 3.5,
-  'gpt-3.5-turbo': 4,
-  'claude': 3.2,
-  'gemini': 3.5,
-  'default': 4,
-};
-
-export function estimateTokens(text: string, model: ModelType = 'default'): number {
-  const ratio = MODEL_TOKEN_RATIOS[model] || 4;
-  return Math.ceil(text.length / ratio);
-}
-
-export function countTokens(text: string, model: ModelType = 'default'): number {
-  return estimateTokens(text, model);
-}
-
-export function fitsInContext(text: string, maxTokens: number, model: ModelType = 'default'): boolean {
-  return estimateTokens(text, model) <= maxTokens;
-}
-
-export function truncateToTokens(text: string, maxTokens: number, model: ModelType = 'default'): string {
-  const estimatedTokens = estimateTokens(text, model);
-  if (estimatedTokens <= maxTokens) return text;
-  const charLimit = Math.floor(maxTokens * (MODEL_TOKEN_RATIOS[model] || 4));
-  return text.slice(0, charLimit) + '...';
-}
-
-export function tokenCost(text: string, model: ModelType, pricePer1kTokens: number): number {
-  const tokens = estimateTokens(text, model);
-  return (tokens / 1000) * pricePer1kTokens;
-}
-
-export const MODEL_PRICES: Record<string, { input: number; output: number }> = {
-  'gpt-4o': { input: 0.0025, output: 0.01 },
-  'gpt-4o-mini': { input: 0.00015, output: 0.0006 },
-  'gpt-4-turbo': { input: 0.01, output: 0.03 },
-  'claude-3-5-sonnet': { input: 0.003, output: 0.015 },
-  'claude-3-haiku': { input: 0.00025, output: 0.00125 },
-  'gemini-1.5-pro': { input: 0.00125, output: 0.005 },
-  'gemini-1.5-flash': { input: 0.000075, output: 0.0003 },
-};

package/scrape/tsconfig.json

@@ -1,28 +0,0 @@
-{
-	"compilerOptions": {
-		"target": "ES2024",
-		"module": "ESNext",
-		"lib": ["ES2024", "DOM", "DOM.Iterable"],
-		"moduleResolution": "bundler",
-		"strict": true,
-		"declaration": true,
-		"declarationMap": true,
-		"sourceMap": true,
-		"outDir": "./dist",
-		"rootDir": ".",
-		"noEmit": true,
-		"esModuleInterop": true,
-		"skipLibCheck": true,
-		"forceConsistentCasingInFileNames": true,
-		"resolveJsonModule": true,
-		"isolatedModules": true,
-		"verbatimModuleSyntax": true,
-		"noUnusedLocals": true,
-		"noUnusedParameters": true,
-		"noImplicitReturns": true,
-		"noFallthroughCasesInSwitch": true,
-		"types": ["node"]
-	},
-	"include": ["*.ts"],
-	"exclude": ["node_modules", "dist", "web", "vite.config.ts", "vitest.config.ts", "tests"]
-}

package/scrape/types.ts

@@ -1,214 +0,0 @@
-import { z } from 'zod';
-
-export const ScrapeFormatSchema = z.enum(['markdown', 'html', 'text', 'json', 'xml', 'redis']);
-export type ScrapeFormat = z.infer<typeof ScrapeFormatSchema>;
-
-export const SerializeTargetSchema = z.enum(['markdown', 'xml', 'json', 'redis', 'text']);
-export type SerializeTarget = z.infer<typeof SerializeTargetSchema>;
-
-export const ScrapeOptionsSchema = z.strictObject({
-  url: z.string().url().optional(),
-  html: z.string().optional(),
-  mode: z.enum(['auto', 'fetch', 'browser']).prefault('auto').optional(),
-  waitUntil: z.enum(['load', 'domcontentloaded', 'networkidle', 'commit']).prefault('networkidle').optional(),
-  timeout: z.number().positive().prefault(30000).optional(),
-  scroll: z.boolean().prefault(false).optional(),
-  scrollDelay: z.number().nonnegative().prefault(300).optional(),
-  maxScrolls: z.number().positive().prefault(50).optional(),
-  removeSelectors: z.array(z.string()).prefault([]).optional(),
-  extractImages: z.boolean().prefault(true).optional(),
-  extractLinks: z.boolean().prefault(true).optional(),
-  extractTables: z.boolean().prefault(true).optional(),
-  maxContentLength: z.number().positive().optional(),
-  format: ScrapeFormatSchema.prefault('markdown').optional(),
-  language: z.string().optional(),
-  proxy: z.string().optional(),
-  retries: z.number().nonnegative().prefault(3).optional(),
-  retryDelay: z.number().positive().prefault(1000).optional(),
-  userAgent: z.string().optional(),
-  headers: z.record(z.string(), z.string()).prefault({}).optional(),
-  cache: z.boolean().prefault(false).optional(),
-  cacheTtlMs: z.number().positive().prefault(300000).optional(),
-  readable: z.boolean().optional(),
-});
-export type ScrapeOptions = z.infer<typeof ScrapeOptionsSchema>;
-
-export interface ScrapeResult {
-  url: string;
-  title: string;
-  content: string;
-  format: ScrapeFormat;
-  text: string;
-  links: LinkInfo[];
-  images: ImageInfo[];
-  tables: TableInfo[];
-  metadata: PageMetadata;
-  extractedAt: string;
-  duration: number;
-  size: number;
-}
-
-export interface PageMetadata {
-  title: string;
-  description?: string;
-  favicon?: string;
-  charset?: string;
-  language?: string;
-  author?: string;
-  publishedDate?: string;
-  ogImage?: string;
-  ogType?: string;
-  keywords?: string[];
-}
-
-export interface LinkInfo {
-  href: string;
-  text: string;
-  isInternal: boolean;
-  isExternal: boolean;
-}
-
-export interface ImageInfo {
-  src: string;
-  alt: string;
-  width?: number;
-  height?: number;
-}
-
-export interface TableInfo {
-  headers: string[];
-  rows: string[][];
-  caption?: string;
-}
-
-export const BrowserAgentConfigSchema = z.strictObject({
-  headless: z.boolean().prefault(true).optional(),
-  userAgent: z.string().optional(),
-  viewport: z
-    .strictObject({
-      width: z.number().positive().prefault(1280),
-      height: z.number().positive().prefault(720),
-    })
-    .prefault({ width: 1280, height: 720 })
-    .optional(),
-  locale: z.string().prefault('en-US').optional(),
-  geolocation: z
-    .strictObject({
-      latitude: z.number(),
-      longitude: z.number(),
-    })
-    .optional(),
-  timeout: z.number().positive().prefault(30000).optional(),
-  recordVideo: z.boolean().prefault(false).optional(),
-  proxy: z.string().prefault('').optional(),
-  storageState: z.string().prefault('').optional(),
-  blockAds: z.boolean().prefault(true).optional(),
-  stealth: z.boolean().prefault(true).optional(),
-});
-export type BrowserAgentConfig = z.infer<typeof BrowserAgentConfigSchema>;
-
-export interface ExtractOptions {
-  readable?: boolean;
-  stripTags?: string[];
-  preserveLinks?: boolean;
-  preserveImages?: boolean;
-  preserveTables?: boolean;
-  maxLength?: number;
-  removeSelectors?: string[];
-  baseUrl?: string;
-}
-
-export interface SerializeOptions {
-  format: SerializeTarget;
-  pretty?: boolean;
-  maxChunkSize?: number;
-  includeMetadata?: boolean;
-  redisKey?: string;
-  xmlRoot?: string;
-}
-
-export interface SerializedOutput {
-  format: SerializeTarget;
-  content: string;
-  chunks?: string[];
-  size: number;
-  metadata?: Record<string, unknown>;
-}
-
-export interface AgentOutput {
-  summary: string;
-  content: string;
-  keyPoints: string[];
-  relevantUrls: string[];
-  structured?: Record<string, unknown>;
-  tokens: number;
-}
-
-export interface RenderOptions {
-  width?: number;
-  height?: number;
-  fullPage?: boolean;
-  type?: 'png' | 'jpeg';
-  quality?: number;
-  output?: string;
-}
-
-export interface ChromeCommand {
-  action: 'goto' | 'click' | 'type' | 'screenshot' | 'evaluate' | 'wait' | 'scroll' | 'extract';
-  args?: Record<string, unknown>;
-}
-
-export interface CliOptions {
-  url?: string;
-  file?: string;
-  format?: ScrapeFormat;
-  output?: string;
-  'no-headless'?: boolean;
-  timeout?: number;
-  scroll?: boolean;
-  readable?: boolean;
-  agent?: boolean;
-  pretty?: boolean;
-  proxy?: string;
-  retries?: number;
-  mode?: 'auto' | 'fetch' | 'browser';
-}
-
-export interface RetryConfig {
-  retries: number;
-  delay: number;
-  backoff: 'exponential' | 'linear' | 'constant';
-  maxDelay?: number;
-  statusCodes?: number[];
-}
-
-export interface ProxyConfig {
-  url: string;
-  username?: string;
-  password?: string;
-}
-
-export interface CacheConfig {
-  enabled: boolean;
-  ttlMs?: number;
-  maxEntries?: number;
-  storage?: 'memory' | 'redis' | 'sqlite' | 'indexeddb';
-}
-
-export interface CrawlStats {
-  totalUrls: number;
-  successful: number;
-  failed: number;
-  skipped: number;
-  duration: number;
-  avgResponseTime: number;
-}
-
-export interface BatchOptions {
-  urls: string[];
-  concurrency?: number;
-  mode?: 'auto' | 'fetch' | 'browser';
-  format?: ScrapeFormat;
-  onProgress?: (completed: number, total: number, currentUrl: string) => void;
-  onError?: (url: string, error: Error) => void;
-}

package/scrape/utils.ts

@@ -1,77 +0,0 @@
-import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
-import { dirname } from 'node:path';
-
-export function ensureDir(path: string): void {
-  const dir = dirname(path);
-  if (!existsSync(dir)) {
-    mkdirSync(dir, { recursive: true });
-  }
-}
-
-export function writeOutput(path: string, content: string): void {
-  ensureDir(path);
-  writeFileSync(path, content, 'utf-8');
-}
-
-export function slugify(text: string): string {
-  return text
-    .toLowerCase()
-    .replace(/[^\w\s-]/g, '')
-    .replace(/[\s_]+/g, '-')
-    .replace(/^-+|-+$/g, '');
-}
-
-export function truncate(text: string, max: number, suffix = '...'): string {
-  if (text.length <= max) return text;
-  return text.slice(0, max - suffix.length) + suffix;
-}
-
-export function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
-export function chunkText(text: string, chunkSize: number): string[] {
-  const chunks: string[] = [];
-  let i = 0;
-  while (i < text.length) {
-    let end = i + chunkSize;
-    if (end < text.length) {
-      const breakIdx = text.lastIndexOf('\n', end);
-      if (breakIdx > i) end = breakIdx + 1;
-    }
-    chunks.push(text.slice(i, end).trim());
-    i = end;
-  }
-  return chunks;
-}
-
-export function delay(ms: number): Promise<void> {
-  return new Promise(r => setTimeout(r, ms));
-}
-
-export function isValidUrl(url: string): boolean {
-  try {
-    new URL(url);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-export function normalizeUrl(url: string, base?: string): string {
-  try {
-    return new URL(url, base).href;
-  } catch {
-    return url;
-  }
-}
-
-export function isInternalUrl(href: string, base: string): boolean {
-  try {
-    const baseUrl = new URL(base);
-    const target = new URL(href, base);
-    return target.hostname === baseUrl.hostname;
-  } catch {
-    return false;
-  }
-}

package/scrape/vite.config.ts

@@ -1,55 +0,0 @@
-import { defineConfig } from "vite";
-import dts from "vite-plugin-dts";
-import { resolve } from "node:path";
-import { randomPort } from './utils/port.js';
-
-const VITE_PORT = randomPort();
-
-export default defineConfig({
-	root: "web",
-	build: {
-		lib: {
-			entry: resolve(__dirname, "index.ts"),
-			name: "DevThinkWebScrape",
-			formats: ["es", "cjs"],
-			fileName: (format) => `index.${format === "es" ? "mjs" : "js"}`,
-		},
-		rollupOptions: {
-			external: [
-				"playwright",
-				"cheerio",
-				"jsdom",
-				"turndown",
-				"turndown-plugin-gfm",
-				"@mozilla/readability",
-				"zod",
-				"p-retry",
-				"p-limit",
-				"p-queue",
-				"robots-parser",
-				"sitemapper",
-				"lru-cache",
-				"keyv",
-				"hono",
-				"hono/cors",
-				"commander",
-				"emittery",
-				"normalize-url",
-				"node:fs",
-				"node:fs/promises",
-				"node:path",
-				"node:child_process",
-				"node:http",
-				"node:crypto",
-			],
-		},
-		sourcemap: true,
-		minify: false,
-	},
-	plugins: [dts()],
-	server: {
-		port: VITE_PORT,
-		host: "127.0.0.1",
-		cors: true,
-	},
-});

package/scrape/vitest.config.ts

@@ -1,17 +0,0 @@
-import { defineConfig } from "vitest/config";
-
-export default defineConfig({
-	test: {
-		environment: "jsdom",
-		include: ["tests/**/*.{test,spec}.{ts,mjs}", "tests/**/*.test.ts"],
-		globals: true,
-		testTimeout: 10000,
-		coverage: {
-			provider: "v8",
-			reporter: ["text", "json", "html"],
-			include: ["*.ts"],
-			exclude: ["tests/**", "scripts/**", "web/**", "ink-components/**", "node_modules/**", "dist/**", "coverage/**"],
-			thresholds: { lines: 60, functions: 50, branches: 50, statements: 60 },
-		},
-	},
-});

package/server/node.js

@@ -1,34 +0,0 @@
-/**
- * node server is an optional adapter around the universal service contract.
- */
-import { createServer } from "node:http";
-
-/** Creates a Node HTTP adapter with explicit host, port, and request handler. */
-export function nodeserver(options = {}) {
-  if (!options.host || !Number.isInteger(options.port) || options.port < 1) throw new TypeError("node server requires host and port");
-  if (typeof options.handle !== "function") throw new TypeError("node server requires handle");
-  /* Request translation stays inside the Node adapter. */
-  const server = createServer(async (request, response) => {
-    try {
-      const chunks = [];
-      for await (const chunk of request) chunks.push(chunk);
-      const body = Buffer.concat(chunks).toString("utf8");
-      const headers = new Headers(request.headers);
-      const webrequest = new Request(new URL(request.url ?? "/", `http://${options.host}:${options.port}`), { method: request.method, headers, body: body || undefined });
-      const result = await options.handle(webrequest);
-      response.statusCode = result.status;
-      result.headers.forEach((value, key) => response.setHeader(key, value));
-      response.end(Buffer.from(await result.arrayBuffer()));
-    } catch (error) {
-      response.statusCode = 500;
-      response.setHeader("content-type", "application/json");
-      response.end(JSON.stringify({ error: error.message }));
-    }
-  });
-  /* Lifecycle methods keep the server optional for library consumers. */
-  return {
-    server,
-    listen() { return new Promise((resolve, reject) => { server.once("error", reject); server.listen(options.port, options.host, () => resolve({ host: options.host, port: options.port })); }); },
-    close() { return new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); }
-  };
-}

package/sessions/file.js

@@ -1,13 +0,0 @@
-/**
- * file session persistence writes one validated JSON document per session.
- */
-import { mkdir, readFile, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
-import { validatesession } from "../domain/sessions.js";
-
-export function filesessions(root) {
-  return {
-    async save(session) { const valid = validatesession(session); const path = join(root, `${valid.id}.json`); await mkdir(dirname(path), { recursive: true }); await writeFile(path, JSON.stringify(valid, null, 2)); return valid; },
-    async load(id) { const value = JSON.parse(await readFile(join(root, `${id}.json`), "utf8")); return validatesession(value); }
-  };
-}

package/sessions/replay.js

@@ -1,61 +0,0 @@
-/**
- * replay maps validated session events to an injected browser adapter.
- */
-export async function replay(session, adapter, options = {}) {
-  if (!session?.events || typeof adapter?.move !== "function") throw new TypeError("replay requires session events and browser adapter");
-  const speed = options.speed ?? 1;
-  let previous = 0;
-  let currentcontext = normalizecontext(options.initialcontext);
-  let contextswitches = 0;
-  for (const event of session.events) {
-    const wait = Math.max(0, (event.t - previous) / speed);
-    if (wait) await delay(wait);
-    previous = event.t;
-    const eventcontext = normalizecontext(event.context ?? event);
-    if (eventcontext && contextchanged(currentcontext, eventcontext)) {
-      await restorecontext(adapter, eventcontext, currentcontext);
-      currentcontext = eventcontext;
-      contextswitches += 1;
-    }
-    if (event.type === "move") await adapter.move(event);
-    else if (event.type === "click") await adapter.click(event);
-    else if (event.type === "drag") await adapter.drag(event);
-    else if (event.type === "scroll") await adapter.scroll(event);
-    else if (event.type === "key") await adapter.key(event);
-  }
-  return { events: session.events.length, duration: previous / speed, contextswitches };
-}
-
-function normalizecontext(value) {
-  if (value === undefined || value === null) return undefined;
-  if (typeof value !== "object" || Array.isArray(value)) throw new TypeError("replay context must be an object");
-  const context = {};
-  for (const name of ["windowid", "tabid", "frameid"]) if (value[name] !== undefined) {
-    if (typeof value[name] !== "string" || !value[name]) throw new TypeError(`replay ${name} must be a non-empty string`);
-    context[name] = value[name];
-  }
-  return Object.keys(context).length ? context : undefined;
-}
-
-function contextchanged(previous, next) {
-  return ["windowid", "tabid", "frameid"].some((name) => previous?.[name] !== next[name]);
-}
-
-async function restorecontext(adapter, next, previous) {
-  if (typeof adapter.restorecontext === "function") {
-    await adapter.restorecontext({ ...next }, previous ? { ...previous } : undefined);
-    return;
-  }
-  const methods = [["windowid", "selectwindow"], ["tabid", "selecttab"], ["frameid", "selectframe"]];
-  for (const [name, method] of methods) if (next[name] !== undefined && previous?.[name] !== next[name]) {
-    if (typeof adapter[method] !== "function") {
-      const error = new Error(`replay adapter cannot restore ${name}`);
-      error.code = "REPLAY_CONTEXT_UNSUPPORTED";
-      error.context = name;
-      throw error;
-    }
-    await adapter[method](next[name], { ...next });
-  }
-}
-
-function delay(milliseconds) { return new Promise((resolve) => setTimeout(resolve, milliseconds)); }

package/sessions/store.js

@@ -1,13 +0,0 @@
-/**
- * jsonl session storage keeps append only traces independent from browser adapters.
- */
-import { appendFile, mkdir, readFile } from "node:fs/promises";
-import { join } from "node:path";
-import { validatesession } from "../domain/sessions.js";
-
-export function sessionstore(root) {
-  return {
-    async append(session) { const valid = validatesession(session); await mkdir(root, { recursive: true }); await appendFile(join(root, `${valid.id}.jsonl`), `${JSON.stringify(valid)}\n`); return valid; },
-    async read(id) { const text = await readFile(join(root, `${id}.jsonl`), "utf8"); return text.trim().split("\n").filter(Boolean).map((line) => validatesession(JSON.parse(line))); }
-  };
-}

package/storage/adapter.js

@@ -1,8 +0,0 @@
-/**
- * storage adapters keep remote services out of the engine core.
- */
-export function storageadapter(methods) {
-  const required = ["put", "get", "head", "delete", "list"];
-  for (const name of required) if (typeof methods?.[name] !== "function") throw new TypeError(`storage adapter requires ${name}`);
-  return methods;
-}

package/storage/cache.js

@@ -1,54 +0,0 @@
-/**
- * tiered cache keeps a bounded hot tier and an optional persistent cold tier with stale-while-revalidate.
- */
-
-/** Creates a cache with caller supplied encode, decode, clock and persistent storage policies. */
-export function tieredcache(options = {}) {
-  const hot = new Map();
-  const maxentries = Number(options.maxentries ?? 256);
-  const ttl = Number(options.ttl ?? 300000);
-  const stale = Number(options.stale ?? ttl);
-  const now = options.now ?? (() => Date.now());
-  const cold = options.storage;
-  const encode = options.encode ?? ((value) => new TextEncoder().encode(JSON.stringify(value)));
-  const decode = options.decode ?? ((bytes) => JSON.parse(new TextDecoder().decode(bytes)));
-  const stats = { hits: 0, misses: 0, stalehits: 0, evictions: 0, revalidations: 0 };
-
-  function readhot(key) {
-    const item = hot.get(key);
-    if (!item) return null;
-    const current = now();
-    if (current <= item.expires) { stats.hits += 1; return { value: item.value, fresh: true }; }
-    if (current <= item.stale) { stats.stalehits += 1; return { value: item.value, fresh: false }; }
-    hot.delete(key);
-    return null;
-  }
-
-  function writehot(key, value, options = {}) {
-    if (hot.size >= maxentries && !hot.has(key)) { hot.delete(hot.keys().next().value); stats.evictions += 1; }
-    const current = now();
-    hot.set(key, { value, expires: current + Number(options.ttl ?? ttl), stale: current + Number(options.stale ?? stale) });
-    return value;
-  }
-
-  async function get(key, options = {}) {
-    const hotvalue = readhot(key);
-    if (hotvalue?.fresh || (hotvalue && options.allowstale !== false)) return hotvalue.value;
-    if (!cold) { stats.misses += 1; return null; }
-    try { const value = decode(await cold.get(key)); writehot(key, value, options); return value; } catch { stats.misses += 1; return null; }
-  }
-
-  async function set(key, value, valueoptions = {}) { writehot(key, value, valueoptions); if (cold) await cold.put({ key, data: encode(value), contenttype: "application/json" }); return value; }
-  async function getorload(key, loader, options = {}) {
-    if (typeof loader !== "function") throw new TypeError("cache loader must be a function");
-    const current = readhot(key);
-    if (current?.fresh) return current.value;
-    if (current && options.allowstale !== false) { stats.revalidations += 1; Promise.resolve(loader()).then((value) => set(key, value, options)).catch(() => undefined); return current.value; }
-    const value = await loader();
-    return set(key, value, options);
-  }
-  async function remove(key) { hot.delete(key); await cold?.delete?.(key); }
-  function clear() { hot.clear(); }
-  function inspect() { return { ...stats, hotentries: hot.size, maxentries, ttl, stale }; }
-  return { get, set, getorload, delete: remove, clear, inspect };
-}

package/storage/checksum.js

@@ -1,17 +0,0 @@
-/**
- * byte collection and hashing are grouped because every storage adapter uses them.
- */
-import { sha256 } from "../core/hash.js";
-
-export { sha256 } from "../core/hash.js";
-
-export async function collectbytes(input) {
-  if (input instanceof Uint8Array) return input;
-  const chunks = [];
-  let size = 0;
-  for await (const chunk of input) { chunks.push(chunk); size += chunk.byteLength; }
-  const output = new Uint8Array(size);
-  let offset = 0;
-  for (const chunk of chunks) { output.set(chunk, offset); offset += chunk.byteLength; }
-  return output;
-}