@wenathlan/saddle 1.8.6 → 1.8.8

package/runtime/retry.js

@@ -0,0 +1,27 @@
+/**
+ * retry context groups transient retry policy and circuit protection for runners,
+ * storage adapters and network-facing surfaces.
+ */
+
+/** Creates bounded exponential retry behavior for retryable failures. */
+export function retrypolicy(options = {}) {
+  const maxattempts = options.maxattempts ?? 3;
+  const base = options.base ?? 1000;
+  const factor = options.factor ?? 2;
+  const cap = options.cap ?? 30000;
+  return { async run(handler) { let last; for (let attempt = 1; attempt <= maxattempts; attempt += 1) { try { return await handler(attempt); } catch (error) { last = error; if (error?.retryable !== true || attempt === maxattempts) throw error; const wait = Math.min(cap, base * factor ** (attempt - 1)) + Math.floor(Math.random() * (options.jitter ?? 0)); options.onretry?.({ attempt, wait, error }); await delay(wait); } } throw last; } };
+}
+
+/** Creates a circuit breaker that opens after repeated handler failures. */
+export function circuitbreaker(options = {}) {
+  const threshold = options.failurethreshold ?? 5;
+  const resettimeout = options.resettimeout ?? 60000;
+  let failures = 0;
+  let openedat = 0;
+  let state = "closed";
+  async function execute(handler) { if (state === "open") { if (Date.now() - openedat < resettimeout) throw new Error("circuit breaker is open"); state = "halfopen"; } try { const result = await handler(); failures = 0; state = "closed"; return result; } catch (error) { failures += 1; if (failures >= threshold) { state = "open"; openedat = Date.now(); } throw error; } }
+  return { execute, status() { return { state, failures, openedat }; }, reset() { failures = 0; openedat = 0; state = "closed"; } };
+}
+
+/** Waits between retry attempts without introducing an external timer dependency. */
+function delay(milliseconds) { return milliseconds ? new Promise((resolve) => setTimeout(resolve, milliseconds)) : Promise.resolve(); }

package/scrape/crawl.js

@@ -0,0 +1,104 @@
+/**
+ * scrape crawl context owns URL normalization traversal frontier and durable crawl state.
+ *
+ * The context keeps single page acquisition injectable while grouping the correlated
+ * crawl responsibilities that previously lived in four separate top level files.
+ */
+
+/** Removes fragments and tracking parameters before crawl deduplication. */
+export function normalizeurl(value) {
+  const url = new URL(value);
+  url.hash = "";
+  for (const key of [...url.searchParams.keys()]) if (/^(utm_|fbclid$|msclkid$|gclid$|gclsrc$|dclid$|gbraid$|wbraid$|twclid$|campaign$|content$|term$|source$|medium$|ref$|share_id$)/i.test(key)) url.searchParams.delete(key);
+  if (url.pathname.length > 1) url.pathname = url.pathname.replace(/\/+$/, "");
+  return url.href;
+}
+
+/** Compares two crawl targets by origin. */
+export function sameorigin(left, right) { return new URL(left).origin === new URL(right).origin; }
+
+/** Creates a bounded priority frontier for crawler and queue adapters. */
+export function crawlfrontier(options = {}) {
+  const maxpages = Number(options.maxpages ?? 20);
+  const maxperdomain = Number(options.maxperdomain ?? maxpages);
+  const queue = [];
+  const seen = new Set();
+  const completed = new Set();
+  const domains = new Map();
+
+  /** Adds a URL once while respecting the global page budget. */
+  function add(input = {}) {
+    const url = String(input.url ?? "");
+    if (!url || seen.has(url) || seen.size >= maxpages) return false;
+    seen.add(url);
+    queue.push({ url, depth: Number(input.depth ?? 0), priority: Number(input.priority ?? 0), discoveredat: Number(input.discoveredat ?? Date.now()) });
+    queue.sort((left, right) => right.priority - left.priority || left.discoveredat - right.discoveredat);
+    return true;
+  }
+
+  /** Removes the next URL that still fits its per-domain budget. */
+  function next() {
+    while (queue.length) {
+      const item = queue.shift();
+      const domain = new URL(item.url).hostname;
+      if ((domains.get(domain) ?? 0) >= maxperdomain) continue;
+      domains.set(domain, (domains.get(domain) ?? 0) + 1);
+      return { ...item };
+    }
+    return null;
+  }
+
+  /** Records a completed URL for diagnostics and persistence-friendly state. */
+  function complete(url) { completed.add(String(url)); }
+
+  /** Returns stable frontier diagnostics without exposing mutable collections. */
+  function state() { return { maxpages, maxperdomain, queued: queue.length, discovered: seen.size, completed: completed.size, domains: Object.fromEntries(domains) }; }
+
+  return { add, next, complete, state, list() { return queue.map((item) => ({ ...item })); } };
+}
+
+/** Creates a durable crawl queue around a caller-owned store. */
+export function persistentqueue(options = {}) {
+  const store = options.store;
+  const values = [];
+  const seen = new Set();
+
+  /** Restores unfinished crawl records from the injected store. */
+  async function restore() { if (typeof store?.list !== "function") return; for (const item of await store.list()) if (!seen.has(item.url) && item.status !== "done") { seen.add(item.url); values.push(item); } }
+
+  /** Adds a crawl record and persists it when the store supports writes. */
+  async function add(item) { if (!item?.url || seen.has(item.url)) return false; const value = { url: item.url, depth: item.depth ?? 0, status: "queued", createdat: Date.now(), metadata: item.metadata ?? {} }; seen.add(value.url); values.push(value); if (typeof store?.save === "function") await store.save(value); return true; }
+
+  /** Claims the next queued crawl record. */
+  async function next() { const item = values.find((value) => value.status === "queued"); if (!item) return null; item.status = "running"; if (typeof store?.update === "function") await store.update(item.url, item); return item; }
+
+  /** Completes a crawl record and persists the resulting status. */
+  async function complete(url, patch = {}) { const item = values.find((value) => value.url === url); if (!item) return null; Object.assign(item, patch, { status: patch.status ?? "done", processedat: Date.now() }); if (typeof store?.update === "function") await store.update(url, item); return item; }
+
+  return { restore, add, next, complete, list() { return values.map((value) => ({ ...value })); } };
+}
+
+/** Runs bounded breadth first traversal through the injected single page scrape contract. */
+export async function crawl(start, options = {}) {
+  const maxdepth = options.maxdepth ?? 1;
+  const maxpages = options.maxpages ?? 20;
+  const sameDomain = options.samedomain ?? true;
+  const frontier = crawlfrontier({ maxpages, maxperdomain: options.maxperdomain ?? maxpages });
+  frontier.add({ url: normalizeurl(start), depth: 0, priority: options.startpriority ?? 0 });
+  const results = [];
+  while (frontier.state().queued && results.length < maxpages) {
+    const current = frontier.next();
+    if (!current || current.depth > maxdepth) continue;
+    const result = await options.scrape(current.url);
+    results.push({ ...result, depth: current.depth });
+    frontier.complete(current.url);
+    if (current.depth >= maxdepth) continue;
+    for (const link of result.links ?? []) {
+      let url;
+      try { url = normalizeurl(link); } catch { continue; }
+      if (sameDomain && !sameorigin(start, url)) continue;
+      frontier.add({ url, depth: current.depth + 1, priority: Number(options.priority?.(url, result) ?? 0) });
+    }
+  }
+  return { results, stats: { ...frontier.state(), completed: results.length, maxdepth, maxpages } };
+}

package/crawl/crawler.js

@@ -1,29 +0,0 @@
-/**
- * crawler performs bounded breadth first traversal through the scraper contract.
- */
-import { normalizeurl, sameorigin } from "./normalize.js";
-import { crawlfrontier } from "./frontier.js";
-
-export async function crawl(start, options = {}) {
-  const maxdepth = options.maxdepth ?? 1;
-  const maxpages = options.maxpages ?? 20;
-  const sameDomain = options.samedomain ?? true;
-  const frontier = crawlfrontier({ maxpages, maxperdomain: options.maxperdomain ?? maxpages });
-  frontier.add({ url: normalizeurl(start), depth: 0, priority: options.startpriority ?? 0 });
-  const results = [];
-  while (frontier.state().queued && results.length < maxpages) {
-    const current = frontier.next();
-    if (!current || current.depth > maxdepth) continue;
-    const result = await options.scrape(current.url);
-    results.push({ ...result, depth: current.depth });
-    frontier.complete(current.url);
-    if (current.depth >= maxdepth) continue;
-    for (const link of result.links ?? []) {
-      let url;
-      try { url = normalizeurl(link); } catch { continue; }
-      if (sameDomain && !sameorigin(start, url)) continue;
-      frontier.add({ url, depth: current.depth + 1, priority: Number(options.priority?.(url, result) ?? 0) });
-    }
-  }
-  return { results, stats: { ...frontier.state(), completed: results.length, maxdepth, maxpages } };
-}

package/crawl/frontier.js

@@ -1,34 +0,0 @@
-/**
- * crawl frontier provides priorities, per-domain budgets and persistent-friendly queue state.
- */
-
-/** Creates a bounded priority frontier for crawler and queue adapters. */
-export function crawlfrontier(options = {}) {
-  const maxpages = Number(options.maxpages ?? 20);
-  const maxperdomain = Number(options.maxperdomain ?? maxpages);
-  const queue = [];
-  const seen = new Set();
-  const completed = new Set();
-  const domains = new Map();
-  function add(input = {}) {
-    const url = String(input.url ?? "");
-    if (!url || seen.has(url) || seen.size >= maxpages) return false;
-    seen.add(url);
-    queue.push({ url, depth: Number(input.depth ?? 0), priority: Number(input.priority ?? 0), discoveredat: Number(input.discoveredat ?? Date.now()) });
-    queue.sort((left, right) => right.priority - left.priority || left.discoveredat - right.discoveredat);
-    return true;
-  }
-  function next() {
-    while (queue.length) {
-      const item = queue.shift();
-      const domain = new URL(item.url).hostname;
-      if ((domains.get(domain) ?? 0) >= maxperdomain) continue;
-      domains.set(domain, (domains.get(domain) ?? 0) + 1);
-      return { ...item };
-    }
-    return null;
-  }
-  function complete(url) { completed.add(String(url)); }
-  function state() { return { maxpages, maxperdomain, queued: queue.length, discovered: seen.size, completed: completed.size, domains: Object.fromEntries(domains) }; }
-  return { add, next, complete, state, list() { return queue.map((item) => ({ ...item })); } };
-}

package/crawl/normalize.js

@@ -1,14 +0,0 @@
-/**
- * url normalization removes tracking noise before frontier deduplication.
- */
-const tracking = /^(utm_|fbclid$|msclkid$|gclid$|gclsrc$|dclid$|gbraid$|wbraid$|twclid$|campaign$|content$|term$|source$|medium$|ref$|share_id$)/i;
-
-export function normalizeurl(value) {
-  const url = new URL(value);
-  url.hash = "";
-  for (const key of [...url.searchParams.keys()]) if (tracking.test(key)) url.searchParams.delete(key);
-  if (url.pathname.length > 1) url.pathname = url.pathname.replace(/\/+$/, "");
-  return url.href;
-}
-
-export function sameorigin(left, right) { return new URL(left).origin === new URL(right).origin; }

package/crawl/persistent.js

@@ -1,13 +0,0 @@
-/**
- * persistent crawl queue uses an injected store and falls back to memory when no store exists.
- */
-export function persistentqueue(options = {}) {
-  const store = options.store;
-  const values = [];
-  const seen = new Set();
-  async function restore() { if (typeof store?.list !== "function") return; for (const item of await store.list()) if (!seen.has(item.url) && item.status !== "done") { seen.add(item.url); values.push(item); } }
-  async function add(item) { if (!item?.url || seen.has(item.url)) return false; const value = { url: item.url, depth: item.depth ?? 0, status: "queued", createdat: Date.now(), metadata: item.metadata ?? {} }; seen.add(value.url); values.push(value); if (typeof store?.save === "function") await store.save(value); return true; }
-  async function next() { const item = values.find((value) => value.status === "queued"); if (!item) return null; item.status = "running"; if (typeof store?.update === "function") await store.update(item.url, item); return item; }
-  async function complete(url, patch = {}) { const item = values.find((value) => value.url === url); if (!item) return null; Object.assign(item, patch, { status: patch.status ?? "done", processedat: Date.now() }); if (typeof store?.update === "function") await store.update(url, item); return item; }
-  return { restore, add, next, complete, list() { return values.map((value) => ({ ...value })); } };
-}

package/errors/taxonomy.js

@@ -1,18 +0,0 @@
-/**
- * scrape errors carry a stable code status retry flag severity and recovery hint.
- */
-export const errorcatalog = Object.freeze({
-  timeout: { code: "E1001", statuscode: 504, retryable: true, recovery: "WAIT_AND_RETRY" },
-  connectionrefused: { code: "E1002", statuscode: 503, retryable: true, recovery: "WAIT_AND_RETRY" },
-  dns: { code: "E1003", statuscode: 503, retryable: true, recovery: "ROTATE_PROXY" },
-  ratelimited: { code: "E2001", statuscode: 429, retryable: true, recovery: "WAIT_AND_RETRY" },
-  forbidden: { code: "E2002", statuscode: 403, retryable: false, recovery: "REVIEW_ROBOTS_TXT" },
-  notfound: { code: "E2003", statuscode: 404, retryable: false, recovery: "STOP_CRAWLING" },
-  parse: { code: "E4002", statuscode: 422, retryable: false, recovery: "STOP_CRAWLING" },
-  captcha: { code: "E4003", statuscode: 403, retryable: false, recovery: "REVIEW_ROBOTS_TXT" },
-  session: { code: "E5001", statuscode: 401, retryable: true, recovery: "ROTATE_USER_AGENT" },
-  config: { code: "E6001", statuscode: 400, retryable: false, recovery: "STOP_CRAWLING" }
-});
-
-export function webscrapeerror(kind, message, options = {}) { const preset = errorcatalog[kind] ?? errorcatalog.config; const error = new Error(message, { cause: options.cause }); error.name = "webscrapeerror"; error.code = options.code ?? preset.code; error.statuscode = options.statuscode ?? preset.statuscode; error.retryable = options.retryable ?? preset.retryable; error.recovery = options.recovery ?? preset.recovery; error.severity = options.severity ?? (error.statuscode >= 500 ? "high" : "medium"); error.details = options.details ?? {}; return error; }
-export function classifyerror(error) { if (error?.name === "webscrapeerror") return error; const message = String(error?.message ?? error); if (/timeout|aborted/i.test(message)) return webscrapeerror("timeout", message, { cause: error }); if (/dns|enotfound/i.test(message)) return webscrapeerror("dns", message, { cause: error }); return webscrapeerror("config", message, { cause: error }); }

package/readme.txt

@@ -1,163 +0,0 @@
-                    SADDLE - README
-                       Version 1.0, August 2026
-
- Copyright (C) August 2026 devthink, nathlan, iakadion, nathu filho, allan neris, andraneris
- Everyone is permitted to view this document, but changing it
- is not allowed. This document is part of Project saddle.
-
-                            Preamble
-
-  Project saddle unifies both README sources.
-
-  Saddle is a JavaScript ESM engine for jobs that move data between storage,
-  a working set, an injected runner and durable artifacts. It includes
-  contracts for scraping, crawling, browser agents, queues, persistence,
-  MCP transport, webhooks and package delivery.
-
-  Core thesis: storage bytes and compute-memory bytes are the same bytes.
-  A Node.js framework runs on other people's runners, loading storage
-  buckets as virtual RAM/GPU via storage->RAM bridge.
-
-  This README is the single source of truth combining Foundation, Engine,
-  and Productization sections from both original READMEs.
-
-                       TERMS AND CONDITIONS
-
-  0. Overview.
-
-  See full readme.md for complete documentation, API, execution model,
-  CLI, security boundaries, package surfaces, development, and repository
-  map.
-
-  1. What is Included.
-
-  Jobs, Storage, Working set, Scraping, Crawl, Browser, Operations,
-  Protocols, Delivery, Agent Browser, Compute Backends, Storage Backends.
-
-  2. License.
-
-  Proprietary - View Only. See license.txt.
-
-                     END OF TERMS AND CONDITIONS
-
-
-# Saddle
-
-<p align="center">
-  <img src="docs/assets/saddlemark.svg" alt="Saddle" width="720" />
-</p>
-
-<p align="center">
-  <strong>Storage-backed jobs, scraping contracts and portable runners for Node.js.</strong><br/>
-  <strong>Binary computing agent, agent browser, computer-use, scraper and packager.</strong><br/>
-  <a href="https://github.com/wenathlan/saddle/actions/workflows/ci.yml"><img src="https://github.com/wenathlan/saddle/actions/workflows/ci.yml/badge.svg" alt="CI" /></a>
-  <a href="https://github.com/wenathlan/saddle/releases/tag/v1.8.2"><img src="https://img.shields.io/badge/release-v1.8.2-d35d3d" alt="Release 1.8.2" /></a>
-  <a href="https://github.com/wenathlan/saddle/blob/main/license.md"><img src="https://img.shields.io/badge/license-Proprietary--View--Only-202a2f" alt="Proprietary View Only" /></a>
-</p>
-
-> **Core idea:** storage is the durable side of the working set; the runner is replaceable; the artifact is the boundary. **Storage == Compute** — RAM and disk are the same construct, differing only by usage flag.
-
-Saddle is a **JavaScript ESM engine** for jobs that move data between storage, a working set, an injected runner and durable artifacts. It is also a **virtual machine you publish as a package** that runs on other people's computers (GitHub Actions, Forgejo, Gitea, GitLab, Codeberg, free Docker containers) and turns unlimited third-party storage buckets into virtual RAM/GPU/CPU. Nothing runs on the operator's local machine.
-
-Ships as a library, CLI, binary, n8n node, CRX extension, Android/iOS and Tauri desktop app. The canonical JavaScript package is `@wenathlan/saddle`; GitHub Packages npm, Maven and GHCR use the transferred `wenathlan` owner namespace, while NuGet and RubyGems retain their unscoped ecosystem package names.
-
-## Start here
-
-Saddle requires **Node.js 22 or newer**.
-
-```bash
-npm install @wenathlan/saddle
-```
-
-```js
-import { scrapeurl, formatforagent } from "@wenathlan/saddle";
-
-const result = await scrapeurl("https://example.com", { format: "markdown" });
-const context = formatforagent(result, { maxchunksize: 2000, keypoints: 4 });
-
-console.log(context.summary);
-```
-
-Deterministic example with no network:
-
-```bash
-node examples/publicapi.js
-```
-
-## What is included
-
-| Area | Contract | Result |
-| --- | --- | --- |
-| Jobs | `engine`, `scheduler`, `inprocess` | `prepare → process → sync → cleanup` |
-| Storage | local, chunked, S3-compatible, GitHub Contents, file hosting | durable objects and chunks |
-| Working set | memory bridge, modes, objects, transforms | storage-to-compute and compute-to-storage |
-| Scraping | robots, cache, extraction, schema, scraper | text, metadata, links and structured output |
-| Crawl | normalization, BFS crawler, persistent frontier | bounded domain-aware crawling |
-| Browser | fingerprint, session, replay and injected agent | browser actions without vendor lock-in |
-| Operations | queues, idempotency, saga, retry, circuit breaker | controlled execution and recovery |
-| Protocols | JSON, NDJSON, SSE, blocks and MCP | transport-neutral messages |
-| Delivery | manifests, workflow registry, binary/container plans | package and runner surfaces |
-| Agent Browser | capture & replay, stealth, fingerprint | Brave capture, movement replay, session recording |
-| Compute Backends | github-actions, huggingface, gitlab-ci, kaggle, oracle-cloud | free runners chain |
-| Storage Backends | HF, Kaggle, Terabox, R2, Telegram, Discord via rclone | unlimited disk as RAM |
-
-## Public API
-
-| Export | Purpose |
-| --- | --- |
-| `saddleurl` | choose fetch or injected browser path |
-| `scrapeurl` | fetch one URL and extract |
-| `scrapehtml` | extract from HTML without network |
-| `extractcontent` | structured extraction |
-| `serializeresult` | serialize as JSON, Markdown, XML |
-| `formatforagent` | summary, chunks, token count |
-| `batchscrape` | bounded URL groups |
-| `crawlurl` | crawl contract |
-| `browseragent` | navigation, click, type, screenshot |
-| `mcpserver` / `mcptransport` | MCP tools over JSONL/HTTP |
-| `nodeserver` | Web Request/Response handler |
-
-Complete API: `docs/libraryapi.md`
-
-## The execution model
-
-Saddle coordinates contracts instead of hiding providers. A repo + CI runner is a virtual processor:
-
-- Repo = Disk (persistent state)
-- CI = CPU (workflow_dispatch = function call)
-- Pages = Bus + CDN
-- Static site = BIOS
-- repository_dispatch = IPC
-
-```js
-import { engine, eventbus, inprocess, localmemory, localstorage, scheduler } from "@wenathlan/saddle";
-const events = eventbus();
-const run = engine({
-  storage: localstorage("./.saddle-data"),
-  memory: localmemory(),
-  scheduler: scheduler([inprocess()]),
-  events
-});
-const result = await run.run(
-  { name: "example", input: { value: 42 } },
-  ({ job }) => ({ jobid: job.id, ok: true })
-);
-```
-
-## CLI
-
-```bash
-saddle help
-saddle modes
-saddle runexample
-saddle mcp
-saddle capture --url <url>
-saddle bot --platform github --token $SBOT_TOKEN
-saddle memory --load repo://owner/repo/path/file.json
-saddle deploy --target netlify
-```
-
-## Security boundaries
-
-| Boundary | Policy |
-| --- | --- |

package/retry/circuit.js

@@ -1,15 +0,0 @@
-/**
- * circuit breaker protects providers from repeated failure storms.
- */
-export function circuitbreaker(options = {}) {
-  const threshold = options.failurethreshold ?? 5;
-  const resettimeout = options.resettimeout ?? 60000;
-  let failures = 0;
-  let openedat = 0;
-  let state = "closed";
-  async function execute(handler) {
-    if (state === "open") { if (Date.now() - openedat < resettimeout) throw new Error("circuit breaker is open"); state = "halfopen"; }
-    try { const result = await handler(); failures = 0; state = "closed"; return result; } catch (error) { failures += 1; if (failures >= threshold) { state = "open"; openedat = Date.now(); } throw error; }
-  }
-  return { execute, status() { return { state, failures, openedat }; }, reset() { failures = 0; openedat = 0; state = "closed"; } };
-}

package/retry/policy.js

@@ -1,12 +0,0 @@
-/**
- * retry policy handles transient errors and keeps non retryable failures terminal.
- */
-export function retrypolicy(options = {}) {
-  const maxattempts = options.maxattempts ?? 3;
-  const base = options.base ?? 1000;
-  const factor = options.factor ?? 2;
-  const cap = options.cap ?? 30000;
-  return { async run(handler) { let last; for (let attempt = 1; attempt <= maxattempts; attempt += 1) { try { return await handler(attempt); } catch (error) { last = error; if (error?.retryable !== true || attempt === maxattempts) throw error; const wait = Math.min(cap, base * factor ** (attempt - 1)) + Math.floor(Math.random() * (options.jitter ?? 0)); options.onretry?.({ attempt, wait, error }); await delay(wait); } } throw last; } };
-}
-
-function delay(milliseconds) { return milliseconds ? new Promise((resolve) => setTimeout(resolve, milliseconds)) : Promise.resolve(); }