@wenathlan/saddle 1.8.1

package/domain/sessions.js

@@ -0,0 +1,34 @@
+/**
+ * session logs are versioned and validated before any replay adapter sees them.
+ */
+import { validationerror } from "../core/errors.js";
+
+const eventtypes = new Set(["move", "click", "drag", "scroll", "key"]);
+
+export function validatesession(value) {
+  if (!value || typeof value !== "object") throw validationerror("session must be an object");
+  if (value.version !== 1 || typeof value.id !== "string" || typeof value.agentname !== "string" || typeof value.originurl !== "string" || typeof value.seed !== "string") throw validationerror("session header is invalid");
+  if (!Array.isArray(value.events)) throw validationerror("session events must be an array");
+  if (!["created", "recording", "closed"].includes(value.status)) throw validationerror("session status is invalid");
+  if (!Number.isFinite(value.startedat) || value.startedat < 0) throw validationerror("session startedat is invalid");
+  return {
+    version: 1,
+    id: value.id,
+    agentname: value.agentname,
+    originurl: value.originurl,
+    seed: value.seed,
+    status: value.status,
+    startedat: value.startedat,
+    finishedat: Number.isFinite(value.finishedat) ? value.finishedat : undefined,
+    events: value.events.map((event, index) => validateevent(event, index))
+  };
+}
+
+function validateevent(value, index) {
+  if (!value || typeof value !== "object" || !Number.isFinite(value.t) || value.t < 0 || !eventtypes.has(value.type)) throw validationerror(`session event ${index} is invalid`);
+  for (const name of ["x", "y", "tx", "ty", "dx", "dy"]) if (value[name] !== undefined && !Number.isFinite(value[name])) throw validationerror(`session event ${index} ${name} is invalid`);
+  if (value.key !== undefined && typeof value.key !== "string") throw validationerror(`session event ${index} key is invalid`);
+  if (value.target !== undefined && typeof value.target !== "string") throw validationerror(`session event ${index} target is invalid`);
+  if (value.button !== undefined && !["left", "right"].includes(value.button)) throw validationerror(`session event ${index} button is invalid`);
+  return { ...value };
+}

package/errors/taxonomy.js

@@ -0,0 +1,18 @@
+/**
+ * scrape errors carry a stable code status retry flag severity and recovery hint.
+ */
+export const errorcatalog = Object.freeze({
+  timeout: { code: "E1001", statuscode: 504, retryable: true, recovery: "WAIT_AND_RETRY" },
+  connectionrefused: { code: "E1002", statuscode: 503, retryable: true, recovery: "WAIT_AND_RETRY" },
+  dns: { code: "E1003", statuscode: 503, retryable: true, recovery: "ROTATE_PROXY" },
+  ratelimited: { code: "E2001", statuscode: 429, retryable: true, recovery: "WAIT_AND_RETRY" },
+  forbidden: { code: "E2002", statuscode: 403, retryable: false, recovery: "REVIEW_ROBOTS_TXT" },
+  notfound: { code: "E2003", statuscode: 404, retryable: false, recovery: "STOP_CRAWLING" },
+  parse: { code: "E4002", statuscode: 422, retryable: false, recovery: "STOP_CRAWLING" },
+  captcha: { code: "E4003", statuscode: 403, retryable: false, recovery: "REVIEW_ROBOTS_TXT" },
+  session: { code: "E5001", statuscode: 401, retryable: true, recovery: "ROTATE_USER_AGENT" },
+  config: { code: "E6001", statuscode: 400, retryable: false, recovery: "STOP_CRAWLING" }
+});
+
+export function webscrapeerror(kind, message, options = {}) { const preset = errorcatalog[kind] ?? errorcatalog.config; const error = new Error(message, { cause: options.cause }); error.name = "webscrapeerror"; error.code = options.code ?? preset.code; error.statuscode = options.statuscode ?? preset.statuscode; error.retryable = options.retryable ?? preset.retryable; error.recovery = options.recovery ?? preset.recovery; error.severity = options.severity ?? (error.statuscode >= 500 ? "high" : "medium"); error.details = options.details ?? {}; return error; }
+export function classifyerror(error) { if (error?.name === "webscrapeerror") return error; const message = String(error?.message ?? error); if (/timeout|aborted/i.test(message)) return webscrapeerror("timeout", message, { cause: error }); if (/dns|enotfound/i.test(message)) return webscrapeerror("dns", message, { cause: error }); return webscrapeerror("config", message, { cause: error }); }

package/examples/localjob.js

@@ -0,0 +1,15 @@
+/**
+ * the local example shows that the library can run without a remote service.
+ */
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { eventbus, engine, inprocess, scheduler } from "../index.js";
+import { localmemory } from "../memory/bridge.js";
+import { localstorage } from "../storage/local.js";
+
+const root = await mkdtemp(join(tmpdir(), "saddleexample"));
+const events = eventbus();
+const run = engine({ storage: localstorage(root), memory: localmemory(), scheduler: scheduler([inprocess()]), events });
+const result = await run.run({ name: "localexample", input: { source: "example" }, outputkey: "results/example.json" }, ({ job }) => ({ jobid: job.id, ok: true }));
+console.log({ jobid: result.job.id, artifact: result.artifact.key, eventcount: events.all().length });

package/examples/publicapi.js

@@ -0,0 +1,7 @@
+/**
+ * public api example with an injected fetcher; no network is required to run it.
+ */
+import { scrapeurl, formatforagent } from "../library/public.js";
+
+const result = await scrapeurl("https://example.com", { fetcher: async () => ({ ok: true, status: 200, text: async () => "<title>Example</title><p>Injected content.</p>" }) });
+console.log(JSON.stringify(formatforagent(result), null, 2));

package/extension/README.md

@@ -0,0 +1,23 @@
+# saddle browser bridge
+
+This directory contains the first Manifest V3 surface for Saddle. It is a user initiated bridge, not a hidden automation service.
+
+## load unpacked
+
+1. Open `chrome://extensions`.
+2. Enable developer mode.
+3. Choose **Load unpacked**.
+4. Select this `extension/` directory.
+5. Open a page, invoke Saddle from the extension action and choose `Snapshot` or `Read page`.
+
+The manifest requests only `activeTab`, `scripting` and `storage`. It does not request broad host permissions, cookies, tabs, webRequest or debugger access. `activeTab` gives temporary access after the user invokes the action for the current tab.
+
+## boundaries
+
+The content bridge runs in Chrome's isolated world. It exposes bounded page metadata, visible text, stable references and user initiated click or fill commands. The service worker forwards versioned messages and stores only the latest snapshot metadata in session storage. No endpoint, credential, remote script or browser profile is embedded.
+
+`protocol.js` and `serviceworker.js` are reusable ESM contracts. `content.js` is intentionally a classic injected file because programmatic Chrome content scripts are loaded as files; it exposes a small global bridge and avoids arbitrary page JavaScript evaluation.
+
+## next slices
+
+The next extension slices should add snapshot diffing, tab and frame identity, resumable command records, optional host permission escalation, browser action results and a deterministic zip workflow. Browser providers, login profiles, captcha solvers and remote runners remain caller owned adapters.

package/extension/content.js

@@ -0,0 +1,85 @@
+/**
+ * content bridge runs in the isolated world and exposes bounded page facts and user initiated actions.
+ * It stays classic JavaScript because Chrome injects programmatic content scripts as files.
+ */
+
+(function installglobalbridge(global) {
+  const installedkey = "__saddlecontentbridge";
+  const protocolversion = 1;
+  const commands = ["snapshot", "readpage", "clickref", "fillref"];
+
+  function createbridge(documentref, now = () => Date.now()) {
+    let snapshotid = null;
+    let sequence = 0;
+    const references = new Map();
+
+    function snapshotpage() {
+      const nextid = `snap${++sequence}${now()}`;
+      const elements = [];
+      references.clear();
+      const candidates = documentref.querySelectorAll?.("a,button,input,textarea,select,[role]") ?? [];
+      for (const element of Array.from(candidates).slice(0, 100)) {
+        if (!visible(element)) continue;
+        const ref = `e${elements.length + 1}`;
+        references.set(ref, { element, snapshotid: nextid });
+        elements.push({ ref, role: roleof(element), name: nameof(element) });
+      }
+      snapshotid = nextid;
+      return { version: protocolversion, snapshotid, createdat: now(), url: String(documentref.location?.href ?? ""), title: String(documentref.title ?? ""), text: String(documentref.body?.innerText ?? "").slice(0, 100000), elements };
+    }
+
+    function readpage() {
+      const page = snapshotpage();
+      return { version: page.version, snapshotid: page.snapshotid, url: page.url, title: page.title, text: page.text };
+    }
+
+    function resolve(ref, requestedid) {
+      if (requestedid !== snapshotid) throw failure("stale_snapshot", "page snapshot is stale");
+      const entry = references.get(String(ref));
+      if (!entry || entry.snapshotid !== snapshotid) throw failure("unknown_reference", `unknown page reference: ${ref}`);
+      return entry.element;
+    }
+
+    function handle(request) {
+      if (request?.version !== protocolversion || request.type !== "command" || !commands.includes(request.command)) throw failure("invalid_message", "invalid content command");
+      const payload = request.payload ?? {};
+      if (request.command === "snapshot") return snapshotpage();
+      if (request.command === "readpage") return readpage();
+      const element = resolve(payload.ref, payload.snapshotid);
+      if (request.command === "clickref") { element.click?.(); return { ref: payload.ref, clicked: true, snapshotid }; }
+      if (!isfillable(element)) throw failure("not_fillable", `element is not fillable: ${payload.ref}`);
+      setvalue(element, payload.value);
+      return { ref: payload.ref, filled: true, snapshotid };
+    }
+
+    return { handle, snapshotpage, readpage };
+  }
+
+  function install(runtime = global.chrome?.runtime, documentref = global.document) {
+    if (!runtime?.onMessage || !documentref) throw new TypeError("content bridge requires runtime and document");
+    if (global[installedkey]) return global[installedkey];
+    const bridge = createbridge(documentref);
+    const listener = (message, sender, sendresponse) => {
+      Promise.resolve().then(() => bridge.handle(message)).then((payload) => sendresponse({ version: protocolversion, type: "response", id: `resp${Date.now()}`, requestid: message?.id, payload })).catch((error) => sendresponse({ version: protocolversion, type: "error", id: `err${Date.now()}`, requestid: message?.id, error: { code: error.code ?? "content_error", message: error.message } }));
+      return true;
+    };
+    runtime.onMessage.addListener(listener);
+    global[installedkey] = { bridge, listener, dispose() { runtime.onMessage.removeListener?.(listener); delete global[installedkey]; } };
+    return global[installedkey];
+  }
+
+  function visible(element) {
+    const style = global.getComputedStyle?.(element);
+    if (style && (style.display === "none" || style.visibility === "hidden")) return false;
+    return typeof element.getClientRects !== "function" || element.getClientRects().length > 0;
+  }
+
+  function roleof(element) { return String(element.getAttribute?.("role") || element.tagName || "generic").toLowerCase(); }
+  function nameof(element) { return String(element.getAttribute?.("aria-label") || element.innerText || element.textContent || element.getAttribute?.("placeholder") || "").trim().replace(/\s+/g, " ").slice(0, 200); }
+  function isfillable(element) { return ["INPUT", "TEXTAREA"].includes(String(element.tagName).toUpperCase()); }
+  function setvalue(element, value) { element.value = String(value ?? ""); element.dispatchEvent?.(new Event("input", { bubbles: true })); element.dispatchEvent?.(new Event("change", { bubbles: true })); }
+  function failure(code, message) { const error = new Error(message); error.code = code; return error; }
+
+  global.saddlecontent = { createbridge, install };
+  if (global.chrome?.runtime?.onMessage && global.document) install();
+})(globalThis);

package/extension/index.js

@@ -0,0 +1,5 @@
+/**
+ * extension public contracts expose browser neutral message and worker routing primitives.
+ */
+export * from "./protocol.js";
+export * from "./serviceworker.js";

package/extension/manifest.json

@@ -0,0 +1,10 @@
+{
+  "manifest_version": 3,
+  "name": "Saddle browser bridge",
+  "version": "1.1.0",
+  "description": "User initiated page snapshots through the Saddle browser contract.",
+  "minimum_chrome_version": "110",
+  "permissions": ["activeTab", "scripting", "storage"],
+  "background": { "service_worker": "worker.js", "type": "module" },
+  "action": { "default_title": "Saddle", "default_popup": "popup.html" }
+}

package/extension/popup.css

@@ -0,0 +1,13 @@
+/* popup keeps the extension surface compact, readable and free of remote assets. */
+:root { color-scheme: light; font-family: system-ui, sans-serif; background: #f7f1e8; color: #202a2f; }
+body { width: 320px; margin: 0; }
+main { padding: 16px; }
+header { display: flex; align-items: center; gap: 10px; }
+.mark { display: grid; place-items: center; width: 28px; height: 28px; border-radius: 9px; background: #d35d3d; color: #fff; font-weight: 800; }
+header strong, header small { display: block; }
+header small { color: #6d7777; font-size: 11px; letter-spacing: .08em; text-transform: uppercase; }
+#status { color: #596663; font-size: 12px; line-height: 1.4; }
+.actions { display: flex; gap: 8px; }
+button { border: 0; border-radius: 8px; padding: 8px 10px; background: #202a2f; color: #fff; cursor: pointer; font: inherit; font-size: 12px; }
+button:hover { background: #d35d3d; }
+pre { max-height: 260px; overflow: auto; margin: 14px 0 0; padding: 10px; border-radius: 8px; background: #fff; color: #394446; font: 11px/1.45 ui-monospace, monospace; white-space: pre-wrap; }

package/extension/popup.html

@@ -0,0 +1,24 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Saddle</title>
+    <link rel="stylesheet" href="popup.css" />
+  </head>
+  <body>
+    <main>
+      <header>
+        <span class="mark">S</span>
+        <div><strong>Saddle</strong><small>page bridge</small></div>
+      </header>
+      <p id="status" role="status">Choose a read action for the active tab.</p>
+      <div class="actions">
+        <button data-command="snapshot">Snapshot</button>
+        <button data-command="readpage">Read page</button>
+      </div>
+      <pre id="output">No request yet.</pre>
+    </main>
+    <script type="module" src="popup.js"></script>
+  </body>
+</html>

package/extension/popup.js

@@ -0,0 +1,25 @@
+/**
+ * popup sends user initiated read commands through the service worker.
+ */
+
+import { createcommand } from "./protocol.js";
+
+const status = document.querySelector("#status");
+const output = document.querySelector("#output");
+
+for (const button of document.querySelectorAll("[data-command]")) button.addEventListener("click", () => request(button.dataset.command));
+
+async function request(command) {
+  try {
+    status.textContent = "Reading active tab…";
+    const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
+    if (!tab?.id) throw new Error("no active tab");
+    const response = await chrome.runtime.sendMessage(createcommand(command, { tabid: tab.id }));
+    if (response?.type === "error") throw new Error(response.error?.message ?? "extension request failed");
+    output.textContent = JSON.stringify(response?.payload ?? response, null, 2);
+    status.textContent = "Complete.";
+  } catch (error) {
+    status.textContent = "Request failed.";
+    output.textContent = String(error.message ?? error);
+  }
+}

package/extension/protocol.js

@@ -0,0 +1,76 @@
+/**
+ * extension protocol defines serializable commands, responses, errors and page snapshots.
+ */
+
+export const protocolversion = 1;
+export const extensioncommands = Object.freeze(["snapshot", "readpage", "clickref", "fillref"]);
+
+/** Creates a compact identifier without embedding a host, port or credential. */
+export function createid(prefix = "msg", source) {
+  const generator = source ?? (() => globalThis.crypto?.randomUUID?.() ?? `${Date.now()}${Math.random().toString(16).slice(2)}`);
+  const value = generator().replaceAll("-", "");
+  return `${prefix}${value}`;
+}
+
+/** Creates a versioned command for the extension message bus. */
+export function createcommand(command, payload = {}, options = {}) {
+  if (!extensioncommands.includes(command)) throw new TypeError(`unsupported extension command: ${command}`);
+  return createmessage("command", { ...options, command, payload });
+}
+
+/** Creates a serializable message envelope. */
+export function createmessage(type, options = {}) {
+  if (!["command", "response", "error", "event"].includes(type)) throw new TypeError(`unsupported extension message type: ${type}`);
+  const message = { version: protocolversion, type, id: options.id ?? createid(type) };
+  if (options.requestid) message.requestid = options.requestid;
+  if (options.command) message.command = options.command;
+  if (options.payload !== undefined) message.payload = options.payload;
+  if (options.error) message.error = options.error;
+  assertserializable(message);
+  return message;
+}
+
+/** Creates a correlated successful response. */
+export function createresponse(request, payload = {}) {
+  assertmessage(request);
+  return createmessage("response", { requestid: request.id, payload });
+}
+
+/** Creates a correlated error response with stable error fields. */
+export function createerror(request, error, options = {}) {
+  const requestid = request?.id ?? options.requestid;
+  const failure = { code: options.code ?? error?.code ?? "extension_error", message: String(error?.message ?? error ?? "extension request failed"), retryable: Boolean(options.retryable ?? error?.retryable) };
+  return createmessage("error", { requestid, error: failure });
+}
+
+/** Validates a message before a privileged context handles it. */
+export function assertmessage(message) {
+  if (!message || typeof message !== "object") throw new TypeError("extension message must be an object");
+  if (message.version !== protocolversion) throw new TypeError(`unsupported extension protocol version: ${message.version}`);
+  if (typeof message.type !== "string" || typeof message.id !== "string") throw new TypeError("extension message requires type and id");
+  if (message.type === "command" && !extensioncommands.includes(message.command)) throw new TypeError(`unsupported extension command: ${message.command}`);
+  if (message.payload !== undefined && (!message.payload || typeof message.payload !== "object" || Array.isArray(message.payload))) throw new TypeError("extension payload must be an object");
+  return message;
+}
+
+/** Creates a structured page snapshot with stable element references. */
+export function createsnapshot(input = {}) {
+  const snapshot = {
+    version: protocolversion,
+    snapshotid: String(input.snapshotid ?? createid("snap")),
+    createdat: Number(input.createdat ?? Date.now()),
+    url: String(input.url ?? ""),
+    title: String(input.title ?? ""),
+    text: String(input.text ?? ""),
+    elements: Array.isArray(input.elements) ? input.elements.map((element) => ({ ref: String(element.ref), role: String(element.role ?? "generic"), name: String(element.name ?? "") })) : []
+  };
+  assertserializable(snapshot);
+  return snapshot;
+}
+
+/** Returns whether a reference still belongs to the current page snapshot. */
+export function isfreshsnapshot(snapshotid, currentid) { return Boolean(snapshotid && currentid && snapshotid === currentid); }
+
+function assertserializable(value) {
+  try { JSON.stringify(value); } catch (error) { throw new TypeError(`extension message is not serializable: ${error.message}`); }
+}

package/extension/serviceworker.js

@@ -0,0 +1,43 @@
+/**
+ * service worker router forwards user initiated commands to the active tab and persists resumable state.
+ */
+
+import { assertmessage } from "./protocol.js";
+
+/** Creates a browser independent router around Chrome tabs, scripting and storage APIs. */
+export function createworkerrouter(options = {}) {
+  const tabs = options.tabs;
+  const scripting = options.scripting;
+  const storage = options.storage;
+  const contentfile = options.contentfile ?? "content.js";
+  const statekey = options.statekey ?? "saddleextensionstate";
+  if (typeof tabs?.sendMessage !== "function") throw new TypeError("extension router requires tabs.sendMessage");
+
+  async function ensurecontent(tabid) {
+    if (!Number.isInteger(tabid)) throw new TypeError("extension command requires a tab id");
+    if (typeof scripting?.executeScript !== "function") throw new TypeError("extension router requires scripting.executeScript");
+    await scripting.executeScript({ target: { tabId: tabid }, files: [contentfile] });
+  }
+
+  async function readstate() {
+    if (typeof storage?.get !== "function") return {};
+    const result = await storage.get(statekey);
+    return result?.[statekey] ?? {};
+  }
+
+  async function savestate(value) {
+    if (typeof storage?.set === "function") await storage.set({ [statekey]: value });
+  }
+
+  async function handle(message, sender = {}) {
+    const request = assertmessage(message);
+    if (request.type !== "command") throw new TypeError("extension router accepts commands only");
+    const tabid = request.payload?.tabid ?? sender.tab?.id;
+    await ensurecontent(tabid);
+    const response = await tabs.sendMessage(tabid, request);
+    if (response?.type === "response" && response.payload?.snapshotid) await savestate({ tabid, snapshotid: response.payload.snapshotid, updatedat: Date.now() });
+    return response;
+  }
+
+  return { ensurecontent, readstate, savestate, handle };
+}

package/extension/worker.js

@@ -0,0 +1,20 @@
+/**
+ * worker binds the generic extension router to the Manifest V3 runtime APIs.
+ */
+
+import { createerror } from "./protocol.js";
+import { createworkerrouter } from "./serviceworker.js";
+
+/** Installs the service worker listeners against a caller supplied Chrome API object. */
+export function startworker(chromeapi = globalThis.chrome) {
+  if (!chromeapi?.runtime?.onMessage || !chromeapi?.tabs || !chromeapi?.scripting) throw new TypeError("extension worker requires runtime tabs and scripting APIs");
+  const router = createworkerrouter({ tabs: chromeapi.tabs, scripting: chromeapi.scripting, storage: chromeapi.storage?.session });
+  const listener = (message, sender, sendresponse) => {
+    router.handle(message, sender).then(sendresponse).catch((error) => sendresponse(createerror(message, error)));
+    return true;
+  };
+  chromeapi.runtime.onMessage.addListener(listener);
+  return { router, dispose() { chromeapi.runtime.onMessage.removeListener?.(listener); } };
+}
+
+if (globalThis.chrome?.runtime?.onMessage) startworker();

package/format/check.js

@@ -0,0 +1,21 @@
+/**
+ * format check validates the public root based JavaScript layout before release.
+ */
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const roots = ["core", "domain", "memory", "runners", "runtime", "storage", "sessions", "modes", "adapters", "persistence", "queue", "dispatch", "scrape", "crawl", "api", "mcp", "browser", "proxy", "captcha", "ai", "webhook", "surfaces", "library", "errors", "retry", "server", "binary", "deploy", "workflow", "packager", "bot", "protocol", "cli"];
+
+/** Finds JavaScript files that do not follow the skill formatting contract. */
+export async function formatissues(root = process.cwd()) {
+  const issues = [];
+  for (const directory of roots) for (const file of await javascriptfiles(join(root, directory))) { const relative = file.slice(root.length + 1); const source = await readFile(file, "utf8"); if (/[A-Z_-]/.test(relative)) issues.push(`${relative}: invalid path format`); if (!source.includes("/**")) issues.push(`${relative}: missing jsdoc`); }
+  return issues;
+}
+
+/** Runs the check as a CLI and throws a short diagnostic on failure. */
+export async function runformatcheck() { const issues = await formatissues(); if (issues.length) throw new Error(issues.join("\n")); return { ok: true, checked: roots.length }; }
+
+async function javascriptfiles(directory) { let entries; try { entries = await readdir(directory, { withFileTypes: true }); } catch { return []; } const files = []; for (const entry of entries) { const file = join(directory, entry.name); if (entry.isDirectory()) files.push(...await javascriptfiles(file)); else if (entry.name.endsWith(".js")) files.push(file); } return files; }
+
+if (import.meta.url === `file://${process.argv[1]}`) runformatcheck().then((result) => console.log(JSON.stringify(result))).catch((error) => { console.error(error.message); process.exitCode = 1; });

package/index.js

@@ -0,0 +1,120 @@
+/**
+ * public entry point exports stable factories and leaves adapters replaceable.
+ */
+export * from "./core/errors.js";
+export * from "./core/events.js";
+export * from "./core/ids.js";
+export * from "./core/hash.js";
+export * from "./domain/artifacts.js";
+export * from "./domain/jobs.js";
+export * from "./domain/providers.js";
+export * from "./domain/runtime.js";
+export * from "./domain/sessions.js";
+export * from "./memory/modes.js";
+export * from "./memory/objects.js";
+export * from "./memory/targets.js";
+export * from "./memory/transforms.js";
+export * from "./memory/engine.js";
+export * from "./runners/inprocess.js";
+export * from "./runners/scheduler.js";
+export * from "./runtime/engine.js";
+export * from "./storage/adapter.js";
+export * from "./storage/cache.js";
+export * from "./storage/content.js";
+export * from "./storage/memory.js";
+export * from "./storage/sync.js";
+export * from "./runners/health.js";
+export * from "./runners/heartbeat.js";
+export * from "./workflow/triggers.js";
+export * from "./dispatch/resumable.js";
+export * from "./scrape/semantic.js";
+export * from "./crawl/frontier.js";
+export * from "./ai/provenance.js";
+export * from "./observability/metrics.js";
+export * from "./api/auth.js";
+export * from "./api/contracts.js";
+export * from "./mcp/browser.js";
+export * from "./apps/registry.js";
+export * from "./bot/permissions.js";
+export * from "./webhook/delivery.js";
+export * from "./storage/chunked.js";
+export * from "./storage/s3compatible.js";
+export * from "./modes/modes.js";
+export * from "./adapters/transport.js";
+export * from "./adapters/github.js";
+export * from "./adapters/socket.js";
+export * from "./adapters/forge.js";
+export * from "./adapters/gitlab.js";
+export * from "./adapters/forgejo.js";
+export * from "./adapters/huggingface.js";
+export * from "./persistence/schema.js";
+export * from "./persistence/adapter.js";
+export * from "./persistence/memory.js";
+export * from "./persistence/sql.js";
+export * from "./persistence/drizzle.js";
+export * from "./persistence/prisma.js";
+export * from "./queue/idempotency.js";
+export * from "./queue/queue.js";
+export * from "./queue/saga.js";
+export * from "./dispatch/workflow.js";
+export * from "./sessions/replay.js";
+export * from "./scrape/robots.js";
+export * from "./scrape/cache.js";
+export * from "./scrape/extract.js";
+export * from "./scrape/scraper.js";
+export * from "./packager/manifest.js";
+export * from "./bot/commands.js";
+export * from "./bot/adapter.js";
+export * from "./bot/bot.js";
+export * from "./protocol/json.js";
+export * from "./protocol/ndjson.js";
+export * from "./protocol/sse.js";
+export * from "./protocol/blocks.js";
+export * from "./workflow/manifest.js";
+export * from "./workflow/templates.js";
+export * from "./workflow/registry.js";
+export * from "./crawl/normalize.js";
+export * from "./crawl/crawler.js";
+export * from "./crawl/persistent.js";
+export * from "./scrape/schema.js";
+export * from "./mcp/server.js";
+export * from "./runtime/detect.js";
+export * from "./runtime/abort.js";
+export * from "./runtime/compatibility.js";
+export * from "./runtime/worker.js";
+export * from "./packager/publish.js";
+export * from "./browser/fingerprint.js";
+export * from "./browser/session.js";
+export * from "./browser/index.js";
+export * from "./proxy/pool.js";
+export * from "./captcha/contract.js";
+export * from "./captcha/guard.js";
+export * from "./ai/tokens.js";
+export * from "./ai/chunk.js";
+export * from "./ai/rag.js";
+export * from "./ai/llmstxt.js";
+export * from "./webhook/signature.js";
+export * from "./webhook/receiver.js";
+export * from "./surfaces/manifest.js";
+export * from "./surfaces/n8n.js";
+export * from "./surfaces/adapters.js";
+export * from "./surfaces/controls.js";
+export * from "./surfaces/operations.js";
+export * from "./library/public.js";
+export * from "./errors/taxonomy.js";
+export * from "./retry/policy.js";
+export * from "./retry/circuit.js";
+export * from "./storage/githubcontents.js";
+export * from "./storage/filehosting.js";
+export * from "./persistence/migrations.js";
+export * from "./mcp/transport.js";
+export * from "./api/security.js";
+export * from "./modes/matrix.js";
+export * from "./modes/resolve.js";
+export * from "./binary/build.js";
+export * from "./surfaces/targets.js";
+export * from "./api/rate.js";
+export * from "./api/http.js";
+export * from "./api/service.js";
+export * from "./api/control.js";
+export * from "./extension/index.js";

package/library/public.js

@@ -0,0 +1,83 @@
+/**
+ * public library helpers compose fetch extraction serialization chunking and crawl contracts.
+ */
+import { crawl } from "../crawl/crawler.js";
+import { chunkmarkdown } from "../ai/chunk.js";
+import { estimatetokens } from "../ai/tokens.js";
+import { extracthtml } from "../scrape/extract.js";
+import { browseragent } from "../browser/agent.js";
+
+/** Selects the fetch or browser execution path. */
+export async function saddleurl(url, options = {}) {
+  const mode = options.mode ?? "fetch";
+  if (mode === "browser") return scrapewithbrowser(url, options);
+  return scrapeurl(url, options);
+}
+
+/** Fetches one URL through the caller supplied transport. */
+export async function scrapeurl(url, options = {}) {
+  const target = safeurl(url);
+  const response = await (options.fetcher ?? fetch)(target, { signal: options.signal, headers: options.headers });
+  if (!response.ok) throw new Error(`scrape request failed with ${response.status}`);
+  const html = await response.text();
+  return formatresult(scrapehtml(html, target, options), options);
+}
+
+/** Extracts a serializable result from HTML without network access. */
+export function scrapehtml(html, url, options = {}) {
+  const result = extracthtml(html, url);
+  return { content: result.text, metadata: { url: result.url, title: result.title, description: result.description, links: result.links }, html: options.includehtml ? html : undefined };
+}
+
+/** Exposes structured extraction for callers that do not need formatting. */
+export function extractcontent(html, options = {}) { return extracthtml(html, options.url); }
+
+/** Runs a scrape through the injected browser agent contract. */
+export async function scrapewithbrowser(url, options = {}) {
+  const agent = browseragent(options.browser);
+  await agent.navigate({ url, waituntil: options.waituntil ?? "networkidle" });
+  return formatresult({ content: await agent.text(), metadata: { url, title: await agent.title(), html: options.includehtml ? await agent.html() : undefined } }, options);
+}
+
+/** Serializes a result into a supported output format. */
+export function serializeresult(result, options = {}) {
+  const format = options.format ?? "json";
+  if (format === "json") return JSON.stringify(result, null, options.pretty ? 2 : 0);
+  if (format === "text") return result.content ?? "";
+  if (format === "markdown") return `# ${result.metadata?.title ?? "result"}\n\n${result.content ?? ""}`;
+  if (format === "xml") return `<result><title>${escape(result.metadata?.title ?? "")}</title><content>${escape(result.content ?? "")}</content></result>`;
+  if (format === "redis") return JSON.stringify({ content: result.content, metadata: result.metadata });
+  throw new TypeError(`unsupported format: ${format}`);
+}
+
+/** Converts local HTML into Markdown. */
+export function serializehtml(html, options = {}) { return serializeresult(scrapehtml(html, options.url), { format: "markdown" }); }
+
+/** Builds compact context for an agent or a vector pipeline. */
+export function formatforagent(result, options = {}) {
+  const content = result.content ?? "";
+  const chunks = chunkmarkdown(content, { maxtokens: options.maxchunksize ?? 4000 });
+  const lines = content.split(/[.!?]\s+/).filter(Boolean);
+  return { summary: lines.slice(0, 2).join(". "), keypoints: lines.slice(0, options.keypoints ?? 5), relevanturls: result.metadata?.links ?? [], chunks, tokencount: estimatetokens(content, options.model) };
+}
+
+/** Processes URLs in bounded groups and emits progress events. */
+export async function batchscrape(options = {}) {
+  const urls = options.urls ?? [];
+  const concurrency = options.concurrency ?? 10;
+  const results = [];
+  for (let index = 0; index < urls.length; index += concurrency) {
+    const group = urls.slice(index, index + concurrency);
+    const completed = await Promise.all(group.map((url) => scrapeurl(url, options)));
+    results.push(...completed);
+    options.onprogress?.({ completed: results.length, total: urls.length });
+  }
+  return results;
+}
+
+/** Runs the crawler through the public scrape contract. */
+export async function crawlurl(url, options = {}) { return crawl(url, { ...options, scrape: (target) => scrapeurl(target, options) }); }
+
+function safeurl(value) { const url = new URL(value); if (!["http:", "https:"].includes(url.protocol)) throw new TypeError("url must use http or https"); return url.href; }
+function formatresult(result, options) { return options.format ? { ...result, serialized: serializeresult(result, options) } : result; }
+function escape(value) { return String(value).replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;"); }