@wenathlan/saddle 1.8.7 → 1.8.8

package/README.md

@@ -6,7 +6,7 @@
   <strong>Storage-backed jobs, scraping contracts and portable runners for Node.js.</strong><br/>
   <strong>Binary computing engine, agent browser, scraper and packager.</strong><br/>
   <a href="https://github.com/wenathlan/saddle/actions/workflows/ci.yml"><img src="https://github.com/wenathlan/saddle/actions/workflows/ci.yml/badge.svg" alt="CI" /></a>
-  <a href="https://github.com/wenathlan/saddle/releases/tag/v1.8.7"><img src="https://img.shields.io/badge/release-v1.8.7-d35d3d" alt="Release 1.8.7" /></a>
+  <a href="https://github.com/wenathlan/saddle/releases/tag/v1.8.8"><img src="https://img.shields.io/badge/release-v1.8.8-d35d3d" alt="Release 1.8.8" /></a>
   <a href="https://github.com/wenathlan/saddle/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-GPL--3.0-202a2f" alt="GPL 3.0 license" /></a>
 </p>
 
@@ -127,7 +127,7 @@ The base permission set is `activeTab`, `scripting` and `storage`. It does not r
 | Failure | retry, circuit breaker, idempotency and resume are configurable |
 | Releases | version comes from the `vX.Y.Z` tag and must match `package.json` |
 
-Version 1.8.7 also removes the obsolete nested `scrape` package manifests and lockfile that generated a separate stale dependency graph. The dependency-free JavaScript scrape contracts remain in `scrape/`. The root lockfile is regenerated and CI runs `npm audit --audit-level=high` plus dependency review for pull requests. See [`docs/securityaudit-1.8.7.md`](docs/securityaudit-1.8.7.md) for the baseline and remediation record.
+Version 1.8.8 preserves the dependency-free JavaScript scrape contracts and groups the related crawl context in `scrape/crawl.js`. The retry and circuit context now lives in `runtime/retry.js`, while scraper-specific error classification is part of `core/errors.js`. The root lockfile and security gates remain authoritative; see [`docs/reorganization-1.8.8.md`](docs/reorganization-1.8.8.md) for the ownership decisions and [`docs/securityaudit-1.8.7.md`](docs/securityaudit-1.8.7.md) for the preceding security baseline.
 
 ## Package surfaces and release automation
 
@@ -173,12 +173,11 @@ The engine test suite is deterministic and does not require real credentials or
 ## Repository map
 
 ```text
-core/          errors, events, identifiers and hashing
+core/          engine errors, scrape error taxonomy, events, identifiers and hashing
 domain/        jobs, artifacts, sessions and providers
 memory/        working-set bridge, modes, objects and transforms
 storage/       local, chunked, remote and file-hosting adapters
-scrape/        dependency-free robots, cache, extraction, schema and normalization contracts
-crawl/         URL normalization, crawler and persistent frontier
+scrape/        robots, cache, extraction, schema, normalization and grouped crawl contracts
 queue/         queue, idempotency, saga and recovery
 browser/       fingerprint, session, agent and Playwright adapter contracts
 extension/     Manifest V3 reference surface and packager
@@ -186,6 +185,7 @@ protocol/      JSON, NDJSON, SSE and block serializers
 workflow/      manifests, templates and registry contracts
 packager/      package and publication plans
 release/       checksums, SBOM and provenance metadata
+runtime/       engine orchestration, capability detection, worker and grouped retry context
 web/           root-based static marketing site
 tests/         deterministic engine and extension coverage
 docs/          architecture, API, security, release and registry notes
@@ -199,7 +199,7 @@ Earlier README snapshots remain in `docs/plans/README.md`, `docs/talks9/README.m
 
 ## Current scope
 
-Version 1.8.7 extends the 1.8.6 engine with dependency remediation, explicit security gates, base-aware Pages assets, removal of the obsolete public debug directory and consolidated documentation. Browser binaries, provider credentials, n8n host registration, persistent databases, captcha solvers and production deployment remain caller-selected adapters. Future work should extend contracts without coupling the core to one forge, registry, browser or storage vendor.
+Version 1.8.8 extends the 1.8.7 engine with conservative context regrouping. Browser binaries, provider credentials, n8n host registration, persistent databases, captcha solvers and production deployment remain caller-selected adapters. Future work should extend contracts without coupling the core to one forge, registry, browser or storage vendor.
 
 ## License
 

package/api/service.js

@@ -1,7 +1,7 @@
 /**
  * saddle service exposes universal routes without choosing hono fastify express or another server.
  */
-import { crawl } from "../crawl/crawler.js";
+import { crawl } from "../scrape/crawl.js";
 import { ratelimiter } from "./rate.js";
 import { errorresponse, jsonresponse, sseresponse } from "./http.js";
 import { assertpublicurl } from "./security.js";

package/core/errors.js

@@ -35,3 +35,23 @@ export function aserror(error, jobid) {
   if (error?.name === "saddleerror") return error;
   return saddleerror(`job ${jobid} failed`, { code: errorcodes.jobfailed, cause: error, details: { jobid } });
 }
+
+/** Scrape error presets keep status, recovery and retry decisions explicit. */
+export const errorcatalog = Object.freeze({
+  timeout: { code: "E1001", statuscode: 504, retryable: true, recovery: "WAIT_AND_RETRY" },
+  connectionrefused: { code: "E1002", statuscode: 503, retryable: true, recovery: "WAIT_AND_RETRY" },
+  dns: { code: "E1003", statuscode: 503, retryable: true, recovery: "ROTATE_PROXY" },
+  ratelimited: { code: "E2001", statuscode: 429, retryable: true, recovery: "WAIT_AND_RETRY" },
+  forbidden: { code: "E2002", statuscode: 403, retryable: false, recovery: "REVIEW_ROBOTS_TXT" },
+  notfound: { code: "E2003", statuscode: 404, retryable: false, recovery: "STOP_CRAWLING" },
+  parse: { code: "E4002", statuscode: 422, retryable: false, recovery: "STOP_CRAWLING" },
+  captcha: { code: "E4003", statuscode: 403, retryable: false, recovery: "REVIEW_ROBOTS_TXT" },
+  session: { code: "E5001", statuscode: 401, retryable: true, recovery: "ROTATE_USER_AGENT" },
+  config: { code: "E6001", statuscode: 400, retryable: false, recovery: "STOP_CRAWLING" }
+});
+
+/** Creates a stable scrape error with recovery metadata. */
+export function webscrapeerror(kind, message, options = {}) { const preset = errorcatalog[kind] ?? errorcatalog.config; const error = new Error(message, { cause: options.cause }); error.name = "webscrapeerror"; error.code = options.code ?? preset.code; error.statuscode = options.statuscode ?? preset.statuscode; error.retryable = options.retryable ?? preset.retryable; error.recovery = options.recovery ?? preset.recovery; error.severity = options.severity ?? (error.statuscode >= 500 ? "high" : "medium"); error.details = options.details ?? {}; return error; }
+
+/** Converts unknown failures into the stable scrape taxonomy. */
+export function classifyerror(error) { if (error?.name === "webscrapeerror") return error; const message = String(error?.message ?? error); if (/timeout|aborted/i.test(message)) return webscrapeerror("timeout", message, { cause: error }); if (/dns|enotfound/i.test(message)) return webscrapeerror("dns", message, { cause: error }); return webscrapeerror("config", message, { cause: error }); }

package/docs/enginearchitecture.md

@@ -12,17 +12,18 @@ The project has no `src` folder. The root is the map of the engine.
 
 | folder | responsibility |
 |---|---|
-| `core` | errors, ids, clock, events, and tracing primitives |
+| `core` | engine errors, scrape error taxonomy, ids, events, and hashing primitives |
 | `domain` | jobs, sessions, artifacts, providers, and runtime records |
 | `storage` | storage adapter contract, local backend, and checksums |
 | `memory` | working set preparation, sync, and cleanup |
 | `runners` | provider factories and deterministic scheduling |
-| `runtime` | engine orchestration and output encoding |
+| `runtime` | engine orchestration, capability detection, worker boundary, retry and circuit context |
+| `scrape` | single page extraction, response normalization, robots policy and grouped crawl context |
 | `cli` | explicit command surface with local error handling |
 | `tests` | local deterministic tests without credentials |
 | `examples` | small runnable integration examples |
 
-All internal file names are lowercase and contain no underscore or hyphen. Related logic stays grouped and each module remains small enough to reason about in isolation.
+All internal file names are lowercase and contain no underscore or hyphen. Related logic stays grouped and each module remains small enough to reason about in isolation. Version 1.8.8 groups crawl URL normalization, traversal, frontier budgets and persistence in `scrape/crawl.js`; it does not merge distinct storage cache, job queue or browser session contracts merely because they use similar words.
 
 ## public contracts
 
@@ -64,7 +65,7 @@ The same contracts support paired modes. A mode can exist without its pair, and
 | browser | headless job definition | capture and replay adapter |
 | network | local deterministic job | remote provider and storage adapter |
 
-The first cut implements library, cli, binary entry point, internal memory, and internal file. The other modes are extension points, not hardcoded promises.
+The first cut implements library, cli, binary entry point, internal memory, and internal file. The other modes are extension points, not hardcoded promises. Version 1.8.8 preserves this boundary while reducing duplicated active context files.
 
 ## infrastructure rules
 

package/docs/libraryapi.md

@@ -28,7 +28,7 @@ The public API is designed around injected transports. Consumers can use the sam
 | `workflowtriggers` / `triggermatch` | normalize and match manual, event, schedule and retry starts |
 | `resumablerun` / `transitionrun` | recover remote run state through legal transitions |
 | `extractsemantic` | expose bounded headings, landmarks, controls and links |
-| `crawlfrontier` | prioritize URLs and enforce page and domain budgets |
+| `crawlfrontier` / `normalizeurl` / `persistentqueue` | grouped crawl context for URL normalization, traversal budgets and durable frontier state |
 | `provenance` / `mergeprovenance` | link context chunks to source and retrieval evidence |
 | `metricstore` | collect bounded counters and duration summaries |
 | `authorize` | verify caller credentials through an injected verifier |
@@ -58,8 +58,8 @@ The `fetcher`, browser adapter, persistence adapter, proxy pool, captcha solver,
 
 The package exposes explicit subpaths for `./browser`, `./bot`, `./captcha`, `./memory-engine`, and `./deploy`. Desktop, mobile, and n8n contracts are exported from the root entry; the root entry remains the complete JavaScript API for consumers that prefer one import.
 
-Version 1.1 adds `./extension`, which exposes browser-neutral message, snapshot and service-worker routing contracts. The concrete Manifest V3 files live in `extension/`; they are not imported by the core at runtime and do not require Chrome when the library is used as a Node package. The browser surface also exposes snapshots, tab/frame context, action results, bounded action batches and recording through `./browser`.
+The `./extension` subpath exposes browser-neutral message, snapshot and service-worker routing contracts. The concrete Manifest V3 files live in `extension/`; they are not imported by the core at runtime and do not require Chrome when the library is used as a Node package. The browser surface also exposes snapshots, tab/frame context, action results, bounded action batches and recording through `./browser`.
 
-The current main branch adds `desktopmanifest`, `mobilemanifest`, `desktopadapter`, `mobileadapter`, `n8nnode`, `n8nmatch`, `n8nexecute`, `controlsurface`, `controlservice`, and `workerbridge`. These factories describe surface boundaries and invoke caller-owned handlers; they do not install a native toolkit, start an n8n server, create a dashboard, or store credentials. They will be included in the next versioned release after the compatibility gates are complete.
+The current release exposes `desktopmanifest`, `mobilemanifest`, `desktopadapter`, `mobileadapter`, `n8nnode`, `n8nmatch`, `n8nexecute`, `controlsurface`, `controlservice`, and `workerbridge`. These factories describe surface boundaries and invoke caller-owned handlers; they do not install a native toolkit, start an n8n server, create a dashboard, or store credentials.
 
 The root entry is cross-runtime safe for the tested core contract. Filesystem, Node HTTP, persistent queue, file sessions, local memory and captcha evidence adapters remain available through explicit Node-only files or subpaths. `runtimecontract` reports the capabilities of the current global scope, while `memorystorage` provides a process-local backend for browser workers, Deno, Bun and deterministic tests.

package/docs/release.md

@@ -6,10 +6,10 @@ The release path is intentionally split into source validation, package validati
 
 | step | owner | condition |
 |---|---|---|
-| package version | repository | `package.json` matches the release tag, for example `1.8.7` |
+| package version | repository | `package.json` matches the release tag, for example `1.8.8` |
 | quality gate | GitHub Actions | `npm run pack:check` passes |
-| tag | repository owner | tag `v1.8.7` points to the validated release commit |
-| GitHub release | repository owner | release `v1.8.7` is created from the validated tag |
+| tag | repository owner | tag `v1.8.8` points to the validated release commit |
+| GitHub release | repository owner | release `v1.8.8` is created from the validated tag |
 | GitHub Packages | GitHub Actions | `publishgithubnpm.yml`, `publishghcr.yml`, `publishmaven.yml`, `publishnuget.yml`, and `publishrubygems.yml` use `GITHUB_TOKEN` |
 | public npmjs | owner-managed GitHub Actions secret | `publishnpmjs.yml` uses `NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}` and derives the version from the release tag or latest release in manual maintenance runs |
 | extension zip | GitHub Actions | `buildextension.yml` derives the version from the release tag, validates the unpacked artifact and attaches `saddle-extension-<version>.zip` |
@@ -22,8 +22,8 @@ The npm token previously sent in chat is compromised and must not be used. GitHu
 
 ```text
 npm run pack:check
-git tag v1.8.7
-git push origin v1.8.7
+git tag v1.8.8
+git push origin v1.8.8
 ```
 
 The release-created event is the publication and extension-asset trigger for the release workflows. A dry-run verifies package shape and local tests, but cannot verify registry ownership, Trusted Publisher configuration, package scope authorization, package visibility or browser-store submission; those remain settings controlled by the owner.

package/docs/releaseassets.md

@@ -4,7 +4,7 @@ The Node-only release adapter creates deterministic metadata for caller-selected
 
 ```bash
 npm run release:assets -- \
-  --version 1.8.7 \
+  --version 1.8.8 \
   --output build/release \
   --artifact build/saddle.tgz \
   --build-type caller-build \

package/docs/reorganization-1.8.8.md

@@ -0,0 +1,31 @@
+# Reorganization 1.8.8
+
+## Purpose
+
+Version 1.8.8 applies the architecture skill's grouping rule to active JavaScript contracts. The objective is not to erase historical features or flatten unrelated domains. The objective is to give each correlated responsibility one canonical owner, reduce redundant folder boundaries, and preserve public names and deterministic behavior.
+
+## Audit result
+
+The active package contains a JavaScript runtime and a legacy TypeScript scrape tree. The TypeScript tree is not imported by the active root entry or package export map. It remains in place because it documents a broader historical surface and contains features that would require a separate compatibility decision before conversion to the current JavaScript library boundary.
+
+| Context | Previous ownership | Version 1.8.8 owner | Decision |
+| --- | --- | --- | --- |
+| Crawl acquisition | `crawl/normalize.js`, `crawl/frontier.js`, `crawl/crawler.js`, `crawl/persistent.js` | `scrape/crawl.js` | Consolidated. URL normalization, traversal, frontier budgets and durable crawl state share one context and preserve all public function names. |
+| Retry protection | `retry/policy.js`, `retry/circuit.js` | `runtime/retry.js` | Consolidated. Both modules protect caller-owned runners, storage and network handlers from transient failure storms. |
+| Scrape error classification | `errors/taxonomy.js` | `core/errors.js` | Consolidated. Generic engine errors and scrape recovery metadata share one stable error boundary. |
+| Page cache | `scrape/cache.js` | `scrape/cache.js` | Kept separate. It caches single-page scrape results and robots policy. |
+| Storage cache | `storage/cache.js` | `storage/cache.js` | Kept separate. It manages hot and cold byte-backed tiers, encoding, eviction and revalidation. |
+| Job persistence | `queue/persistent.js` | `queue/persistent.js` | Kept separate. It persists generic job lifecycle records rather than crawl URLs. |
+| Browser sessions | `browser/session.js`, `domain/sessions.js`, `sessions/*` | existing owners | Kept separate. These contracts represent browser identity, domain validation and file or replay persistence. |
+
+## Compatibility rule
+
+The root export barrel continues to expose `normalizeurl`, `sameorigin`, `crawlfrontier`, `persistentqueue`, `crawl`, `retrypolicy`, `circuitbreaker`, `webscrapeerror` and `classifyerror`. Internal imports, tests and package payload declarations now point to the canonical owners. No public export was renamed for the regrouping.
+
+## Verification rule
+
+The reorganization is accepted only when syntax checks, deterministic tests, format checks, package dry-run, dependency audit, web typecheck and Pages build pass. A deleted file is considered redundant only after its imports, tests, package payload and documented public names have been checked.
+
+## Deferred historical surface
+
+The legacy TypeScript modules under `scrape/` remain a separate historical surface. They include richer browser, renderer, cache, middleware, server, session, pool, sitemap and format contracts. They are not silently merged into the active JavaScript modules because doing so would either discard features or introduce an undeclared TypeScript and external dependency build into the published core.

package/extension/manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 3,
   "name": "Saddle browser bridge",
-  "version": "1.8.7",
+  "version": "1.8.8",
   "description": "User initiated page snapshots through the Saddle browser contract.",
   "minimum_chrome_version": "110",
   "permissions": ["activeTab", "scripting", "storage"],

package/index.js

@@ -28,7 +28,7 @@ export * from "./runners/heartbeat.js";
 export * from "./workflow/triggers.js";
 export * from "./dispatch/resumable.js";
 export * from "./scrape/semantic.js";
-export * from "./crawl/frontier.js";
+export * from "./scrape/crawl.js";
 export * from "./ai/provenance.js";
 export * from "./observability/metrics.js";
 export * from "./api/auth.js";
@@ -73,9 +73,6 @@ export * from "./protocol/blocks.js";
 export * from "./workflow/manifest.js";
 export * from "./workflow/templates.js";
 export * from "./workflow/registry.js";
-export * from "./crawl/normalize.js";
-export * from "./crawl/crawler.js";
-export * from "./crawl/persistent.js";
 export * from "./scrape/schema.js";
 export * from "./scrape/normalize.js";
 export * from "./mcp/server.js";
@@ -102,9 +99,7 @@ export * from "./surfaces/adapters.js";
 export * from "./surfaces/controls.js";
 export * from "./surfaces/operations.js";
 export * from "./library/public.js";
-export * from "./errors/taxonomy.js";
-export * from "./retry/policy.js";
-export * from "./retry/circuit.js";
+export * from "./runtime/retry.js";
 export * from "./storage/githubcontents.js";
 export * from "./storage/filehosting.js";
 export * from "./persistence/migrations.js";

package/library/public.js

@@ -1,7 +1,7 @@
 /**
  * public library helpers compose fetch extraction serialization chunking and crawl contracts.
  */
-import { crawl } from "../crawl/crawler.js";
+import { crawl } from "../scrape/crawl.js";
 import { chunkmarkdown } from "../ai/chunk.js";
 import { estimatetokens } from "../ai/tokens.js";
 import { extracthtml } from "../scrape/extract.js";

package/mcp/server.js

@@ -1,7 +1,7 @@
 /**
  * mcp server implements a small JSON RPC tool surface and remains transport agnostic.
  */
-import { crawl } from "../crawl/crawler.js";
+import { crawl } from "../scrape/crawl.js";
 import { extractwithschema } from "../scrape/schema.js";
 import { jsonencode } from "../protocol/json.js";
 import { browsertools } from "./browser.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wenathlan/saddle",
-  "version": "1.8.7",
+  "version": "1.8.8",
   "description": "binary computing engine that turns distributed storage into a publishable working set",
   "type": "module",
   "private": false,
@@ -84,7 +84,6 @@
     "queue",
     "dispatch",
     "scrape",
-    "crawl",
     "api",
     "mcp",
     "browser",
@@ -94,8 +93,6 @@
     "webhook",
     "surfaces",
     "library",
-    "errors",
-    "retry",
     "server",
     "binary",
     "format",
@@ -113,7 +110,7 @@
     "LICENSE"
   ],
   "scripts": {
-    "check": "node --check index.js && node --check runtime/engine.js && node --check cli/main.js && node --check extension/protocol.js && node --check extension/serviceworker.js && node --check extension/worker.js && node --check extension/content.js && node --check extension/pagebridge.js && node --check extension/popup.js && node --check extension/permissions.js && node --check extension/build.js && node --check release/assets.js && node --check sessions/replay.js && node --check browser/recorder.js && node --check scrape/normalize.js",
+    "check": "node --check index.js && node --check runtime/engine.js && node --check runtime/retry.js && node --check cli/main.js && node --check extension/protocol.js && node --check extension/serviceworker.js && node --check extension/worker.js && node --check extension/content.js && node --check extension/pagebridge.js && node --check extension/popup.js && node --check extension/permissions.js && node --check extension/build.js && node --check release/assets.js && node --check sessions/replay.js && node --check browser/recorder.js && node --check scrape/normalize.js && node --check scrape/crawl.js",
     "formatcheck": "node format/check.js",
     "test": "node --test tests/*.test.js",
     "run": "node cli/main.js",

package/runtime/retry.js

@@ -0,0 +1,27 @@
+/**
+ * retry context groups transient retry policy and circuit protection for runners,
+ * storage adapters and network-facing surfaces.
+ */
+
+/** Creates bounded exponential retry behavior for retryable failures. */
+export function retrypolicy(options = {}) {
+  const maxattempts = options.maxattempts ?? 3;
+  const base = options.base ?? 1000;
+  const factor = options.factor ?? 2;
+  const cap = options.cap ?? 30000;
+  return { async run(handler) { let last; for (let attempt = 1; attempt <= maxattempts; attempt += 1) { try { return await handler(attempt); } catch (error) { last = error; if (error?.retryable !== true || attempt === maxattempts) throw error; const wait = Math.min(cap, base * factor ** (attempt - 1)) + Math.floor(Math.random() * (options.jitter ?? 0)); options.onretry?.({ attempt, wait, error }); await delay(wait); } } throw last; } };
+}
+
+/** Creates a circuit breaker that opens after repeated handler failures. */
+export function circuitbreaker(options = {}) {
+  const threshold = options.failurethreshold ?? 5;
+  const resettimeout = options.resettimeout ?? 60000;
+  let failures = 0;
+  let openedat = 0;
+  let state = "closed";
+  async function execute(handler) { if (state === "open") { if (Date.now() - openedat < resettimeout) throw new Error("circuit breaker is open"); state = "halfopen"; } try { const result = await handler(); failures = 0; state = "closed"; return result; } catch (error) { failures += 1; if (failures >= threshold) { state = "open"; openedat = Date.now(); } throw error; } }
+  return { execute, status() { return { state, failures, openedat }; }, reset() { failures = 0; openedat = 0; state = "closed"; } };
+}
+
+/** Waits between retry attempts without introducing an external timer dependency. */
+function delay(milliseconds) { return milliseconds ? new Promise((resolve) => setTimeout(resolve, milliseconds)) : Promise.resolve(); }

package/scrape/crawl.js

@@ -0,0 +1,104 @@
+/**
+ * scrape crawl context owns URL normalization traversal frontier and durable crawl state.
+ *
+ * The context keeps single page acquisition injectable while grouping the correlated
+ * crawl responsibilities that previously lived in four separate top level files.
+ */
+
+/** Removes fragments and tracking parameters before crawl deduplication. */
+export function normalizeurl(value) {
+  const url = new URL(value);
+  url.hash = "";
+  for (const key of [...url.searchParams.keys()]) if (/^(utm_|fbclid$|msclkid$|gclid$|gclsrc$|dclid$|gbraid$|wbraid$|twclid$|campaign$|content$|term$|source$|medium$|ref$|share_id$)/i.test(key)) url.searchParams.delete(key);
+  if (url.pathname.length > 1) url.pathname = url.pathname.replace(/\/+$/, "");
+  return url.href;
+}
+
+/** Compares two crawl targets by origin. */
+export function sameorigin(left, right) { return new URL(left).origin === new URL(right).origin; }
+
+/** Creates a bounded priority frontier for crawler and queue adapters. */
+export function crawlfrontier(options = {}) {
+  const maxpages = Number(options.maxpages ?? 20);
+  const maxperdomain = Number(options.maxperdomain ?? maxpages);
+  const queue = [];
+  const seen = new Set();
+  const completed = new Set();
+  const domains = new Map();
+
+  /** Adds a URL once while respecting the global page budget. */
+  function add(input = {}) {
+    const url = String(input.url ?? "");
+    if (!url || seen.has(url) || seen.size >= maxpages) return false;
+    seen.add(url);
+    queue.push({ url, depth: Number(input.depth ?? 0), priority: Number(input.priority ?? 0), discoveredat: Number(input.discoveredat ?? Date.now()) });
+    queue.sort((left, right) => right.priority - left.priority || left.discoveredat - right.discoveredat);
+    return true;
+  }
+
+  /** Removes the next URL that still fits its per-domain budget. */
+  function next() {
+    while (queue.length) {
+      const item = queue.shift();
+      const domain = new URL(item.url).hostname;
+      if ((domains.get(domain) ?? 0) >= maxperdomain) continue;
+      domains.set(domain, (domains.get(domain) ?? 0) + 1);
+      return { ...item };
+    }
+    return null;
+  }
+
+  /** Records a completed URL for diagnostics and persistence-friendly state. */
+  function complete(url) { completed.add(String(url)); }
+
+  /** Returns stable frontier diagnostics without exposing mutable collections. */
+  function state() { return { maxpages, maxperdomain, queued: queue.length, discovered: seen.size, completed: completed.size, domains: Object.fromEntries(domains) }; }
+
+  return { add, next, complete, state, list() { return queue.map((item) => ({ ...item })); } };
+}
+
+/** Creates a durable crawl queue around a caller-owned store. */
+export function persistentqueue(options = {}) {
+  const store = options.store;
+  const values = [];
+  const seen = new Set();
+
+  /** Restores unfinished crawl records from the injected store. */
+  async function restore() { if (typeof store?.list !== "function") return; for (const item of await store.list()) if (!seen.has(item.url) && item.status !== "done") { seen.add(item.url); values.push(item); } }
+
+  /** Adds a crawl record and persists it when the store supports writes. */
+  async function add(item) { if (!item?.url || seen.has(item.url)) return false; const value = { url: item.url, depth: item.depth ?? 0, status: "queued", createdat: Date.now(), metadata: item.metadata ?? {} }; seen.add(value.url); values.push(value); if (typeof store?.save === "function") await store.save(value); return true; }
+
+  /** Claims the next queued crawl record. */
+  async function next() { const item = values.find((value) => value.status === "queued"); if (!item) return null; item.status = "running"; if (typeof store?.update === "function") await store.update(item.url, item); return item; }
+
+  /** Completes a crawl record and persists the resulting status. */
+  async function complete(url, patch = {}) { const item = values.find((value) => value.url === url); if (!item) return null; Object.assign(item, patch, { status: patch.status ?? "done", processedat: Date.now() }); if (typeof store?.update === "function") await store.update(url, item); return item; }
+
+  return { restore, add, next, complete, list() { return values.map((value) => ({ ...value })); } };
+}
+
+/** Runs bounded breadth first traversal through the injected single page scrape contract. */
+export async function crawl(start, options = {}) {
+  const maxdepth = options.maxdepth ?? 1;
+  const maxpages = options.maxpages ?? 20;
+  const sameDomain = options.samedomain ?? true;
+  const frontier = crawlfrontier({ maxpages, maxperdomain: options.maxperdomain ?? maxpages });
+  frontier.add({ url: normalizeurl(start), depth: 0, priority: options.startpriority ?? 0 });
+  const results = [];
+  while (frontier.state().queued && results.length < maxpages) {
+    const current = frontier.next();
+    if (!current || current.depth > maxdepth) continue;
+    const result = await options.scrape(current.url);
+    results.push({ ...result, depth: current.depth });
+    frontier.complete(current.url);
+    if (current.depth >= maxdepth) continue;
+    for (const link of result.links ?? []) {
+      let url;
+      try { url = normalizeurl(link); } catch { continue; }
+      if (sameDomain && !sameorigin(start, url)) continue;
+      frontier.add({ url, depth: current.depth + 1, priority: Number(options.priority?.(url, result) ?? 0) });
+    }
+  }
+  return { results, stats: { ...frontier.state(), completed: results.length, maxdepth, maxpages } };
+}

package/crawl/crawler.js

@@ -1,29 +0,0 @@
-/**
- * crawler performs bounded breadth first traversal through the scraper contract.
- */
-import { normalizeurl, sameorigin } from "./normalize.js";
-import { crawlfrontier } from "./frontier.js";
-
-export async function crawl(start, options = {}) {
-  const maxdepth = options.maxdepth ?? 1;
-  const maxpages = options.maxpages ?? 20;
-  const sameDomain = options.samedomain ?? true;
-  const frontier = crawlfrontier({ maxpages, maxperdomain: options.maxperdomain ?? maxpages });
-  frontier.add({ url: normalizeurl(start), depth: 0, priority: options.startpriority ?? 0 });
-  const results = [];
-  while (frontier.state().queued && results.length < maxpages) {
-    const current = frontier.next();
-    if (!current || current.depth > maxdepth) continue;
-    const result = await options.scrape(current.url);
-    results.push({ ...result, depth: current.depth });
-    frontier.complete(current.url);
-    if (current.depth >= maxdepth) continue;
-    for (const link of result.links ?? []) {
-      let url;
-      try { url = normalizeurl(link); } catch { continue; }
-      if (sameDomain && !sameorigin(start, url)) continue;
-      frontier.add({ url, depth: current.depth + 1, priority: Number(options.priority?.(url, result) ?? 0) });
-    }
-  }
-  return { results, stats: { ...frontier.state(), completed: results.length, maxdepth, maxpages } };
-}

package/crawl/frontier.js

@@ -1,34 +0,0 @@
-/**
- * crawl frontier provides priorities, per-domain budgets and persistent-friendly queue state.
- */
-
-/** Creates a bounded priority frontier for crawler and queue adapters. */
-export function crawlfrontier(options = {}) {
-  const maxpages = Number(options.maxpages ?? 20);
-  const maxperdomain = Number(options.maxperdomain ?? maxpages);
-  const queue = [];
-  const seen = new Set();
-  const completed = new Set();
-  const domains = new Map();
-  function add(input = {}) {
-    const url = String(input.url ?? "");
-    if (!url || seen.has(url) || seen.size >= maxpages) return false;
-    seen.add(url);
-    queue.push({ url, depth: Number(input.depth ?? 0), priority: Number(input.priority ?? 0), discoveredat: Number(input.discoveredat ?? Date.now()) });
-    queue.sort((left, right) => right.priority - left.priority || left.discoveredat - right.discoveredat);
-    return true;
-  }
-  function next() {
-    while (queue.length) {
-      const item = queue.shift();
-      const domain = new URL(item.url).hostname;
-      if ((domains.get(domain) ?? 0) >= maxperdomain) continue;
-      domains.set(domain, (domains.get(domain) ?? 0) + 1);
-      return { ...item };
-    }
-    return null;
-  }
-  function complete(url) { completed.add(String(url)); }
-  function state() { return { maxpages, maxperdomain, queued: queue.length, discovered: seen.size, completed: completed.size, domains: Object.fromEntries(domains) }; }
-  return { add, next, complete, state, list() { return queue.map((item) => ({ ...item })); } };
-}

package/crawl/normalize.js

@@ -1,14 +0,0 @@
-/**
- * url normalization removes tracking noise before frontier deduplication.
- */
-const tracking = /^(utm_|fbclid$|msclkid$|gclid$|gclsrc$|dclid$|gbraid$|wbraid$|twclid$|campaign$|content$|term$|source$|medium$|ref$|share_id$)/i;
-
-export function normalizeurl(value) {
-  const url = new URL(value);
-  url.hash = "";
-  for (const key of [...url.searchParams.keys()]) if (tracking.test(key)) url.searchParams.delete(key);
-  if (url.pathname.length > 1) url.pathname = url.pathname.replace(/\/+$/, "");
-  return url.href;
-}
-
-export function sameorigin(left, right) { return new URL(left).origin === new URL(right).origin; }

package/crawl/persistent.js

@@ -1,13 +0,0 @@
-/**
- * persistent crawl queue uses an injected store and falls back to memory when no store exists.
- */
-export function persistentqueue(options = {}) {
-  const store = options.store;
-  const values = [];
-  const seen = new Set();
-  async function restore() { if (typeof store?.list !== "function") return; for (const item of await store.list()) if (!seen.has(item.url) && item.status !== "done") { seen.add(item.url); values.push(item); } }
-  async function add(item) { if (!item?.url || seen.has(item.url)) return false; const value = { url: item.url, depth: item.depth ?? 0, status: "queued", createdat: Date.now(), metadata: item.metadata ?? {} }; seen.add(value.url); values.push(value); if (typeof store?.save === "function") await store.save(value); return true; }
-  async function next() { const item = values.find((value) => value.status === "queued"); if (!item) return null; item.status = "running"; if (typeof store?.update === "function") await store.update(item.url, item); return item; }
-  async function complete(url, patch = {}) { const item = values.find((value) => value.url === url); if (!item) return null; Object.assign(item, patch, { status: patch.status ?? "done", processedat: Date.now() }); if (typeof store?.update === "function") await store.update(url, item); return item; }
-  return { restore, add, next, complete, list() { return values.map((value) => ({ ...value })); } };
-}

package/errors/taxonomy.js

@@ -1,18 +0,0 @@
-/**
- * scrape errors carry a stable code status retry flag severity and recovery hint.
- */
-export const errorcatalog = Object.freeze({
-  timeout: { code: "E1001", statuscode: 504, retryable: true, recovery: "WAIT_AND_RETRY" },
-  connectionrefused: { code: "E1002", statuscode: 503, retryable: true, recovery: "WAIT_AND_RETRY" },
-  dns: { code: "E1003", statuscode: 503, retryable: true, recovery: "ROTATE_PROXY" },
-  ratelimited: { code: "E2001", statuscode: 429, retryable: true, recovery: "WAIT_AND_RETRY" },
-  forbidden: { code: "E2002", statuscode: 403, retryable: false, recovery: "REVIEW_ROBOTS_TXT" },
-  notfound: { code: "E2003", statuscode: 404, retryable: false, recovery: "STOP_CRAWLING" },
-  parse: { code: "E4002", statuscode: 422, retryable: false, recovery: "STOP_CRAWLING" },
-  captcha: { code: "E4003", statuscode: 403, retryable: false, recovery: "REVIEW_ROBOTS_TXT" },
-  session: { code: "E5001", statuscode: 401, retryable: true, recovery: "ROTATE_USER_AGENT" },
-  config: { code: "E6001", statuscode: 400, retryable: false, recovery: "STOP_CRAWLING" }
-});
-
-export function webscrapeerror(kind, message, options = {}) { const preset = errorcatalog[kind] ?? errorcatalog.config; const error = new Error(message, { cause: options.cause }); error.name = "webscrapeerror"; error.code = options.code ?? preset.code; error.statuscode = options.statuscode ?? preset.statuscode; error.retryable = options.retryable ?? preset.retryable; error.recovery = options.recovery ?? preset.recovery; error.severity = options.severity ?? (error.statuscode >= 500 ? "high" : "medium"); error.details = options.details ?? {}; return error; }
-export function classifyerror(error) { if (error?.name === "webscrapeerror") return error; const message = String(error?.message ?? error); if (/timeout|aborted/i.test(message)) return webscrapeerror("timeout", message, { cause: error }); if (/dns|enotfound/i.test(message)) return webscrapeerror("dns", message, { cause: error }); return webscrapeerror("config", message, { cause: error }); }

package/retry/circuit.js

@@ -1,15 +0,0 @@
-/**
- * circuit breaker protects providers from repeated failure storms.
- */
-export function circuitbreaker(options = {}) {
-  const threshold = options.failurethreshold ?? 5;
-  const resettimeout = options.resettimeout ?? 60000;
-  let failures = 0;
-  let openedat = 0;
-  let state = "closed";
-  async function execute(handler) {
-    if (state === "open") { if (Date.now() - openedat < resettimeout) throw new Error("circuit breaker is open"); state = "halfopen"; }
-    try { const result = await handler(); failures = 0; state = "closed"; return result; } catch (error) { failures += 1; if (failures >= threshold) { state = "open"; openedat = Date.now(); } throw error; }
-  }
-  return { execute, status() { return { state, failures, openedat }; }, reset() { failures = 0; openedat = 0; state = "closed"; } };
-}

package/retry/policy.js

@@ -1,12 +0,0 @@
-/**
- * retry policy handles transient errors and keeps non retryable failures terminal.
- */
-export function retrypolicy(options = {}) {
-  const maxattempts = options.maxattempts ?? 3;
-  const base = options.base ?? 1000;
-  const factor = options.factor ?? 2;
-  const cap = options.cap ?? 30000;
-  return { async run(handler) { let last; for (let attempt = 1; attempt <= maxattempts; attempt += 1) { try { return await handler(attempt); } catch (error) { last = error; if (error?.retryable !== true || attempt === maxattempts) throw error; const wait = Math.min(cap, base * factor ** (attempt - 1)) + Math.floor(Math.random() * (options.jitter ?? 0)); options.onretry?.({ attempt, wait, error }); await delay(wait); } } throw last; } };
-}
-
-function delay(milliseconds) { return milliseconds ? new Promise((resolve) => setTimeout(resolve, milliseconds)) : Promise.resolve(); }