@opencraw/core 0.1.2 → 0.1.4

package/dist/index.esm.js

@@ -1,12 +1,15 @@
 import { z } from 'zod';
 import { randomInt, randomUUID } from 'node:crypto';
-import { readFile, mkdir, writeFile, stat, readdir } from 'node:fs/promises';
-import { resolve as resolve$1, dirname, extname, join as join$1 } from 'node:path';
+import { readFile, mkdir, writeFile, rm, stat, readdir } from 'node:fs/promises';
+import { resolve as resolve$1, join as join$1, dirname, extname } from 'node:path';
 import { firefox, webkit, chromium, request } from 'playwright';
 import { createWriteStream } from 'node:fs';
 import { once } from 'node:events';
 import { JSONPath } from 'jsonpath-plus';
 import { load } from 'cheerio';
+import { gunzipSync } from 'node:zlib';
+import { DOMParser, DOMImplementation, XMLSerializer } from '@xmldom/xmldom';
+import { useNamespaces } from 'xpath';
 import { fileURLToPath } from 'node:url';
 
 /** Resource types a page may skip loading, to save bandwidth on per-GB proxies. */
@@ -67,10 +70,20 @@ const pluginSchema = z.strictObject({
   options: z.record(z.string(), z.unknown()).optional()
 });
 const accessProfileSchema = z.union([directSchema, proxySchema, poolSchema, cdpSchema, pluginSchema]);
+const hostRuleSchema = z.strictObject({
+  delayMs: z.int().nonnegative().optional(),
+  concurrency: z.int().min(1).max(256).optional()
+});
+const throttleConfigSchema = z.strictObject({
+  delayMs: z.int().nonnegative().optional(),
+  concurrency: z.int().min(1).max(256).optional(),
+  domains: z.record(z.string().regex(/^[\w.-]+$/, 'a domain such as example.com'), hostRuleSchema).optional()
+});
 const accessConfigSchema = z.strictObject({
   $schema: z.string().optional(),
   profiles: z.record(z.string().regex(/^[\w-]+$/, 'a profile name is letters, digits, hyphens and underscores'), accessProfileSchema),
-  default: z.string().optional()
+  default: z.string().optional(),
+  throttle: throttleConfigSchema.optional()
 }).refine(config => config.default === undefined || Object.hasOwn(config.profiles, config.default), {
   message: 'default names a profile that does not exist',
   path: ['default']
@@ -1179,6 +1192,17 @@ async function blockResources(context, types) {
     await route.continue();
   });
 }
+/**
+ * What a context gets after it opened: the cookies to add and the resource
+ * types to skip.
+ *
+ * @param context - The context.
+ * @param options - The session options.
+ */
+async function applySessionExtras(context, options) {
+  if (options.cookies !== undefined && options.cookies.length > 0) await context.addCookies(options.cookies);
+  if (options.blockResources !== undefined && options.blockResources.length > 0) await blockResources(context, new Set(options.blockResources));
+}
 /** A launched browser; sessions are opened from it and closed independently. */
 class BrowserClient {
   browser;
@@ -1238,8 +1262,7 @@ class BrowserClient {
     };
     const context = await this.browser.newContext(contextOptions);
     if (this.config.timeoutMs !== undefined) context.setDefaultTimeout(this.config.timeoutMs);
-    if (options.cookies !== undefined && options.cookies.length > 0) await context.addCookies(options.cookies);
-    if (options.blockResources !== undefined && options.blockResources.length > 0) await blockResources(context, new Set(options.blockResources));
+    await applySessionExtras(context, options);
     const page = await context.newPage();
     return new BrowserSession(context, page);
   }
@@ -1248,13 +1271,206 @@ class BrowserClient {
   }
 }
 
+/** The file that marks a profile as in use, holding the owning process id. */
+const LOCK_FILE = '.opencraw.lock';
+/** A browser profile name: it becomes a directory, so no separators or dots. */
+const BROWSER_PROFILE_NAME = /^[\w-]+$/;
+/**
+ * Browser profiles that persist between runs: each is a directory of a real
+ * browser's user data (cookies, local storage, IndexedDB, cache, service
+ * workers), so a login, a consent choice or a site's trust in a returning
+ * visitor carries over to the next run. The browser equivalent of a user who
+ * never clears their history.
+ *
+ * A profile directory can be open in one browser at a time. Within this
+ * crawler, a second use waits for the first to close; the same owner (one
+ * recipe run reopening after a rotation) takes it over instead. Another
+ * crawler holding it, in this process or another, is reported, not waited
+ * for: a lock file in the profile names the process, and one left by a
+ * process that died is taken over. (Chromium's own profile lock is not
+ * enough: headless builds do not take it.)
+ */
+class BrowserProfiles {
+  directory;
+  config;
+  held = new Map();
+  waiting = new Map();
+  /**
+   * @param directory - Where the profiles live, one subdirectory each.
+   * @param config - The crawler's browser settings (type, binary, headless, timeouts).
+   */
+  constructor(directory, config = {}) {
+    this.directory = directory;
+    this.config = config;
+  }
+  async take(name, owner) {
+    for (;;) {
+      const holder = this.held.get(name);
+      if (holder === undefined) return;
+      if (holder.owner === owner) {
+        await holder.session.close();
+        continue;
+      }
+      await new Promise(resolve => {
+        const queue = this.waiting.get(name) ?? [];
+        queue.push(resolve);
+        this.waiting.set(name, queue);
+      });
+    }
+  }
+  free(name) {
+    this.held.delete(name);
+    this.waiting.get(name)?.shift()?.();
+  }
+  launch(path, options) {
+    const type = this.config.browserType ?? DEFAULT_BROWSER_CONFIG.browserType;
+    const launcher = type === 'firefox' ? firefox : type === 'webkit' ? webkit : chromium;
+    return launcher.launchPersistentContext(path, {
+      headless: this.config.headless ?? DEFAULT_BROWSER_CONFIG.headless,
+      slowMo: this.config.slowMo,
+      executablePath: this.config.executablePath,
+      proxy: options.proxy ?? this.config.proxy,
+      extraHTTPHeaders: options.headers,
+      userAgent: options.userAgent,
+      viewport: options.viewport,
+      ignoreHTTPSErrors: this.config.ignoreHTTPSErrors === true || options.ignoreHTTPSErrors === true
+    });
+  }
+  /**
+   * The profile's directory.
+   *
+   * @param name - A profile name.
+   * @returns The absolute path.
+   */
+  pathOf(name) {
+    if (!BROWSER_PROFILE_NAME.test(name)) throw new Error(`browser profile "${name}": a name is letters, digits, hyphens and underscores`);
+    return resolve$1(this.directory, name);
+  }
+  /**
+   * Opens a profile in its own browser, waiting while another run of this
+   * crawler uses it.
+   *
+   * @param name - The profile.
+   * @param options - Proxy, headers, viewport, cookies to add. `storageState` is ignored: the profile has its own.
+   * @param owner - Who opens it; the same owner reopening closes its previous session first.
+   * @returns The session; closing it frees the profile.
+   */
+  async open(name, options, owner) {
+    const path = this.pathOf(name);
+    await this.take(name, owner);
+    let context;
+    let unlock = unlocked;
+    try {
+      await mkdir(path, {
+        recursive: true
+      });
+      unlock = await lockProfile(path, name);
+      context = await this.launch(path, options);
+    } catch (error) {
+      await unlock();
+      this.free(name);
+      const message = error instanceof Error ? error.message : String(error);
+      if (/ProcessSingleton|SingletonLock|already in use/i.test(message)) throw new Error(`browser profile "${name}" is open in another browser (${path}); close it or use another profile`, {
+        cause: error
+      });
+      throw error;
+    }
+    if (this.config.timeoutMs !== undefined) context.setDefaultTimeout(this.config.timeoutMs);
+    await applySessionExtras(context, options);
+    const page = context.pages()[0] ?? (await context.newPage());
+    let closed = false;
+    const session = new BrowserSession(context, page, async () => {
+      if (closed) return;
+      closed = true;
+      try {
+        await context.close();
+      } finally {
+        await unlock();
+        this.free(name);
+      }
+    });
+    this.held.set(name, {
+      owner,
+      session
+    });
+    return session;
+  }
+}
+/**
+ * Marks a profile as used by this process, refusing one a live process
+ * holds and taking over one a dead process left behind.
+ *
+ * @param path - The profile directory.
+ * @param name - The profile name, for the message.
+ * @returns The unlock.
+ * @throws Error when another crawler holds the profile.
+ */
+async function lockProfile(path, name) {
+  const file = join$1(path, LOCK_FILE);
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try {
+      await writeFile(file, String(process.pid), {
+        flag: 'wx'
+      });
+      return async () => {
+        await rm(file, {
+          force: true
+        });
+      };
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      const holder = Number(await readHolder(file));
+      if (Number.isSafeInteger(holder) && holder > 0 && isAlive(holder)) throw new Error(`browser profile "${name}" is open in another browser (${path}, process ${holder}); close it or use another profile`, {
+        cause: error
+      });
+      await rm(file, {
+        force: true
+      });
+    }
+  }
+  throw new Error(`browser profile "${name}": cannot lock ${path}`);
+}
+/** The unlock of a profile that was never locked. */
+async function unlocked() {}
+async function readHolder(file) {
+  try {
+    return await readFile(file, 'utf8');
+  } catch {
+    return '';
+  }
+}
+function isAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error.code === 'EPERM';
+  }
+}
+
+/**
+ * Disposes a runner, ignoring a failure: a tab whose browser already went
+ * away (a rotation, a crash) has nothing left to close.
+ *
+ * @param runner - The runner.
+ */
+async function disposeQuietly(runner) {
+  try {
+    await runner.dispose();
+  } catch {
+    // already gone
+  }
+}
+
 /**
  * Runs a body once per item of a list (`over`), or once per live element
  * matching `selector`, each in a fresh child scope with the item bound under
  * `as`; emits a record per iteration when asked.
  *
- * With a concurrent gate, iterations run as permits allow and records come
- * out in completion order; without one, in list order.
+ * With a concurrent gate, iterations of a list run as permits allow and
+ * records come out in completion order; without one, in list order. In web
+ * mode each parallel iteration runs in a tab of its own (`runner.fork`); a
+ * loop over live elements stays sequential, since its elements live on one page.
  *
  * @param step - The forEach step.
  * @param scope - The scope the list lives in.
@@ -1264,7 +1480,7 @@ class BrowserClient {
 async function runForEach(step, scope, walk) {
   const items = await itemsOf$1(step, scope, walk);
   const gate = walk.gate;
-  if (gate?.concurrent === true) return runPooled(step, scope, walk, items, gate);
+  if (gate?.concurrent === true && step.selector === undefined && (walk.recipe.mode === 'api' || walk.runner.fork !== undefined)) return runPooled(step, scope, walk, items, gate);
   for (const item of items) {
     if ((await runIteration(step, scope, walk, item)) === 'stop') return 'stop';
   }
@@ -1286,17 +1502,21 @@ async function runPooled(step, scope, walk, items, gate) {
   let stopped = false;
   let failure;
   const tasks = [];
-  const overrides = {
-    gate: gate.nested()
-  };
+  const nested = gate.nested();
   const iterate = async (item, release) => {
+    let runner;
     try {
-      if ((await runIteration(step, scope, walk, item, overrides)) === 'stop') stopped = true;
+      runner = walk.runner.fork === undefined ? walk.runner : await walk.runner.fork();
+      if ((await runIteration(step, scope, walk, item, {
+        gate: nested,
+        runner
+      })) === 'stop') stopped = true;
     } catch (error) {
       failure ??= {
         error
       };
     } finally {
+      if (runner !== undefined && runner !== walk.runner) await disposeQuietly(runner);
       release();
     }
   };
@@ -1604,6 +1824,7 @@ function logThrough(walk) {
 class RunGate {
   permits;
   minIntervalMs;
+  hosts;
   shared;
   inFlight = 0;
   waiting = [];
@@ -1611,11 +1832,13 @@ class RunGate {
   /**
    * @param permits - Iterations allowed in flight; 1 is sequential.
    * @param minIntervalMs - Minimum time between two request starts across the run.
+   * @param hosts - The crawler's per-site throttle, shared with every other recipe.
    * @param shared - The throttle state to share (internal: `nested` gates keep their parent's).
    */
-  constructor(permits, minIntervalMs, shared) {
+  constructor(permits, minIntervalMs, hosts, shared) {
     this.permits = permits;
     this.minIntervalMs = minIntervalMs;
+    this.hosts = hosts;
     this.shared = shared;
   }
   /** Whether this gate lets more than one iteration run at once. */
@@ -1652,11 +1875,258 @@ class RunGate {
     state.lastStart = at;
     await sleep(at - now);
   }
+  /**
+   * Waits until a request to `url` may start: the recipe's interval, then its
+   * site's turn in the crawler's per-site throttle.
+   *
+   * @param url - Where the request goes.
+   * @returns The release of the site's lane: call it once the response arrived or the request failed.
+   */
+  async request(url) {
+    await this.throttle();
+    return this.hosts === undefined ? noop$1 : this.hosts.slot(url);
+  }
   /** The gate for a body running inside an iteration that holds a permit: sequential, same throttle. */
   nested() {
-    return new RunGate(1, this.minIntervalMs, this.shared ?? this);
+    return new RunGate(1, this.minIntervalMs, this.hosts, this.shared ?? this);
+  }
+}
+function noop$1() {}
+
+/**
+ * Spaces and bounds requests per site, shared by every recipe of a crawler,
+ * so two recipes (or two parallel iterations) that hit one site add up to one
+ * polite client rather than two. A recipe's own `limits.delayMs` still applies
+ * on top, per recipe.
+ *
+ * Like a single-lane bridge with a traffic light: whoever arrives waits for
+ * the car ahead to be far enough, and for a free lane.
+ */
+class HostThrottle {
+  config;
+  buckets = new Map();
+  domains;
+  constructor(config = {}) {
+    this.config = config;
+    this.domains = Object.entries(config.domains ?? {}).map(([domain, rule]) => [domain.toLowerCase().replace(/^\.+/, ''), rule]).sort((first, second) => second[0].length - first[0].length);
+  }
+  bucketFor(url, always = false) {
+    const host = hostOf(url);
+    if (host === undefined) return undefined;
+    const match = this.domains.find(([domain]) => host === domain || host.endsWith(`.${domain}`));
+    const key = match?.[0] ?? host;
+    let bucket = this.buckets.get(key);
+    if (bucket === undefined) {
+      const rule = {
+        ...pick(this.config),
+        ...match?.[1]
+      };
+      if (!always && !hasLimit(rule)) return undefined;
+      bucket = {
+        rule,
+        inFlight: 0,
+        waiting: [],
+        lastStart: Promise.resolve(-Infinity),
+        pausedTo: 0
+      };
+      this.buckets.set(key, bucket);
+    }
+    return bucket;
+  }
+  /** Whether any rule can hold a request back. */
+  get active() {
+    return hasLimit(this.config) || this.domains.some(([, rule]) => hasLimit(rule)) || this.buckets.size > 0;
+  }
+  /**
+   * Waits until a request to `url` may start, then holds one of its site's
+   * lanes until the returned release is called.
+   *
+   * @param url - Where the request goes; anything but `http(s):` passes at once.
+   * @returns The release: call it once, when the response arrived or the request failed.
+   */
+  async slot(url) {
+    const bucket = this.bucketFor(url);
+    if (bucket === undefined) return noop;
+    const concurrency = bucket.rule.concurrency ?? Infinity;
+    if (bucket.inFlight >= concurrency) await new Promise(resolve => {
+      bucket.waiting.push(resolve);
+    });
+    bucket.inFlight += 1;
+    const previous = bucket.lastStart;
+    const turn = (async () => {
+      const after = await previous;
+      // A pause may arrive while waiting (a Retry-After), so check it again after each sleep.
+      for (;;) {
+        const now = Date.now();
+        const at = Math.max(now, after + (bucket.rule.delayMs ?? 0), bucket.pausedTo);
+        if (at <= now) return now;
+        await sleep(at - now);
+      }
+    })();
+    bucket.lastStart = turn;
+    await turn;
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      bucket.inFlight -= 1;
+      bucket.waiting.shift()?.();
+    };
+  }
+  /**
+   * Holds every request to the site of `url` back until `untilMs` (a `Retry-After`).
+   *
+   * @param url - A URL of the site.
+   * @param untilMs - An epoch time.
+   */
+  pause(url, untilMs) {
+    const bucket = this.bucketFor(url, true);
+    if (bucket !== undefined) bucket.pausedTo = Math.max(bucket.pausedTo, untilMs);
+  }
+}
+function noop() {}
+function hostOf(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? parsed.hostname.toLowerCase() : undefined;
+  } catch {
+    return undefined;
+  }
+}
+function pick(config) {
+  return {
+    delayMs: config.delayMs,
+    concurrency: config.concurrency
+  };
+}
+function hasLimit(rule) {
+  return (rule.delayMs ?? 0) > 0 || rule.concurrency !== undefined;
+}
+
+/** Statuses a server uses for "not now": timeout, too early, too many requests, and the 5xx that pass. */
+const RETRY_STATUSES = [408, 425, 429, 500, 502, 503, 504];
+/** Three tries, one second then two apart, never a wait over 30 seconds. */
+const DEFAULT_RETRY_RULE = {
+  attempts: 3,
+  backoffMs: 1000,
+  maxDelayMs: 30_000,
+  statuses: [...RETRY_STATUSES]
+};
+/**
+ * Errors that say the connection failed rather than the site answered:
+ * resets, refusals, timeouts, a DNS lookup that could not run, a proxy that
+ * dropped the tunnel. A name that does not resolve at all (`ENOTFOUND`,
+ * `ERR_NAME_NOT_RESOLVED`) is not here: retrying a typo only wastes time.
+ */
+const TRANSIENT_ERROR = /ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|socket hang up|net::ERR_(?:CONNECTION_(?:RESET|REFUSED|CLOSED|ABORTED|TIMED_OUT)|TIMED_OUT|EMPTY_RESPONSE|NETWORK_CHANGED|INTERNET_DISCONNECTED|PROXY_CONNECTION_FAILED|TUNNEL_CONNECTION_FAILED|HTTP2_PROTOCOL_ERROR|NETWORK_IO_SUSPENDED)|NS_ERROR_NET_(?:RESET|INTERRUPT|TIMEOUT)|Timeout \d+ms exceeded/;
+/**
+ * The retry rule a recipe runs with: its `limits.retry` over the crawler's
+ * default over `DEFAULT_RETRY_RULE`.
+ *
+ * @param own - The recipe's `limits.retry`.
+ * @param crawler - The crawler's `retry` option.
+ * @returns The rule.
+ */
+function resolveRetryRule(own, crawler) {
+  return {
+    ...DEFAULT_RETRY_RULE,
+    ...definedOf(crawler),
+    ...definedOf(own)
+  };
+}
+/**
+ * Whether an error is a connection failure worth another try.
+ *
+ * @param error - What the request threw.
+ * @returns The reason, or `undefined`.
+ */
+function transientError(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const match = TRANSIENT_ERROR.exec(message);
+  return match === null ? undefined : {
+    reason: match[0]
+  };
+}
+/**
+ * How long to wait before attempt `attempt + 1`: the server's `Retry-After`
+ * when it gave one, else `backoffMs` doubling per attempt with a little
+ * jitter; `undefined` when the server asks for longer than `maxDelayMs` (it
+ * means "come back much later", which a crawl cannot wait for).
+ *
+ * @param rule - The retry rule.
+ * @param attempt - The attempt that just failed, from 1.
+ * @param retryAfter - The `Retry-After` header: seconds, or an HTTP date.
+ * @param now - The current time, for dates.
+ * @returns Milliseconds, or `undefined` for no retry.
+ */
+function retryDelay(rule, attempt, retryAfter, now = Date.now()) {
+  const asked = retryAfterMs(retryAfter, now);
+  if (asked !== undefined) return asked > rule.maxDelayMs ? undefined : asked;
+  const base = rule.backoffMs * 2 ** (attempt - 1);
+  const jittered = base * (0.75 + Math.random() * 0.5);
+  return Math.min(Math.round(jittered), rule.maxDelayMs);
+}
+/**
+ * Sends a request through the gate (the recipe's rate, the site's lane), and
+ * sends it again after a pause while it fails in a passing way, up to
+ * `rule.attempts` tries in all. A `Retry-After` holds back every request to
+ * that site, not only this one. Each retry is reported as `request:retry`.
+ *
+ * Like redialling a busy number: wait a moment, dial again, give up after a
+ * few tries; and if the other end said "call back in a minute", wait that minute.
+ *
+ * @param url - Where the request goes.
+ * @param attempt - How to send it and how to judge the outcome.
+ * @param context - The recipe, gate, events and rule.
+ * @returns What the last try gave.
+ * @throws What the last try threw.
+ */
+async function withTransportRetry(url, attempt, context) {
+  const {
+    rule
+  } = context;
+  for (let tries = 1;; tries += 1) {
+    const release = await context.gate.request(url);
+    let outcome;
+    try {
+      outcome = {
+        value: await attempt.run()
+      };
+    } catch (error) {
+      outcome = {
+        error
+      };
+    } finally {
+      release();
+    }
+    const transient = tries < rule.attempts ? attempt.problem(outcome) : undefined;
+    const delay = transient === undefined ? undefined : retryDelay(rule, tries, transient.retryAfter);
+    if (transient === undefined || delay === undefined) {
+      if ('error' in outcome) throw outcome.error;
+      return outcome.value;
+    }
+    if (transient.retryAfter !== undefined) context.gate.hosts?.pause(url, Date.now() + delay);
+    context.events.emit({
+      type: 'request:retry',
+      recipeId: context.recipeId,
+      url,
+      attempt: tries + 1,
+      reason: transient.reason,
+      delayMs: delay
+    });
+    await sleep(delay);
   }
 }
+function retryAfterMs(header, now) {
+  if (header === undefined || header.trim() === '') return undefined;
+  const seconds = Number(header.trim());
+  if (Number.isFinite(seconds)) return Math.max(0, Math.round(seconds * 1000));
+  const date = Date.parse(header);
+  return Number.isNaN(date) ? undefined : Math.max(0, date - now);
+}
+function definedOf(rule) {
+  return Object.fromEntries(Object.entries(rule ?? {}).filter(([, value]) => value !== undefined));
+}
 
 /** A block unless a recipe says otherwise: forbidden, rate limited, or an AWS WAF challenge (IMDb answers 202 with it). */
 const DEFAULT_BLOCK_RULE = {
@@ -2229,6 +2699,10 @@ function traceLine(event) {
       {
         return `${indent(1)}↻ new access lease (attempt ${event.attempt})`;
       }
+    case 'request:retry':
+      {
+        return `${indent(1)}↺ ${event.url}: ${event.reason}, try ${event.attempt} in ${event.delayMs} ms`;
+      }
     case 'captcha:detected':
       {
         return `${indent(1)}⚿ captcha ${event.kind} on ${event.url}`;
@@ -2437,26 +2911,36 @@ async function existingKeys(target) {
   });
 }
 
-/** Drops records whose key was already seen. First record wins; keyless records always pass. */
+/**
+ * Drops records whose key was already seen. First record wins; keyless
+ * records always pass. Recipes running in parallel each get their own view:
+ * under `recipe` scope they never see each other's keys, under `run` scope
+ * they share them (and whichever emits a key first keeps it).
+ */
 class DedupePolicy {
   scope;
-  seen = new Set();
+  shared = new Set();
   constructor(scope = 'run') {
     this.scope = scope;
   }
-  /** Called when an input recipe starts; forgets keys under `recipe` scope. */
-  startRecipe() {
-    if (this.scope === 'recipe') this.seen = new Set();
-  }
   /**
-   * @param record - A validated record.
-   * @returns `true` when the record repeats an earlier key and must be dropped.
+   * The de-duplication one input recipe run uses.
+   *
+   * @returns Its view: keys shared with the run, its own, or none checked.
    */
-  isDuplicate(record) {
-    if (this.scope === 'off' || record.key === null) return false;
-    if (this.seen.has(record.key)) return true;
-    this.seen.add(record.key);
-    return false;
+  forRecipe() {
+    if (this.scope === 'off') return {
+      isDuplicate: () => false
+    };
+    const seen = this.scope === 'run' ? this.shared : new Set();
+    return {
+      isDuplicate: record => {
+        if (record.key === null) return false;
+        if (seen.has(record.key)) return true;
+        seen.add(record.key);
+        return false;
+      }
+    };
   }
 }
 
@@ -2507,10 +2991,13 @@ const DOCUMENT = /^\s*(?:<!doctype|<html)/i;
  *
  * @param html - The markup (a whole document or a fragment).
  * @param selector - A CSS selector.
+ * @param xml - Read the markup as XML: names keep their case, `take: "html"` gives XML.
  * @returns Every match, in document order.
  */
-function selectHtml(html, selector) {
-  const api = DOCUMENT.test(html) ? load(html) : load(html, undefined, false);
+function selectHtml(html, selector, xml = false) {
+  const api = xml ? load(html, {
+    xml: true
+  }) : DOCUMENT.test(html) ? load(html) : load(html, undefined, false);
   return api(selector).map((_, element) => ({
     api,
     element: api(element)
@@ -3557,8 +4044,8 @@ function filledGrid(sheet) {
 /** `B10:B13` as 0-based bounds; `undefined` for anything else. */
 function rangeOf(reference) {
   const [from, to = from] = reference.split(':', 2);
-  const start = cellOf(from);
-  const end = cellOf(to);
+  const start = cellOf$1(from);
+  const end = cellOf$1(to);
   if (start === undefined || end === undefined) return undefined;
   return {
     top: Math.min(start.row, end.row),
@@ -3567,7 +4054,7 @@ function rangeOf(reference) {
     right: Math.max(start.column, end.column)
   };
 }
-function cellOf(reference) {
+function cellOf$1(reference) {
   const match = /^\$?([A-Z]+)\$?(\d+)$/i.exec(reference.trim());
   if (match === null) return undefined;
   const letters = match[1].toUpperCase();
@@ -3700,13 +4187,194 @@ function shapeTables(document, slides, query) {
   }));
 }
 
+/**
+ * Whether a value is a read XML document.
+ *
+ * @param value - Anything bound in a scope.
+ * @returns `true` for an `XmlDocument`.
+ */
+function isXmlDocument(value) {
+  return typeof value === 'object' && value !== null && value.kind === 'xml' && typeof value.xml === 'string';
+}
+
+/** The documents already parsed, by the text they came from, so a recipe's extracts parse a body once. */
+const parsed = new Map();
+const PARSED_LIMIT = 16;
+/**
+ * Parses XML strictly but safely: a malformed document is an error naming
+ * the URL and the line; entities a DOCTYPE declares are never expanded (no
+ * "billion laughs") and nothing external is ever fetched (no XXE), they stay
+ * as they are.
+ *
+ * @param xml - The document text.
+ * @param where - The URL or id, for messages.
+ * @returns The document.
+ * @throws Error when the text is not well-formed XML.
+ */
+function parseXml(xml, where) {
+  const cached = parsed.get(xml);
+  if (cached !== undefined) return cached;
+  let document;
+  try {
+    document = new DOMParser({
+      onError: failOnFatal
+    }).parseFromString(xml, 'text/xml');
+  } catch (error) {
+    throw new Error(`${where}: not well-formed XML (${firstLine(error)})`, {
+      cause: error
+    });
+  }
+  if (document.documentElement === null) throw new Error(`${where}: not an XML document (no root element)`);
+  remember(xml, document);
+  return document;
+}
+/**
+ * HTML as an XML document, for XPath on fetched pages: parsed the way a
+ * browser parses it (a forgiving HTML5 parser that inserts `<tbody>` and
+ * closes what the page left open), without namespaces, so `//table/tbody/tr`
+ * finds what it finds on the live page.
+ *
+ * @param html - A page or a fragment.
+ * @returns The document.
+ */
+function htmlAsXml(html) {
+  const cached = parsed.get(html);
+  if (cached !== undefined) return cached;
+  const whole = /^\s*(?:<!doctype|<html)/i.test(html);
+  // A fragment (a row taken with `take: "html"`) keeps its cells, and gets one root to be a document.
+  const xml = whole ? load(html).xml() : `<fragment>${load(html, undefined, false).xml()}</fragment>`;
+  const document = withoutNamespaces(new DOMParser({
+    onError: ignore
+  }).parseFromString(xml, 'text/xml'));
+  remember(html, document);
+  return document;
+}
+/**
+ * A copy of a document with every namespace dropped: elements and attributes
+ * keep their local names, `xmlns` declarations go. What `ignoreNamespaces`
+ * queries run on: `//entry/title` instead of `//atom:entry/atom:title`.
+ *
+ * @param document - A parsed document.
+ * @returns The copy.
+ */
+function withoutNamespaces(document) {
+  const copy = new DOMImplementation().createDocument(null, '', null);
+  const clone = node => {
+    if (node.nodeType === node.ELEMENT_NODE) {
+      const element = node;
+      const target = copy.createElement(element.localName);
+      for (const attribute of element.attributes) {
+        if (attribute.name === 'xmlns' || attribute.prefix === 'xmlns') continue;
+        target.setAttribute(attribute.localName, attribute.value);
+      }
+      for (const child of element.childNodes) {
+        const cloned = clone(child);
+        if (cloned !== undefined) target.insertBefore(cloned, null);
+      }
+      return target;
+    }
+    if (node.nodeType === node.TEXT_NODE || node.nodeType === node.CDATA_SECTION_NODE) return copy.createTextNode(node.nodeValue ?? '');
+    return undefined;
+  };
+  const root = document.documentElement === null ? undefined : clone(document.documentElement);
+  if (root !== undefined) copy.insertBefore(root, null);
+  return copy;
+}
+/**
+ * A node as markup: the element with its children (`outer`), or its children only.
+ *
+ * @param node - A node.
+ * @param outer - Whether to include the node itself.
+ * @returns The markup.
+ */
+function serializeXml(node, outer) {
+  const serializer = new XMLSerializer();
+  if (outer) return serializer.serializeToString(node);
+  return Array.from(node.childNodes, child => serializer.serializeToString(child)).join('');
+}
+/**
+ * The text of an XML body, gunzipped first when it is gzip (a `sitemap.xml.gz`).
+ *
+ * @param bytes - The body.
+ * @returns The bytes to decode.
+ */
+function gunzipIfNeeded(bytes) {
+  return bytes.length > 2 && bytes[0] === 0x1F && bytes[1] === 0x8B ? gunzipSync(bytes) : bytes;
+}
+function remember(text, document) {
+  if (parsed.size >= PARSED_LIMIT) parsed.delete(parsed.keys().next().value);
+  parsed.set(text, document);
+}
+function failOnFatal(level, message) {
+  if (level === 'fatalError') throw new Error(message);
+}
+function ignore() {}
+function firstLine(error) {
+  return (error instanceof Error ? error.message : String(error)).split('\n', 1)[0];
+}
+
+/**
+ * Runs an XPath 1.0 query on a parsed document. Prefixes the root element
+ * declares are known without being listed; a default namespace has no
+ * prefix, so name one in `namespaces` or use `ignoreNamespaces`.
+ *
+ * @param document - The document.
+ * @param expression - The query.
+ * @param options - Namespaces.
+ * @returns Every node selected, or the one value a function returned.
+ * @throws Error naming the query when it does not parse or uses an unknown prefix.
+ */
+function selectXpath(document, expression, options = {}) {
+  const target = options.ignoreNamespaces === true ? withoutNamespaces(document) : document;
+  const namespaces = {
+    ...declaredPrefixes(target),
+    ...options.namespaces
+  };
+  let result;
+  try {
+    result = useNamespaces(namespaces)(expression, target);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const hint = /prefix|namespace|QName/i.test(message) ? ' (declare it in "namespaces", or set "ignoreNamespaces": true)' : '';
+    throw new Error(`xpath ${expression}: ${message}${hint}`, {
+      cause: error
+    });
+  }
+  if (Array.isArray(result)) return result;
+  if (result === null) return [];
+  return [result];
+}
+/**
+ * The value of an XPath result.
+ *
+ * @param value - A node or a function result.
+ * @param take - `text` (whitespace collapsed), `html` (the node's inner markup), `json` (its outer markup), `value` (its text as is), `attr:<name>`.
+ * @returns The value; `undefined` for a missing attribute.
+ */
+function takeFromXml(value, take) {
+  if (typeof value !== 'object') return take === 'text' ? collapse(String(value)) : value;
+  if (take === 'text') return collapse(value.textContent ?? '');
+  if (take === 'value') return value.textContent ?? '';
+  if (take === 'html') return serializeXml(value, false);
+  if (take === 'json') return serializeXml(value, true);
+  if (value.nodeType !== value.ELEMENT_NODE) return undefined;
+  const element = value;
+  const name = take.slice('attr:'.length);
+  return element.hasAttribute(name) ? element.getAttribute(name) : undefined;
+}
+function declaredPrefixes(document) {
+  const root = document.documentElement;
+  if (root === null) return {};
+  return Object.fromEntries([...root.attributes].filter(attribute => attribute.prefix === 'xmlns').map(attribute => [attribute.localName, attribute.value]));
+}
+
 /**
  * Runs an `extract` step against a static document: the value bound under
- * `from`, else the scope's current document. `css` reads HTML, `jsonpath`
- * reads JSON (or a read PDF, workbook or deck as data), `table` reads the
- * tables of a PDF, a workbook (a spreadsheet, a CSV), a deck (a presentation)
- * or HTML (its `<table>`s), `regex` reads any document as text; `xpath` needs
- * a live page and is refused here.
+ * `from`, else the scope's current document. `css` reads HTML (or XML),
+ * `xpath` reads XML (a feed, a sitemap) or HTML parsed as a browser parses
+ * it, `jsonpath` reads JSON (or a read PDF, workbook or deck as data), `table`
+ * reads the tables of a PDF, a workbook (a spreadsheet, a CSV), a deck (a
+ * presentation) or HTML (its `<table>`s), `regex` reads any document as text.
  *
  * A `jsonpath` extract whose `from` is text parses that text as JSON, and a
  * list of texts (every `<script type="application/ld+json">` of a page) becomes
@@ -3725,6 +4393,7 @@ function extractFromDocument(step, scope) {
   switch (step.kind) {
     case 'jsonpath':
       {
+        if (document.kind === 'xml') throw new Error('jsonpath needs a JSON document; the current document is xml (read it with kind "xpath")');
         if (document.kind === 'html' || document.kind === 'text') throw new Error(`jsonpath needs a JSON document; the current document is ${document.kind}`);
         values = selectJson(document.kind === 'json' ? document.data : document, selector).map(node => takeFromJson(node, take));
         break;
@@ -3736,10 +4405,22 @@ function extractFromDocument(step, scope) {
       }
     case 'css':
       {
+        if (document.kind === 'xml') {
+          values = selectHtml(document.xml, selector, true).map(match => takeFromHtml(match, take));
+          break;
+        }
         if (document.kind !== 'html') throw new Error(`css needs an HTML document; the current document is ${document.kind}${['workbook', 'pdf', 'deck'].includes(document.kind) ? ' (read it with kind "table", "regex" or "jsonpath")' : ''}`);
         values = selectHtml(document.html, selector).map(match => takeFromHtml(match, take));
         break;
       }
+    case 'xpath':
+      {
+        values = selectXpath(xpathDocument(document), selector, {
+          namespaces: step.namespaces,
+          ignoreNamespaces: step.ignoreNamespaces
+        }).map(node => takeFromXml(node, take));
+        break;
+      }
     case 'regex':
       {
         values = selectRegex(textOf$1(document), selector);
@@ -3747,7 +4428,7 @@ function extractFromDocument(step, scope) {
       }
     default:
       {
-        throw new Error('xpath works on a live page only; use css on fetched HTML');
+        throw new Error(`unknown extract kind ${String(step.kind)}`);
       }
   }
   if (step.many === true) {
@@ -3853,9 +4534,32 @@ function patternOf(source, where) {
     });
   }
 }
+/** What an xpath extract queries: XML as it is, HTML parsed the way a browser would. */
+function xpathDocument(document) {
+  if (document.kind === 'xml') return parseXml(document.xml, 'the document');
+  if (document.kind === 'html') return htmlAsXml(document.html);
+  if (document.kind === 'text') return parseXml(document.text, 'the document');
+  throw new Error(`xpath reads XML or HTML; the current document is ${document.kind} (read it with kind ${document.kind === 'json' ? '"jsonpath"' : '"table", "regex" or "jsonpath"'})`);
+}
+/** Markup bound to an id: XML when it is well-formed (a feed entry taken with `take: "json"`), else HTML. */
+function markupDocument(markup) {
+  try {
+    parseXml(markup, 'markup');
+    return {
+      kind: 'xml',
+      xml: markup
+    };
+  } catch {
+    return {
+      kind: 'html',
+      html: markup
+    };
+  }
+}
 /** The text a regex extract reads: markup, text, a PDF's or a workbook's rows, or JSON re-serialised (a list of texts joined by newlines). */
 function textOf$1(document) {
   if (document.kind === 'html') return document.html;
+  if (document.kind === 'xml') return document.xml;
   if (document.kind === 'text') return document.text;
   if (document.kind === 'pdf') return pdfText(document);
   if (document.kind === 'workbook') return workbookText(document);
@@ -3871,7 +4575,11 @@ function documentFor(step, scope) {
   }
   const source = scope.get(step.from);
   if (source === undefined) throw new Error(`"${step.from}" is not bound`);
-  if (isPdfDocument(source) || isWorkbookDocument(source) || isDeckDocument(source)) return source;
+  if (isPdfDocument(source) || isWorkbookDocument(source) || isDeckDocument(source) || isXmlDocument(source)) return source;
+  if (step.kind === 'xpath') {
+    if (typeof source !== 'string') throw new Error(`"${step.from}" is not markup; xpath reads XML or HTML text`);
+    return markupDocument(source);
+  }
   if (step.kind === 'table') throw new Error(`"${step.from}" is not a PDF, a workbook or a deck; request it with "as": "pdf", "csv", "xlsx" or "pptx"`);
   if (step.kind === 'regex') {
     if (typeof source === 'string') return {
@@ -3960,6 +4668,12 @@ async function readYaml(text, source, scalars = 'typed') {
   };
 }
 
+/**
+ * `<` as the JSON escape `\u003c`, so a front matter value holding `</script>`
+ * cannot close its element. Built from char codes: a formatter would turn a
+ * literal escape back into `<`.
+ */
+const ESCAPED_LESS_THAN = String.fromCodePoint(92, 117, 48, 48, 51, 99);
 /** A leading `---` block of YAML. */
 const FRONT_MATTER = /^---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/;
 /**
@@ -3993,14 +4707,22 @@ async function readMarkdown(text, source) {
     async: false
   });
   const sections = sectioned(rendered);
-  const head = front === undefined ? '' : `<script type="application/json" data-front-matter>${JSON.stringify(front.data ?? null).replaceAll('<', '<')}</script>`;
+  const head = front === undefined ? '' : `<script type="application/json" data-front-matter>${JSON.stringify(front.data ?? null).replaceAll('<', () => ESCAPED_LESS_THAN)}</script>`;
   return {
     html: `<!doctype html><html><head>${head}</head><body>${sections}</body></html>`,
     frontMatter: front?.data,
     warnings: front?.warnings ?? []
   };
 }
-/** Wraps each heading and what follows it, up to the next heading of the same or a higher level, in a section. */
+/**
+ * Wraps each heading and what follows it, up to the next heading of the same
+ * or a higher level, in `<section data-heading="…" data-level="…">`, sections
+ * nesting; headings get slug ids. Rendered Markdown and Word documents both go
+ * through it, so one selector finds "the table under *Prezzi*" in either.
+ *
+ * @param html - Top-level HTML: headings among paragraphs, lists, tables.
+ * @returns The same content, sectioned.
+ */
 function sectioned(html) {
   const $ = load(html, null, false);
   const open = [];
@@ -4021,7 +4743,7 @@ function sectioned(html) {
     const heading = $(node);
     const text = heading.text().replaceAll(/\s+/g, ' ').trim();
     heading.attr('id', uniqueSlug(text, used));
-    out += `<section data-heading="${escapeAttribute(text)}" data-level="${depth}">${$.html(node)}`;
+    out += `<section data-heading="${escapeAttribute$1(text)}" data-level="${depth}">${$.html(node)}`;
     open.push(depth);
   }
   return out + '</section>'.repeat(open.length);
@@ -4033,10 +4755,165 @@ function uniqueSlug(text, used) {
   used.set(slug, count + 1);
   return count === 0 ? slug : `${slug}-${count}`;
 }
-function escapeAttribute(text) {
+function escapeAttribute$1(text) {
   return text.replaceAll('&', '&amp;').replaceAll('"', '&quot;').replaceAll('<', '&lt;');
 }
 
+/**
+ * Reads a `.docx` Word document into HTML, through `@opencraw/office-reader`,
+ * so every selector that reads a page reads a Word document too:
+ *
+ * - headings become `<h1>`…`<h6>` inside `<section data-heading="…">`, as in
+ *   rendered Markdown;
+ * - list items become nested `<ul>` / `<ol>`;
+ * - tables become `<table>`s with their merged cells as `colspan` / `rowspan`,
+ *   so a `table` extract reads them like any HTML table;
+ * - a paragraph keeps its style as `data-style` and its links as `<a href>`;
+ * - headers, footers and notes follow the body, in `<header>`, `<footer>`
+ *   and `<aside data-part="notes">`; the title goes to `<title>`.
+ *
+ * The reader is imported on first use.
+ *
+ * @param bytes - The file.
+ * @param source - Where it came from, for messages.
+ * @returns The HTML document.
+ * @throws Error naming the source, and saying what to do, for a file that is
+ * not a readable Word document (a legacy `.doc`, a password-protected file, an `.odt`…).
+ */
+async function readDocxHtml(bytes, source) {
+  const {
+    readDocx,
+    OfficeReadError
+  } = await import('@opencraw/office-reader/docx');
+  let document;
+  try {
+    document = await readDocx(bytes);
+  } catch (error) {
+    if (error instanceof OfficeReadError) throw new Error(`${source}: ${error.message}`, {
+      cause: error
+    });
+    throw error;
+  }
+  const head = document.title === undefined ? '' : `<title>${escapeText(document.title)}</title>`;
+  const headers = document.headers.map(blocks => `<header data-part="header">${blocksHtml(blocks)}</header>`).join('');
+  const footers = document.footers.map(blocks => `<footer data-part="footer">${blocksHtml(blocks)}</footer>`).join('');
+  const notes = document.notes.length === 0 ? '' : `<aside data-part="notes"><ol>${document.notes.map(note => `<li id="${note.kind}-${escapeAttribute(note.id)}" data-kind="${note.kind}">${textHtml(note.text)}</li>`).join('')}</ol></aside>`;
+  return `<!doctype html><html><head>${head}</head><body>${headers}${sectioned(blocksHtml(document.body))}${footers}${notes}</body></html>`;
+}
+/** Blocks as HTML: headings, paragraphs, nested lists, tables. */
+function blocksHtml(blocks) {
+  let html = '';
+  const lists = [];
+  const closeLists = depth => {
+    while (lists.length > depth) {
+      const list = lists.pop();
+      html += `${list?.itemOpen === true ? '</li>' : ''}</${list?.tag ?? 'ul'}>`;
+    }
+  };
+  for (const block of blocks) {
+    if (block.kind === 'table') {
+      closeLists(0);
+      html += tableHtml(block);
+      continue;
+    }
+    if (block.list === undefined) {
+      closeLists(0);
+      html += paragraphHtml(block);
+      continue;
+    }
+    const tag = block.list.ordered ? 'ol' : 'ul';
+    const depth = block.list.level + 1;
+    closeLists(depth);
+    if (lists.length === depth && lists.at(-1)?.tag !== tag) closeLists(depth - 1);
+    while (lists.length < depth) {
+      html += `<${tag}>`;
+      lists.push({
+        tag,
+        itemOpen: false
+      });
+    }
+    const list = lists[depth - 1];
+    html += `${list.itemOpen ? '</li>' : ''}<li${styleAttribute(block)}>${inlineHtml(block)}`;
+    list.itemOpen = true;
+  }
+  closeLists(0);
+  return html;
+}
+function paragraphHtml(paragraph) {
+  if (paragraph.heading !== undefined) {
+    const level = Math.min(paragraph.heading, 6);
+    return `<h${level}${styleAttribute(paragraph)}>${inlineHtml(paragraph)}</h${level}>`;
+  }
+  return `<p${styleAttribute(paragraph)}>${inlineHtml(paragraph)}</p>`;
+}
+/** A paragraph's text with its links as anchors, in the order they appear. */
+function inlineHtml(paragraph) {
+  let html = '';
+  let rest = paragraph.text;
+  const links = paragraph.links ?? [];
+  for (const link of links) {
+    const at = rest.indexOf(link.text);
+    if (at === -1) continue;
+    html += `${textHtml(rest.slice(0, at))}<a href="${escapeAttribute(link.href)}">${textHtml(link.text)}</a>`;
+    rest = rest.slice(at + link.text.length);
+  }
+  return html + textHtml(rest);
+}
+/** A grid as a table: the top-left cell of a merged range spans it, the cells it covers are left out. */
+function tableHtml(table) {
+  const spans = new Map();
+  const covered = new Set();
+  for (const range of table.merges) {
+    const [from, to] = range.split(':').map(reference => cellOf(reference));
+    if (from === undefined || to === undefined) continue;
+    spans.set(`${from.row},${from.column}`, {
+      rows: to.row - from.row + 1,
+      columns: to.column - from.column + 1
+    });
+    for (let row = from.row; row <= to.row; row += 1) {
+      for (let column = from.column; column <= to.column; column += 1) {
+        if (row !== from.row || column !== from.column) covered.add(`${row},${column}`);
+      }
+    }
+  }
+  const rows = table.rows.map((cells, row) => {
+    const html = cells.map((text, column) => {
+      const key = `${row},${column}`;
+      if (covered.has(key)) return '';
+      const span = spans.get(key);
+      const attributes = span === undefined ? '' : `${span.rows > 1 ? ` rowspan="${span.rows}"` : ''}${span.columns > 1 ? ` colspan="${span.columns}"` : ''}`;
+      return `<td${attributes}>${textHtml(text)}</td>`;
+    }).join('');
+    return `<tr>${html}</tr>`;
+  }).join('');
+  return `<table data-name="${escapeAttribute(table.name)}">${rows}</table>`;
+}
+/** An A1 reference as 0-based row and column. */
+function cellOf(reference) {
+  const match = /^([A-Z]+)(\d+)$/.exec(reference);
+  if (match === null) return undefined;
+  let column = 0;
+  const letters = match[1];
+  for (const letter of letters) column = column * 26 + (letter.codePointAt(0) ?? 64) - 64;
+  return {
+    row: Number(match[2]) - 1,
+    column: column - 1
+  };
+}
+function styleAttribute(paragraph) {
+  return paragraph.style === undefined ? '' : ` data-style="${escapeAttribute(paragraph.style)}"`;
+}
+/** Text as HTML: escaped, line breaks as `<br>`. */
+function textHtml(text) {
+  return escapeText(text).replaceAll('\n', '<br>');
+}
+function escapeText(text) {
+  return text.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
+}
+function escapeAttribute(text) {
+  return escapeText(text).replaceAll('"', '&quot;');
+}
+
 /** A response with a 4xx or 5xx status. */
 class HttpError extends Error {
   status;
@@ -4196,7 +5073,7 @@ class HttpClient {
 }
 async function readBody(response, httpRequest) {
   const contentType = response.headers()['content-type'] ?? '';
-  const format = httpRequest.as ?? formatFromContentType(contentType);
+  const format = httpRequest.as ?? formatFromContentType(contentType, response.url());
   return parseBody(format, await response.body(), response.url(), {
     ...httpRequest,
     charset: charsetOf(contentType)
@@ -4214,7 +5091,7 @@ async function readLocalFile(httpRequest) {
     body,
     warnings,
     format
-  } = await parseBody(httpRequest.as ?? formatFromExtension(extname(path)), bytes, httpRequest.url, httpRequest);
+  } = await parseBody(httpRequest.as ?? formatFromExtension(path), bytes, httpRequest.url, httpRequest);
   return {
     status: 200,
     url: httpRequest.url,
@@ -4244,6 +5121,35 @@ async function parseBody(format, bytes, url, reading) {
       format
     };
   }
+  if (format === 'xml') {
+    // A sitemap is often served gzipped as a file (`sitemap.xml.gz`), not as a compressed response.
+    const {
+      text
+    } = decodeText(gunzipIfNeeded(bytes), reading);
+    try {
+      parseXml(text, url);
+    } catch (error) {
+      throw new Error(`${error.message}${/^\s*(?:<!doctype html|<html)/i.test(text) ? '; the body is HTML: read it with "as": "html"' : ''}`, {
+        cause: error
+      });
+    }
+    return {
+      body: {
+        kind: 'xml',
+        xml: text
+      },
+      warnings: [],
+      format
+    };
+  }
+  if (format === 'docx') return {
+    body: {
+      kind: 'html',
+      html: await readDocxHtml(bytes, url)
+    },
+    warnings: [],
+    format
+  };
   if (format === 'markdown') {
     const {
       text
@@ -4309,24 +5215,31 @@ function looksLikeJsonLines(text) {
   const first = parseJsonLike(lines[0]);
   return 'value' in first;
 }
-function formatFromContentType(contentType) {
+function formatFromContentType(contentType, url) {
   const type = contentType.toLowerCase().split(';', 1)[0].trim();
+  // A gzipped sitemap is served as an archive; its name says what is inside.
+  if (GZIP_TYPES.has(type) && /\.xml\.gz$/i.test(new URL(url).pathname)) return 'xml';
   if (CSV_TYPES.has(type)) return 'csv';
   if (JSON_LINES_TYPES.has(type)) return 'jsonl';
   // A legacy .xls or .ppt goes to the Office reader too, which says what to do with it.
   if (type.includes('spreadsheetml') || type.startsWith('application/vnd.ms-excel')) return 'xlsx';
   if (type.includes('presentationml') || type.startsWith('application/vnd.ms-powerpoint')) return 'pptx';
+  if (type === 'application/msword' || type.includes('wordprocessingml') || type.startsWith('application/vnd.ms-word')) return 'docx';
   if (YAML_TYPES.has(type)) return 'yaml';
   if (type === 'text/markdown' || type === 'text/x-markdown') return 'markdown';
   if (type.includes('json')) return 'json';
   if (type.includes('pdf')) return 'pdf';
-  if (type.includes('html') || type.includes('xml')) return 'html';
+  if (type.includes('html')) return 'html';
+  if (type.includes('xml')) return 'xml';
   return 'text';
 }
 const JSON_LINES_TYPES = new Set(['application/x-ndjson', 'application/ndjson', 'application/jsonl', 'application/x-jsonlines', 'application/jsonlines']);
 const YAML_TYPES = new Set(['application/yaml', 'application/x-yaml', 'text/yaml', 'text/x-yaml']);
+const GZIP_TYPES = new Set(['application/gzip', 'application/x-gzip', 'application/octet-stream']);
 const CSV_TYPES = new Set(['text/csv', 'application/csv', 'text/x-csv', 'application/x-csv', 'text/comma-separated-values', 'text/tab-separated-values']);
-function formatFromExtension(extension) {
+function formatFromExtension(path) {
+  if (/\.xml\.gz$/i.test(path)) return 'xml';
+  const extension = extname(path);
   const formats = {
     '.json': 'json',
     '.jsonl': 'jsonl',
@@ -4341,21 +5254,30 @@ function formatFromExtension(extension) {
     '.pptm': 'pptx',
     '.ppsx': 'pptx',
     '.ppt': 'pptx',
+    '.docx': 'docx',
+    '.docm': 'docx',
+    '.dotx': 'docx',
+    '.doc': 'docx',
     '.yaml': 'yaml',
     '.yml': 'yaml',
     '.md': 'markdown',
     '.markdown': 'markdown',
     '.html': 'html',
     '.htm': 'html',
-    '.xml': 'html'
+    '.xml': 'xml',
+    '.rss': 'xml',
+    '.atom': 'xml',
+    '.kml': 'xml',
+    '.gpx': 'xml'
   };
   return formats[extension.toLowerCase()] ?? 'text';
 }
 
 /**
  * Sends a `request` step: renders its templates, waits for the gate's throttle,
- * sends, checks the response against the recipe's block rule, then binds it as
- * the scope's current document (and under the step id).
+ * sends (again, after a pause, while it fails in passing: `limits.retry`),
+ * checks the response against the recipe's block rule, then binds it as the
+ * scope's current document (and under the step id).
  *
  * @param step - The request step.
  * @param scope - The scope to render in and bind into.
@@ -4368,20 +5290,29 @@ function formatFromExtension(extension) {
 async function sendRequest(step, scope, client, recipe, gate, events) {
   const lookup = path => scope.lookup(path);
   const url = resolveUrl(renderText(step.url, lookup), scope.pageState?.url);
-  await gate.throttle();
+  const request = {
+    method: step.method,
+    url,
+    query: step.query === undefined ? undefined : renderMap(step.query, lookup),
+    headers: step.headers === undefined ? undefined : renderMap(step.headers, lookup),
+    body: renderDeep(step.body, lookup),
+    as: step.as,
+    encoding: step.encoding,
+    delimiter: step.delimiter,
+    scalars: step.scalars,
+    timeoutMs: recipe.limits?.timeoutMs
+  };
+  const rule = resolveRetryRule(recipe.limits?.retry);
   let response;
   try {
-    response = await client.send({
-      method: step.method,
-      url,
-      query: step.query === undefined ? undefined : renderMap(step.query, lookup),
-      headers: step.headers === undefined ? undefined : renderMap(step.headers, lookup),
-      body: renderDeep(step.body, lookup),
-      as: step.as,
-      encoding: step.encoding,
-      delimiter: step.delimiter,
-      scalars: step.scalars,
-      timeoutMs: recipe.limits?.timeoutMs
+    response = await withTransportRetry(url, {
+      run: () => client.send(request),
+      problem: outcome => 'error' in outcome ? problemOf(outcome.error, rule.statuses) : undefined
+    }, {
+      recipeId: recipe.id,
+      gate,
+      events,
+      rule
     });
   } catch (error) {
     if (!(error instanceof HttpError)) throw error;
@@ -4431,6 +5362,14 @@ async function sendRequest(step, scope, client, recipe, gate, events) {
   });
   if (step.id !== undefined) scope.set(step.id, documentValue(response.body));
 }
+/** A retry status (with the server's `Retry-After`), or a connection that failed. */
+function problemOf(error, statuses) {
+  if (error instanceof HttpError) return statuses.includes(error.status) ? {
+    reason: `HTTP ${error.status}`,
+    retryAfter: error.headers['retry-after']
+  } : undefined;
+  return transientError(error);
+}
 /** The class names of the widgets `session.captcha` solves. Checked only when a recipe declares it. */
 const CAPTCHA_MARKUP = /\b(?:g-recaptcha|h-captcha|cf-turnstile)\b/;
 function bodyText(body) {
@@ -4438,6 +5377,7 @@ function bodyText(body) {
   if (body.kind === 'pdf') return pdfText(body);
   if (body.kind === 'workbook') return workbookText(body);
   if (body.kind === 'deck') return deckText(body);
+  if (body.kind === 'xml') return body.xml;
   return body.kind === 'html' ? body.html : body.text;
 }
 function renderMap(map, lookup) {
@@ -5713,18 +6653,34 @@ function readElements(elements) {
 
 /**
  * Runs a `goto` step: renders the URL (relative to the current page), waits for
- * the gate's throttle (`delayMs`), navigates, records the page's real URL in the
- * scope, and checks the response against the recipe's block rule.
+ * the gate's throttle (`delayMs`), navigates (again, after a pause, while it
+ * fails in passing: `limits.retry`), records the page's real URL in the scope,
+ * and checks the response against the recipe's block rule.
  *
  * @throws BlockedError when the response is a block.
  */
 async function navigate(step, page, scope, recipe, gate, events) {
   const target = renderText(step.url, path => scope.lookup(path));
   const url = new URL(target, scope.pageState?.url ?? page.url()).href;
-  await gate.throttle();
-  const response = await page.goto(url, {
-    waitUntil: step.waitUntil,
-    timeout: recipe.limits?.timeoutMs
+  const rule = resolveRetryRule(recipe.limits?.retry);
+  const response = await withTransportRetry(url, {
+    run: () => page.goto(url, {
+      waitUntil: step.waitUntil,
+      timeout: recipe.limits?.timeoutMs
+    }),
+    problem: outcome => {
+      if ('error' in outcome) return transientError(outcome.error);
+      const status = outcome.value?.status();
+      return status !== undefined && rule.statuses.includes(status) ? {
+        reason: `HTTP ${status}`,
+        retryAfter: outcome.value?.headers()['retry-after']
+      } : undefined;
+    }
+  }, {
+    recipeId: recipe.id,
+    gate,
+    events,
+    rule
   });
   scope.setPage({
     url: page.url()
@@ -5874,8 +6830,13 @@ class WebStepRunner {
     const link = this.page.locator(next.selector).first();
     if (!(await appears(link, NEXT_LINK_TIMEOUT_MS))) return null;
     const before = this.page.url();
-    await link.click();
-    await this.page.waitForLoadState();
+    const release = await this.gate.request(before);
+    try {
+      await link.click();
+      await this.page.waitForLoadState();
+    } finally {
+      release();
+    }
     if (this.page.url() === before) await this.page.waitForTimeout(NEXT_LINK_TIMEOUT_MS / 4);
     this.events.emit({
       type: 'page:visit',
@@ -5889,6 +6850,23 @@ class WebStepRunner {
       url: this.page.url()
     };
   }
+  /**
+   * A runner on a new tab of the same context, for one parallel iteration:
+   * it shares cookies, the gate and the captcha guard; disposing it closes the tab only.
+   *
+   * @returns The forked runner.
+   */
+  async fork() {
+    const {
+      context
+    } = this.session;
+    const page = await context.newPage();
+    const viewport = this.recipe.session?.viewport;
+    if (viewport !== undefined) await page.setViewportSize(viewport);
+    return new WebStepRunner(new BrowserSession(context, page, async () => {
+      await page.close();
+    }), this.recipe, this.events, this.gate, this.captcha);
+  }
   async elements(selector) {
     return snapshotElements(selector, this.page);
   }
@@ -5925,16 +6903,30 @@ function accessOptions(lease, headers) {
  * The bootstrap runs through the same access lease as the crawl that follows,
  * so a login and the requests that use its cookies come from one IP.
  *
+ * With `session.browserProfile`, the bootstrap runs in that profile, and
+ * without a bootstrap the profile's own cookies and storage are the state: an
+ * api recipe picks up a login a browser left in the profile.
+ *
  * @param recipe - The input recipe.
  * @param deps - Browser, hooks, events.
  * @param lease - The recipe run's access; direct when omitted.
  * @param captcha - Solves the bootstrap's captchas (a login form's).
+ * @param owner - The recipe run, which a browser profile is held by.
  * @returns The state, or `undefined` when the recipe declares none.
  */
-async function resolveStorageState(recipe, deps, lease, captcha) {
+async function resolveStorageState(recipe, deps, lease, captcha, owner = {}) {
   const saved = await readSavedState(recipe, deps);
+  if (saved !== undefined) return saved;
   const session = recipe.session;
-  if (saved !== undefined || session?.bootstrap === undefined) return saved;
+  if (session?.browserProfile !== undefined) {
+    const browserSession = await openBrowserProfile(recipe, deps, lease, owner);
+    try {
+      return session.bootstrap === undefined ? await browserSession.storageState() : await runBootstrap(recipe, browserSession, deps, captcha);
+    } finally {
+      await browserSession.close();
+    }
+  }
+  if (session?.bootstrap === undefined) return undefined;
   const browser = await deps.browser();
   const browserSession = await browser.newSession({
     cookies: session.cookies,
@@ -5948,6 +6940,27 @@ async function resolveStorageState(recipe, deps, lease, captcha) {
     await browserSession.close();
   }
 }
+/**
+ * Opens the recipe's `session.browserProfile` with its session options and
+ * the lease's proxy.
+ *
+ * @param recipe - A recipe with `session.browserProfile`.
+ * @param deps - For `profiles`.
+ * @param lease - The access lease.
+ * @param owner - The recipe run.
+ * @returns The session in the profile.
+ */
+async function openBrowserProfile(recipe, deps, lease, owner) {
+  const session = recipe.session;
+  const name = session?.browserProfile ?? '';
+  if (deps.profiles === undefined) throw new Error(`recipe "${recipe.id}" uses browser profile "${name}", but this crawler has no profiles directory (CrawlOptions.profilesDir)`);
+  return deps.profiles.open(name, {
+    cookies: session?.cookies,
+    userAgent: session?.userAgent,
+    viewport: session?.viewport,
+    ...accessOptions(lease, session?.headers)
+  }, owner);
+}
 /**
  * The storage state saved by an earlier bootstrap (`session.storageStatePath`), if the recipe names one.
  *
@@ -5977,7 +6990,7 @@ async function runBootstrap(recipe, browserSession, deps, captcha) {
     cookies: [],
     origins: []
   };
-  const runner = new WebStepRunner(browserSession, recipe, deps.events, undefined, captcha);
+  const runner = new WebStepRunner(browserSession, recipe, deps.events, new RunGate(1, recipe.limits?.delayMs ?? 0, deps.hosts), captcha);
   const scope = new ExtractionScope();
   scope.set('vars', recipe.vars ?? {});
   scope.set('start', {
@@ -6082,6 +7095,66 @@ class RotatingRunner {
       throw error;
     }
   }
+  /**
+   * A runner for one parallel iteration, forked from whichever runner is
+   * current when it runs a step: after a rotation it forks again from the new
+   * one, since the old context is gone (or going). Blocks are noted and
+   * rotated like the main runner's.
+   *
+   * @returns The iteration's runner.
+   */
+  async fork() {
+    // `own`: a tab this iteration opened and must close; an api runner is shared and never disposed here.
+    let forked;
+    const disposeForked = async () => {
+      if (forked?.own === true) await disposeQuietly(forked.runner);
+      forked = undefined;
+    };
+    const current = async () => {
+      if (forked?.generation === this.generation) return forked.runner;
+      await disposeForked();
+      const inner = this.inner;
+      forked = inner.fork === undefined ? {
+        generation: this.generation,
+        runner: inner,
+        own: false
+      } : {
+        generation: this.generation,
+        runner: await inner.fork(),
+        own: true
+      };
+      return forked.runner;
+    };
+    return {
+      runLeaf: async (step, scope) => {
+        const generation = this.generation;
+        try {
+          const runner = await current();
+          await runner.runLeaf(step, scope);
+        } catch (error) {
+          this.note(error, generation);
+          throw error;
+        }
+      },
+      nextPage: async (next, scope) => {
+        const generation = this.generation;
+        try {
+          const runner = await current();
+          return await runner.nextPage(next, scope);
+        } catch (error) {
+          this.note(error, generation);
+          throw error;
+        }
+      },
+      elements: async (selector, scope) => {
+        const runner = await current();
+        if (runner.elements === undefined) throw new Error('forEach over selector iterates live elements and needs a browser; this recipe runs in api mode');
+        return runner.elements(selector, scope);
+      },
+      rotate: error => this.rotate(error),
+      dispose: disposeForked
+    };
+  }
   async elements(selector, scope) {
     if (this.inner.elements === undefined) throw new Error('forEach over selector iterates live elements and needs a browser; this recipe runs in api mode');
     return this.inner.elements(selector, scope);
@@ -6130,12 +7203,19 @@ class RotatingRunner {
  * the sink sees one record at a time and `maxRecords` is exact: once reached,
  * every later emit returns `stop` before mapping.
  *
- * @param input - The input recipe.
+ * @param recipe - The input recipe.
  * @param output - The output recipe it feeds.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @returns What happened.
  */
-async function runInputRecipe(input, output, deps) {
+async function runInputRecipe(recipe, output, deps) {
+  const input = {
+    ...recipe,
+    limits: {
+      ...recipe.limits,
+      retry: resolveRetryRule(recipe.limits?.retry, deps.retry)
+    }
+  };
   const started = Date.now();
   const report = {
     recipeId: input.id,
@@ -6154,8 +7234,8 @@ async function runInputRecipe(input, output, deps) {
     failed: 0
   };
   const limits = input.limits ?? {};
-  // A web recipe drives one page, so only api mode runs iterations in parallel.
-  const gate = new RunGate(input.mode === 'web' ? 1 : limits.concurrency ?? 1, limits.delayMs ?? 0);
+  // Parallel iterations: requests in api mode, tabs of the recipe's context in web mode.
+  const gate = new RunGate(limits.concurrency ?? 1, limits.delayMs ?? 0, deps.hosts);
   let stopped = false;
   let chain = Promise.resolve();
   const unsubscribe = deps.events.subscribe(event => {
@@ -6170,7 +7250,7 @@ async function runInputRecipe(input, output, deps) {
     recipeId: input.id,
     mode: input.mode
   });
-  deps.dedupe.startRecipe();
+  const dedupe = deps.dedupe.forRecipe();
   let runner;
   try {
     const onBlock = input.session?.onBlock;
@@ -6278,7 +7358,7 @@ async function runInputRecipe(input, output, deps) {
           url,
           key: record.key
         });
-      } else if (deps.dedupe.isDuplicate(record)) {
+      } else if (dedupe.isDuplicate(record)) {
         report.duplicates += 1;
         deps.events.emit({
           type: 'record:duplicate',
@@ -6370,7 +7450,8 @@ async function openRunner(input, deps, context, lease) {
     lease
   });
   if (lease.cdp !== undefined) return openRemoteRunner(input, deps, context, lease, lease.cdp);
-  const storageState = await resolveStorageState(input, deps, lease, captcha);
+  if (input.mode === 'web' && input.session?.browserProfile !== undefined) return openProfileRunner(input, deps, context, lease, captcha);
+  const storageState = await resolveStorageState(input, deps, lease, captcha, context);
   const session = input.session;
   const access = accessOptions(lease, session?.headers);
   if (input.mode === 'web') {
@@ -6394,6 +7475,28 @@ async function openRunner(input, deps, context, lease) {
   });
   return new ApiStepRunner(client, input, deps.events, gate);
 }
+/**
+ * A web runner in a persistent browser profile. The bootstrap runs in the
+ * same browser as the crawl, and what both leave behind (cookies, storage)
+ * stays in the profile for the next run.
+ */
+async function openProfileRunner(input, deps, context, lease, captcha) {
+  const saved = await readSavedState(input, deps);
+  const browserSession = await openBrowserProfile(saved === undefined ? input : {
+    ...input,
+    session: {
+      ...input.session,
+      cookies: [...saved.cookies, ...(input.session?.cookies ?? [])]
+    }
+  }, deps, lease, context);
+  try {
+    if (saved === undefined && input.session?.bootstrap !== undefined) await runBootstrap(input, browserSession, deps, captcha);
+  } catch (error) {
+    await browserSession.close();
+    throw error;
+  }
+  return new WebStepRunner(browserSession, input, deps.events, context.gate, captcha);
+}
 /**
  * A web runner in a remote browser. The bootstrap runs in the same remote
  * session as the crawl: providers tie the IP and fingerprint to the
@@ -6408,6 +7511,7 @@ async function openRemoteRunner(input, deps, context, lease, cdp) {
     lease
   });
   if (input.mode === 'api') throw new AccessConfigError(`recipe "${input.id}" runs in api mode, but access profile "${lease.profile}" is a remote browser; api recipes need a proxy profile`);
+  if (input.session?.browserProfile !== undefined) throw new AccessConfigError(`recipe "${input.id}" uses browser profile "${input.session.browserProfile}", which needs a local browser, but access profile "${lease.profile}" is a remote browser`);
   const session = input.session;
   const storageState = await readSavedState(input, deps);
   const browserSession = await BrowserClient.connectOverCDP(cdp, {
@@ -6426,27 +7530,42 @@ async function openRemoteRunner(input, deps, context, lease, cdp) {
 }
 
 /**
- * Runs every input recipe of a set, one after another, into one sink.
+ * Runs every input recipe of a set into one sink, `parallel` at a time
+ * (default one after another). Reports come back in the set's order whatever
+ * order the recipes finish in. Under `onRecipeError: 'stop'`, a failed recipe
+ * stops the ones not started yet; those already running finish.
  *
  * @param set - The bound recipes.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @param onRecipeError - Whether a failed recipe stops the run.
+ * @param parallel - How many input recipes run at once.
  * @returns The report.
  */
-async function runCrawl(set, deps, onRecipeError) {
+async function runCrawl(set, deps, onRecipeError, parallel = 1) {
   const started = Date.now();
   await deps.sink.open(set.output);
-  const recipes = [];
+  const reports = [];
   let sink;
   try {
-    for (const input of set.inputs) {
-      const report = await runInputRecipe(input, set.output, deps);
-      recipes.push(report);
-      if (onRecipeError === 'stop' && report.error !== undefined) break;
-    }
+    let next = 0;
+    let stopped = false;
+    const lane = async () => {
+      while (!stopped && next < set.inputs.length) {
+        const index = next;
+        next += 1;
+        const report = await runInputRecipe(set.inputs[index], set.output, deps);
+        reports[index] = report;
+        if (onRecipeError === 'stop' && report.error !== undefined) stopped = true;
+      }
+    };
+    const lanes = Math.max(1, Math.min(parallel, set.inputs.length));
+    await Promise.all(Array.from({
+      length: lanes
+    }, lane));
   } finally {
     sink = await deps.sink.close();
   }
+  const recipes = reports.filter(report => report !== undefined);
   return {
     outputId: set.output.id,
     recipes,
@@ -6471,6 +7590,8 @@ function createCrawler(options = {}) {
   const access = new AccessBroker(options.access, options.accessPlugins);
   const hooks = new HookRegistry(options.hooks);
   const captchaSolvers = new CaptchaSolverRegistry(options.captchaSolvers);
+  const hosts = new HostThrottle(options.throttle ?? options.access?.throttle);
+  const profiles = new BrowserProfiles(options.profilesDir ?? resolve$1(options.storageStateDir ?? '.', '.opencraw', 'profiles'), options.browser);
   const events = new EventBus(options.onEvent);
   let browser;
   const launch = () => {
@@ -6489,8 +7610,11 @@ function createCrawler(options = {}) {
       debug: options.debug === true,
       access,
       captchaSolvers,
+      hosts,
+      profiles,
+      retry: options.retry,
       ignoreHTTPSErrors: options.browser?.ignoreHTTPSErrors
-    }, options.onRecipeError ?? 'continue'),
+    }, options.onRecipeError ?? 'continue', options.parallel ?? 1),
     async close() {
       const launched = browser;
       browser = undefined;
@@ -6506,7 +7630,7 @@ const CRAWL_MODES = ['web', 'api'];
 const SELECTOR_KINDS = ['css', 'xpath', 'jsonpath', 'regex', 'table'];
 /** `take` also accepts `attr:<name>`, which is validated by pattern rather than listed. */
 const TAKE_KINDS = ['text', 'html', 'value', 'json'];
-const BODY_KINDS = ['json', 'jsonl', 'html', 'text', 'pdf', 'csv', 'xlsx', 'pptx', 'yaml', 'markdown'];
+const BODY_KINDS = ['json', 'jsonl', 'html', 'text', 'pdf', 'csv', 'xlsx', 'pptx', 'yaml', 'markdown', 'xml', 'docx'];
 const YAML_SCALARS = ['typed', 'text'];
 /** How a PDF table aligns a row's values against a cell wrapped over several lines. */
 const TABLE_ALIGNS = ['auto', 'top', 'center', 'bottom'];
@@ -6712,8 +7836,20 @@ const extractStep = z.strictObject({
   fillDown: z.array(z.string().min(1)).min(1).optional(),
   includeHidden: z.boolean().optional(),
   slide: z.string().min(1).optional(),
-  shapes: z.boolean().optional()
+  shapes: z.boolean().optional(),
+  namespaces: z.record(z.string().regex(/^[A-Z_][\w.-]*$/i, 'a namespace prefix such as atom'), z.string().min(1)).optional(),
+  ignoreNamespaces: z.boolean().optional()
 }).check(context => {
+  if (context.value.kind !== 'xpath') {
+    for (const key of ['namespaces', 'ignoreNamespaces']) {
+      if (context.value[key] !== undefined) context.issues.push({
+        code: 'custom',
+        input: context.value,
+        path: [key],
+        message: `"${key}" belongs to kind "xpath"`
+      });
+    }
+  }
   if (context.value.kind === 'table') return;
   for (const key of tableOnly) {
     if (context.value[key] !== undefined) context.issues.push({
@@ -6962,13 +8098,21 @@ const sessionSpecSchema = z.strictObject({
   access: sessionAccessSchema.optional(),
   blockedWhen: blockRuleSchema.optional(),
   onBlock: blockRotationSchema.optional(),
-  captcha: captchaSettingsSchema.optional()
+  captcha: captchaSettingsSchema.optional(),
+  browserProfile: z.string().regex(/^[\w-]+$/, 'a browser profile name is letters, digits, hyphens and underscores').optional()
+});
+const retryRuleSchema = z.strictObject({
+  attempts: z.int().min(1).max(10).optional(),
+  backoffMs: z.int().nonnegative().optional(),
+  maxDelayMs: z.int().nonnegative().optional(),
+  statuses: z.array(z.int().min(400).max(599)).optional()
 });
 const limitsSchema = z.strictObject({
   maxRecords: z.int().positive().optional(),
   delayMs: z.int().nonnegative().optional(),
   timeoutMs: z.int().positive().optional(),
-  concurrency: z.int().min(1).max(64).optional()
+  concurrency: z.int().min(1).max(64).optional(),
+  retry: retryRuleSchema.optional()
 });
 const inputRecipeSchema = z.strictObject({
   $schema: z.string().optional(),
@@ -7499,5 +8643,176 @@ function isSameOutput(document, output) {
   return recipeKindOf(document.content) === 'output' && document.content.id === output.id;
 }
 
-export { ACCESS_PRESETS, AccessBroker, AccessConfigError, BrowserClient, BrowserSession, CaptchaError, DEFAULT_CAPTCHA_SELECTOR, HttpClient, HttpError, MappingFailedError, PdfReadError, RecipeBindingError, RecipeSet, RecipeValidationError, RecordRejectedError, StepFailure, TransformError, UnknownHookError, accessConfigJsonSchema, accessConfigSchema, bindRecipeSet, createCrawler, csvWorkbook, deckText, detectChallenge, detectDelimiter, fillDown, findDeckTables, findGridTables, findTables, htmlTableSheets, inputRecipeJsonSchema, inputRecipeSchema, isDeckDocument, isWorkbookDocument, jsonLinesSink, loadAccessConfig, loadRecipeSet, loadRecipes, memorySink, outputRecipeJsonSchema, outputRecipeSchema, parseCsv, parseInputRecipe, parseOutputRecipe, pdfText, readMarkdown, readPdf, readRecipeSource, readYaml, traceLine, tryParseJson, validateBinding, workbookText };
+const BOOKKEEPING = new Set(['_source', '_key']);
+/**
+ * Compares two runs' records by key: what was added, what was removed, and
+ * for the records in both, which fields changed, before and after. Like a
+ * `git diff` of a price list, but row by row instead of line by line, so a
+ * reordered file is not a change.
+ *
+ * @param previous - The earlier run's records.
+ * @param current - The later run's records.
+ * @param options - The key fields, the fields to ignore, the shrink threshold.
+ * @returns The differences.
+ * @throws Error when records carry no key: without `key`, every line needs `_key`.
+ */
+function diffRecords(previous, current, options = {}) {
+  const ignore = new Set([...BOOKKEEPING, ...(options.ignore ?? [])]);
+  const keyOf = keyReader(options.key);
+  let repeated = 0;
+  const index = (records, side) => {
+    const byKey = new Map();
+    for (const [line, record] of records.entries()) {
+      const key = keyOf(record, `${side} record ${line + 1}`);
+      if (byKey.has(key)) repeated += 1;else byKey.set(key, record);
+    }
+    return byKey;
+  };
+  const before = index(previous, 'previous');
+  const after = index(current, 'current');
+  const removed = [];
+  const changed = [];
+  const added = [];
+  let unchanged = 0;
+  for (const [key, record] of before) {
+    const now = after.get(key);
+    if (now === undefined) {
+      removed.push({
+        change: 'removed',
+        key,
+        before: record
+      });
+      continue;
+    }
+    const fields = fieldChanges(record, now, ignore, '');
+    if (fields.length === 0) unchanged += 1;else changed.push({
+      change: 'changed',
+      key,
+      before: record,
+      after: now,
+      fields
+    });
+  }
+  for (const [key, record] of after) {
+    if (!before.has(key)) added.push({
+      change: 'added',
+      key,
+      after: record
+    });
+  }
+  const threshold = options.shrink ?? 0.5;
+  const shrunk = previous.length > 0 && current.length < previous.length * (1 - threshold);
+  return {
+    added: added.length,
+    removed: removed.length,
+    changed: changed.length,
+    unchanged,
+    changes: [...removed, ...changed, ...added],
+    repeated,
+    counts: {
+      previous: previous.length,
+      current: current.length
+    },
+    ...(shrunk && {
+      shrunk: {
+        previous: previous.length,
+        current: current.length
+      }
+    })
+  };
+}
+/**
+ * The diff options an output recipe implies: its key fields, and its fields
+ * the engine fills differently every run (`generated: now`, `uuid`).
+ *
+ * @param output - The output recipe.
+ * @returns The key and the fields to ignore.
+ */
+function diffOptionsFor(output) {
+  const fields = Object.entries(output.fields);
+  return {
+    key: fields.filter(([, field]) => field.key === true).map(([name]) => name),
+    ignore: fields.filter(([, field]) => field.generated === 'now' || field.generated === 'uuid').map(([name]) => name)
+  };
+}
+/**
+ * A key as people read it: the key fields' values joined (`Pandina · 1.0 Hybrid`).
+ *
+ * @param key - A record key (a JSON array of the key values).
+ * @returns The readable form.
+ */
+function readableKey(key) {
+  try {
+    const values = JSON.parse(key);
+    return Array.isArray(values) ? values.map(value => typeof value === 'string' ? value : JSON.stringify(value)).join(' · ') : key;
+  } catch {
+    return key;
+  }
+}
+function keyReader(fields) {
+  if (fields !== undefined && fields.length > 0) return record => JSON.stringify(fields.map(field => record[field] ?? null));
+  return storedKey;
+}
+/** The `_key` a sink in append mode wrote on the line. */
+function storedKey(record, where) {
+  const key = record._key;
+  if (typeof key !== 'string') throw new Error(`${where} has no _key: name the fields that identify a record (--key model,version, or the output recipe's key fields)`);
+  return key;
+}
+function fieldChanges(before, after, ignore, prefix) {
+  const names = [...new Set([...Object.keys(before), ...Object.keys(after)])];
+  const changes = [];
+  for (const name of names) {
+    const field = `${prefix}${name}`;
+    if (ignore.has(field) || prefix === '' && ignore.has(name)) continue;
+    const was = before[name];
+    const is = after[name];
+    if (isPlainObject(was) && isPlainObject(is)) changes.push(...fieldChanges(was, is, ignore, `${field}.`));else if (!sameValue(was, is)) changes.push({
+      field,
+      before: was ?? null,
+      after: is ?? null
+    });
+  }
+  return changes;
+}
+function isPlainObject(value) {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/** Equal as JSON, whatever the order of object keys. */
+function sameValue(first, second) {
+  return stableJson(first ?? null) === stableJson(second ?? null);
+}
+function stableJson(value) {
+  if (Array.isArray(value)) return `[${value.map(entry => stableJson(entry)).join(',')}]`;
+  if (isPlainObject(value)) return `{${Object.keys(value).sort((first, second) => first.localeCompare(second)).map(key => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(',')}}`;
+  return JSON.stringify(value) ?? 'null';
+}
+
+/**
+ * The records of a JSON Lines file, as a sink wrote them.
+ *
+ * @param path - The file.
+ * @returns One record per non-empty line.
+ * @throws Error naming the file and the line that is not a JSON object.
+ */
+async function readRecordsFile(path) {
+  const text = await readFile(path, 'utf8');
+  const records = [];
+  for (const [index, line] of text.split(/\r?\n/).entries()) {
+    if (line.trim() === '') continue;
+    let value;
+    try {
+      value = JSON.parse(line);
+    } catch (error) {
+      throw new Error(`${path}:${index + 1}: not JSON (${error.message})`, {
+        cause: error
+      });
+    }
+    if (typeof value !== 'object' || value === null || Array.isArray(value)) throw new Error(`${path}:${index + 1}: not a record (a JSON object)`);
+    records.push(value);
+  }
+  return records;
+}
+
+export { ACCESS_PRESETS, AccessBroker, AccessConfigError, BrowserClient, BrowserSession, CaptchaError, DEFAULT_CAPTCHA_SELECTOR, DEFAULT_RETRY_RULE, HostThrottle, HttpClient, HttpError, MappingFailedError, PdfReadError, RecipeBindingError, RecipeSet, RecipeValidationError, RecordRejectedError, StepFailure, TransformError, UnknownHookError, accessConfigJsonSchema, accessConfigSchema, bindRecipeSet, createCrawler, csvWorkbook, deckText, detectChallenge, detectDelimiter, diffOptionsFor, diffRecords, fillDown, findDeckTables, findGridTables, findTables, htmlTableSheets, inputRecipeJsonSchema, inputRecipeSchema, isDeckDocument, isWorkbookDocument, isXmlDocument, jsonLinesSink, loadAccessConfig, loadRecipeSet, loadRecipes, memorySink, outputRecipeJsonSchema, outputRecipeSchema, parseCsv, parseInputRecipe, parseOutputRecipe, parseXml, pdfText, readMarkdown, readPdf, readRecipeSource, readRecordsFile, readYaml, readableKey, retryRuleSchema, selectXpath, takeFromXml, throttleConfigSchema, traceLine, tryParseJson, validateBinding, workbookText };
 //# sourceMappingURL=index.esm.js.map