@opencraw/core 0.1.2 → 0.1.3

package/dist/index.esm.js

@@ -1,7 +1,7 @@
 import { z } from 'zod';
 import { randomInt, randomUUID } from 'node:crypto';
-import { readFile, mkdir, writeFile, stat, readdir } from 'node:fs/promises';
-import { resolve as resolve$1, dirname, extname, join as join$1 } from 'node:path';
+import { readFile, mkdir, writeFile, rm, stat, readdir } from 'node:fs/promises';
+import { resolve as resolve$1, join as join$1, dirname, extname } from 'node:path';
 import { firefox, webkit, chromium, request } from 'playwright';
 import { createWriteStream } from 'node:fs';
 import { once } from 'node:events';
@@ -67,10 +67,20 @@ const pluginSchema = z.strictObject({
   options: z.record(z.string(), z.unknown()).optional()
 });
 const accessProfileSchema = z.union([directSchema, proxySchema, poolSchema, cdpSchema, pluginSchema]);
+const hostRuleSchema = z.strictObject({
+  delayMs: z.int().nonnegative().optional(),
+  concurrency: z.int().min(1).max(256).optional()
+});
+const throttleConfigSchema = z.strictObject({
+  delayMs: z.int().nonnegative().optional(),
+  concurrency: z.int().min(1).max(256).optional(),
+  domains: z.record(z.string().regex(/^[\w.-]+$/, 'a domain such as example.com'), hostRuleSchema).optional()
+});
 const accessConfigSchema = z.strictObject({
   $schema: z.string().optional(),
   profiles: z.record(z.string().regex(/^[\w-]+$/, 'a profile name is letters, digits, hyphens and underscores'), accessProfileSchema),
-  default: z.string().optional()
+  default: z.string().optional(),
+  throttle: throttleConfigSchema.optional()
 }).refine(config => config.default === undefined || Object.hasOwn(config.profiles, config.default), {
   message: 'default names a profile that does not exist',
   path: ['default']
@@ -1179,6 +1189,17 @@ async function blockResources(context, types) {
     await route.continue();
   });
 }
+/**
+ * What a context gets after it opened: the cookies to add and the resource
+ * types to skip.
+ *
+ * @param context - The context.
+ * @param options - The session options.
+ */
+async function applySessionExtras(context, options) {
+  if (options.cookies !== undefined && options.cookies.length > 0) await context.addCookies(options.cookies);
+  if (options.blockResources !== undefined && options.blockResources.length > 0) await blockResources(context, new Set(options.blockResources));
+}
 /** A launched browser; sessions are opened from it and closed independently. */
 class BrowserClient {
   browser;
@@ -1238,8 +1259,7 @@ class BrowserClient {
     };
     const context = await this.browser.newContext(contextOptions);
     if (this.config.timeoutMs !== undefined) context.setDefaultTimeout(this.config.timeoutMs);
-    if (options.cookies !== undefined && options.cookies.length > 0) await context.addCookies(options.cookies);
-    if (options.blockResources !== undefined && options.blockResources.length > 0) await blockResources(context, new Set(options.blockResources));
+    await applySessionExtras(context, options);
     const page = await context.newPage();
     return new BrowserSession(context, page);
   }
@@ -1248,13 +1268,206 @@ class BrowserClient {
   }
 }
 
+/** The file that marks a profile as in use, holding the owning process id. */
+const LOCK_FILE = '.opencraw.lock';
+/** A browser profile name: it becomes a directory, so no separators or dots. */
+const BROWSER_PROFILE_NAME = /^[\w-]+$/;
+/**
+ * Browser profiles that persist between runs: each is a directory of a real
+ * browser's user data (cookies, local storage, IndexedDB, cache, service
+ * workers), so a login, a consent choice or a site's trust in a returning
+ * visitor carries over to the next run. The browser equivalent of a user who
+ * never clears their history.
+ *
+ * A profile directory can be open in one browser at a time. Within this
+ * crawler, a second use waits for the first to close; the same owner (one
+ * recipe run reopening after a rotation) takes it over instead. Another
+ * crawler holding it, in this process or another, is reported, not waited
+ * for: a lock file in the profile names the process, and one left by a
+ * process that died is taken over. (Chromium's own profile lock is not
+ * enough: headless builds do not take it.)
+ */
+class BrowserProfiles {
+  directory;
+  config;
+  held = new Map();
+  waiting = new Map();
+  /**
+   * @param directory - Where the profiles live, one subdirectory each.
+   * @param config - The crawler's browser settings (type, binary, headless, timeouts).
+   */
+  constructor(directory, config = {}) {
+    this.directory = directory;
+    this.config = config;
+  }
+  async take(name, owner) {
+    for (;;) {
+      const holder = this.held.get(name);
+      if (holder === undefined) return;
+      if (holder.owner === owner) {
+        await holder.session.close();
+        continue;
+      }
+      await new Promise(resolve => {
+        const queue = this.waiting.get(name) ?? [];
+        queue.push(resolve);
+        this.waiting.set(name, queue);
+      });
+    }
+  }
+  free(name) {
+    this.held.delete(name);
+    this.waiting.get(name)?.shift()?.();
+  }
+  launch(path, options) {
+    const type = this.config.browserType ?? DEFAULT_BROWSER_CONFIG.browserType;
+    const launcher = type === 'firefox' ? firefox : type === 'webkit' ? webkit : chromium;
+    return launcher.launchPersistentContext(path, {
+      headless: this.config.headless ?? DEFAULT_BROWSER_CONFIG.headless,
+      slowMo: this.config.slowMo,
+      executablePath: this.config.executablePath,
+      proxy: options.proxy ?? this.config.proxy,
+      extraHTTPHeaders: options.headers,
+      userAgent: options.userAgent,
+      viewport: options.viewport,
+      ignoreHTTPSErrors: this.config.ignoreHTTPSErrors === true || options.ignoreHTTPSErrors === true
+    });
+  }
+  /**
+   * The profile's directory.
+   *
+   * @param name - A profile name.
+   * @returns The absolute path.
+   */
+  pathOf(name) {
+    if (!BROWSER_PROFILE_NAME.test(name)) throw new Error(`browser profile "${name}": a name is letters, digits, hyphens and underscores`);
+    return resolve$1(this.directory, name);
+  }
+  /**
+   * Opens a profile in its own browser, waiting while another run of this
+   * crawler uses it.
+   *
+   * @param name - The profile.
+   * @param options - Proxy, headers, viewport, cookies to add. `storageState` is ignored: the profile has its own.
+   * @param owner - Who opens it; the same owner reopening closes its previous session first.
+   * @returns The session; closing it frees the profile.
+   */
+  async open(name, options, owner) {
+    const path = this.pathOf(name);
+    await this.take(name, owner);
+    let context;
+    let unlock = unlocked;
+    try {
+      await mkdir(path, {
+        recursive: true
+      });
+      unlock = await lockProfile(path, name);
+      context = await this.launch(path, options);
+    } catch (error) {
+      await unlock();
+      this.free(name);
+      const message = error instanceof Error ? error.message : String(error);
+      if (/ProcessSingleton|SingletonLock|already in use/i.test(message)) throw new Error(`browser profile "${name}" is open in another browser (${path}); close it or use another profile`, {
+        cause: error
+      });
+      throw error;
+    }
+    if (this.config.timeoutMs !== undefined) context.setDefaultTimeout(this.config.timeoutMs);
+    await applySessionExtras(context, options);
+    const page = context.pages()[0] ?? (await context.newPage());
+    let closed = false;
+    const session = new BrowserSession(context, page, async () => {
+      if (closed) return;
+      closed = true;
+      try {
+        await context.close();
+      } finally {
+        await unlock();
+        this.free(name);
+      }
+    });
+    this.held.set(name, {
+      owner,
+      session
+    });
+    return session;
+  }
+}
+/**
+ * Marks a profile as used by this process, refusing one a live process
+ * holds and taking over one a dead process left behind.
+ *
+ * @param path - The profile directory.
+ * @param name - The profile name, for the message.
+ * @returns The unlock.
+ * @throws Error when another crawler holds the profile.
+ */
+async function lockProfile(path, name) {
+  const file = join$1(path, LOCK_FILE);
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try {
+      await writeFile(file, String(process.pid), {
+        flag: 'wx'
+      });
+      return async () => {
+        await rm(file, {
+          force: true
+        });
+      };
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      const holder = Number(await readHolder(file));
+      if (Number.isSafeInteger(holder) && holder > 0 && isAlive(holder)) throw new Error(`browser profile "${name}" is open in another browser (${path}, process ${holder}); close it or use another profile`, {
+        cause: error
+      });
+      await rm(file, {
+        force: true
+      });
+    }
+  }
+  throw new Error(`browser profile "${name}": cannot lock ${path}`);
+}
+/** The unlock of a profile that was never locked. */
+async function unlocked() {}
+async function readHolder(file) {
+  try {
+    return await readFile(file, 'utf8');
+  } catch {
+    return '';
+  }
+}
+function isAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    return error.code === 'EPERM';
+  }
+}
+
+/**
+ * Disposes a runner, ignoring a failure: a tab whose browser already went
+ * away (a rotation, a crash) has nothing left to close.
+ *
+ * @param runner - The runner.
+ */
+async function disposeQuietly(runner) {
+  try {
+    await runner.dispose();
+  } catch {
+    // already gone
+  }
+}
+
 /**
  * Runs a body once per item of a list (`over`), or once per live element
  * matching `selector`, each in a fresh child scope with the item bound under
  * `as`; emits a record per iteration when asked.
  *
- * With a concurrent gate, iterations run as permits allow and records come
- * out in completion order; without one, in list order.
+ * With a concurrent gate, iterations of a list run as permits allow and
+ * records come out in completion order; without one, in list order. In web
+ * mode each parallel iteration runs in a tab of its own (`runner.fork`); a
+ * loop over live elements stays sequential, since its elements live on one page.
  *
  * @param step - The forEach step.
  * @param scope - The scope the list lives in.
@@ -1264,7 +1477,7 @@ class BrowserClient {
 async function runForEach(step, scope, walk) {
   const items = await itemsOf$1(step, scope, walk);
   const gate = walk.gate;
-  if (gate?.concurrent === true) return runPooled(step, scope, walk, items, gate);
+  if (gate?.concurrent === true && step.selector === undefined && (walk.recipe.mode === 'api' || walk.runner.fork !== undefined)) return runPooled(step, scope, walk, items, gate);
   for (const item of items) {
     if ((await runIteration(step, scope, walk, item)) === 'stop') return 'stop';
   }
@@ -1286,17 +1499,21 @@ async function runPooled(step, scope, walk, items, gate) {
   let stopped = false;
   let failure;
   const tasks = [];
-  const overrides = {
-    gate: gate.nested()
-  };
+  const nested = gate.nested();
   const iterate = async (item, release) => {
+    let runner;
     try {
-      if ((await runIteration(step, scope, walk, item, overrides)) === 'stop') stopped = true;
+      runner = walk.runner.fork === undefined ? walk.runner : await walk.runner.fork();
+      if ((await runIteration(step, scope, walk, item, {
+        gate: nested,
+        runner
+      })) === 'stop') stopped = true;
     } catch (error) {
       failure ??= {
         error
       };
     } finally {
+      if (runner !== undefined && runner !== walk.runner) await disposeQuietly(runner);
       release();
     }
   };
@@ -1604,6 +1821,7 @@ function logThrough(walk) {
 class RunGate {
   permits;
   minIntervalMs;
+  hosts;
   shared;
   inFlight = 0;
   waiting = [];
@@ -1611,11 +1829,13 @@ class RunGate {
   /**
    * @param permits - Iterations allowed in flight; 1 is sequential.
    * @param minIntervalMs - Minimum time between two request starts across the run.
+   * @param hosts - The crawler's per-site throttle, shared with every other recipe.
    * @param shared - The throttle state to share (internal: `nested` gates keep their parent's).
    */
-  constructor(permits, minIntervalMs, shared) {
+  constructor(permits, minIntervalMs, hosts, shared) {
     this.permits = permits;
     this.minIntervalMs = minIntervalMs;
+    this.hosts = hosts;
     this.shared = shared;
   }
   /** Whether this gate lets more than one iteration run at once. */
@@ -1652,11 +1872,258 @@ class RunGate {
     state.lastStart = at;
     await sleep(at - now);
   }
+  /**
+   * Waits until a request to `url` may start: the recipe's interval, then its
+   * site's turn in the crawler's per-site throttle.
+   *
+   * @param url - Where the request goes.
+   * @returns The release of the site's lane: call it once the response arrived or the request failed.
+   */
+  async request(url) {
+    await this.throttle();
+    return this.hosts === undefined ? noop$1 : this.hosts.slot(url);
+  }
   /** The gate for a body running inside an iteration that holds a permit: sequential, same throttle. */
   nested() {
-    return new RunGate(1, this.minIntervalMs, this.shared ?? this);
+    return new RunGate(1, this.minIntervalMs, this.hosts, this.shared ?? this);
+  }
+}
+function noop$1() {}
+
+/**
+ * Spaces and bounds requests per site, shared by every recipe of a crawler,
+ * so two recipes (or two parallel iterations) that hit one site add up to one
+ * polite client rather than two. A recipe's own `limits.delayMs` still applies
+ * on top, per recipe.
+ *
+ * Like a single-lane bridge with a traffic light: whoever arrives waits for
+ * the car ahead to be far enough, and for a free lane.
+ */
+class HostThrottle {
+  config;
+  buckets = new Map();
+  domains;
+  constructor(config = {}) {
+    this.config = config;
+    this.domains = Object.entries(config.domains ?? {}).map(([domain, rule]) => [domain.toLowerCase().replace(/^\.+/, ''), rule]).sort((first, second) => second[0].length - first[0].length);
+  }
+  bucketFor(url, always = false) {
+    const host = hostOf(url);
+    if (host === undefined) return undefined;
+    const match = this.domains.find(([domain]) => host === domain || host.endsWith(`.${domain}`));
+    const key = match?.[0] ?? host;
+    let bucket = this.buckets.get(key);
+    if (bucket === undefined) {
+      const rule = {
+        ...pick(this.config),
+        ...match?.[1]
+      };
+      if (!always && !hasLimit(rule)) return undefined;
+      bucket = {
+        rule,
+        inFlight: 0,
+        waiting: [],
+        lastStart: Promise.resolve(-Infinity),
+        pausedTo: 0
+      };
+      this.buckets.set(key, bucket);
+    }
+    return bucket;
+  }
+  /** Whether any rule can hold a request back. */
+  get active() {
+    return hasLimit(this.config) || this.domains.some(([, rule]) => hasLimit(rule)) || this.buckets.size > 0;
+  }
+  /**
+   * Waits until a request to `url` may start, then holds one of its site's
+   * lanes until the returned release is called.
+   *
+   * @param url - Where the request goes; anything but `http(s):` passes at once.
+   * @returns The release: call it once, when the response arrived or the request failed.
+   */
+  async slot(url) {
+    const bucket = this.bucketFor(url);
+    if (bucket === undefined) return noop;
+    const concurrency = bucket.rule.concurrency ?? Infinity;
+    if (bucket.inFlight >= concurrency) await new Promise(resolve => {
+      bucket.waiting.push(resolve);
+    });
+    bucket.inFlight += 1;
+    const previous = bucket.lastStart;
+    const turn = (async () => {
+      const after = await previous;
+      // A pause may arrive while waiting (a Retry-After), so check it again after each sleep.
+      for (;;) {
+        const now = Date.now();
+        const at = Math.max(now, after + (bucket.rule.delayMs ?? 0), bucket.pausedTo);
+        if (at <= now) return now;
+        await sleep(at - now);
+      }
+    })();
+    bucket.lastStart = turn;
+    await turn;
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      bucket.inFlight -= 1;
+      bucket.waiting.shift()?.();
+    };
+  }
+  /**
+   * Holds every request to the site of `url` back until `untilMs` (a `Retry-After`).
+   *
+   * @param url - A URL of the site.
+   * @param untilMs - An epoch time.
+   */
+  pause(url, untilMs) {
+    const bucket = this.bucketFor(url, true);
+    if (bucket !== undefined) bucket.pausedTo = Math.max(bucket.pausedTo, untilMs);
+  }
+}
+function noop() {}
+function hostOf(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? parsed.hostname.toLowerCase() : undefined;
+  } catch {
+    return undefined;
   }
 }
+function pick(config) {
+  return {
+    delayMs: config.delayMs,
+    concurrency: config.concurrency
+  };
+}
+function hasLimit(rule) {
+  return (rule.delayMs ?? 0) > 0 || rule.concurrency !== undefined;
+}
+
+/** Statuses a server uses for "not now": timeout, too early, too many requests, and the 5xx that pass. */
+const RETRY_STATUSES = [408, 425, 429, 500, 502, 503, 504];
+/** Three tries, one second then two apart, never a wait over 30 seconds. */
+const DEFAULT_RETRY_RULE = {
+  attempts: 3,
+  backoffMs: 1000,
+  maxDelayMs: 30_000,
+  statuses: [...RETRY_STATUSES]
+};
+/**
+ * Errors that say the connection failed rather than the site answered:
+ * resets, refusals, timeouts, a DNS lookup that could not run, a proxy that
+ * dropped the tunnel. A name that does not resolve at all (`ENOTFOUND`,
+ * `ERR_NAME_NOT_RESOLVED`) is not here: retrying a typo only wastes time.
+ */
+const TRANSIENT_ERROR = /ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|socket hang up|net::ERR_(?:CONNECTION_(?:RESET|REFUSED|CLOSED|ABORTED|TIMED_OUT)|TIMED_OUT|EMPTY_RESPONSE|NETWORK_CHANGED|INTERNET_DISCONNECTED|PROXY_CONNECTION_FAILED|TUNNEL_CONNECTION_FAILED|HTTP2_PROTOCOL_ERROR|NETWORK_IO_SUSPENDED)|NS_ERROR_NET_(?:RESET|INTERRUPT|TIMEOUT)|Timeout \d+ms exceeded/;
+/**
+ * The retry rule a recipe runs with: its `limits.retry` over the crawler's
+ * default over `DEFAULT_RETRY_RULE`.
+ *
+ * @param own - The recipe's `limits.retry`.
+ * @param crawler - The crawler's `retry` option.
+ * @returns The rule.
+ */
+function resolveRetryRule(own, crawler) {
+  return {
+    ...DEFAULT_RETRY_RULE,
+    ...definedOf(crawler),
+    ...definedOf(own)
+  };
+}
+/**
+ * Whether an error is a connection failure worth another try.
+ *
+ * @param error - What the request threw.
+ * @returns The reason, or `undefined`.
+ */
+function transientError(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const match = TRANSIENT_ERROR.exec(message);
+  return match === null ? undefined : {
+    reason: match[0]
+  };
+}
+/**
+ * How long to wait before attempt `attempt + 1`: the server's `Retry-After`
+ * when it gave one, else `backoffMs` doubling per attempt with a little
+ * jitter; `undefined` when the server asks for longer than `maxDelayMs` (it
+ * means "come back much later", which a crawl cannot wait for).
+ *
+ * @param rule - The retry rule.
+ * @param attempt - The attempt that just failed, from 1.
+ * @param retryAfter - The `Retry-After` header: seconds, or an HTTP date.
+ * @param now - The current time, for dates.
+ * @returns Milliseconds, or `undefined` for no retry.
+ */
+function retryDelay(rule, attempt, retryAfter, now = Date.now()) {
+  const asked = retryAfterMs(retryAfter, now);
+  if (asked !== undefined) return asked > rule.maxDelayMs ? undefined : asked;
+  const base = rule.backoffMs * 2 ** (attempt - 1);
+  const jittered = base * (0.75 + Math.random() * 0.5);
+  return Math.min(Math.round(jittered), rule.maxDelayMs);
+}
+/**
+ * Sends a request through the gate (the recipe's rate, the site's lane), and
+ * sends it again after a pause while it fails in a passing way, up to
+ * `rule.attempts` tries in all. A `Retry-After` holds back every request to
+ * that site, not only this one. Each retry is reported as `request:retry`.
+ *
+ * Like redialling a busy number: wait a moment, dial again, give up after a
+ * few tries; and if the other end said "call back in a minute", wait that minute.
+ *
+ * @param url - Where the request goes.
+ * @param attempt - How to send it and how to judge the outcome.
+ * @param context - The recipe, gate, events and rule.
+ * @returns What the last try gave.
+ * @throws What the last try threw.
+ */
+async function withTransportRetry(url, attempt, context) {
+  const {
+    rule
+  } = context;
+  for (let tries = 1;; tries += 1) {
+    const release = await context.gate.request(url);
+    let outcome;
+    try {
+      outcome = {
+        value: await attempt.run()
+      };
+    } catch (error) {
+      outcome = {
+        error
+      };
+    } finally {
+      release();
+    }
+    const transient = tries < rule.attempts ? attempt.problem(outcome) : undefined;
+    const delay = transient === undefined ? undefined : retryDelay(rule, tries, transient.retryAfter);
+    if (transient === undefined || delay === undefined) {
+      if ('error' in outcome) throw outcome.error;
+      return outcome.value;
+    }
+    if (transient.retryAfter !== undefined) context.gate.hosts?.pause(url, Date.now() + delay);
+    context.events.emit({
+      type: 'request:retry',
+      recipeId: context.recipeId,
+      url,
+      attempt: tries + 1,
+      reason: transient.reason,
+      delayMs: delay
+    });
+    await sleep(delay);
+  }
+}
+function retryAfterMs(header, now) {
+  if (header === undefined || header.trim() === '') return undefined;
+  const seconds = Number(header.trim());
+  if (Number.isFinite(seconds)) return Math.max(0, Math.round(seconds * 1000));
+  const date = Date.parse(header);
+  return Number.isNaN(date) ? undefined : Math.max(0, date - now);
+}
+function definedOf(rule) {
+  return Object.fromEntries(Object.entries(rule ?? {}).filter(([, value]) => value !== undefined));
+}
 
 /** A block unless a recipe says otherwise: forbidden, rate limited, or an AWS WAF challenge (IMDb answers 202 with it). */
 const DEFAULT_BLOCK_RULE = {
@@ -2229,6 +2696,10 @@ function traceLine(event) {
       {
         return `${indent(1)}↻ new access lease (attempt ${event.attempt})`;
       }
+    case 'request:retry':
+      {
+        return `${indent(1)}↺ ${event.url}: ${event.reason}, try ${event.attempt} in ${event.delayMs} ms`;
+      }
     case 'captcha:detected':
       {
         return `${indent(1)}⚿ captcha ${event.kind} on ${event.url}`;
@@ -2437,26 +2908,36 @@ async function existingKeys(target) {
   });
 }
 
-/** Drops records whose key was already seen. First record wins; keyless records always pass. */
+/**
+ * Drops records whose key was already seen. First record wins; keyless
+ * records always pass. Recipes running in parallel each get their own view:
+ * under `recipe` scope they never see each other's keys, under `run` scope
+ * they share them (and whichever emits a key first keeps it).
+ */
 class DedupePolicy {
   scope;
-  seen = new Set();
+  shared = new Set();
   constructor(scope = 'run') {
     this.scope = scope;
   }
-  /** Called when an input recipe starts; forgets keys under `recipe` scope. */
-  startRecipe() {
-    if (this.scope === 'recipe') this.seen = new Set();
-  }
   /**
-   * @param record - A validated record.
-   * @returns `true` when the record repeats an earlier key and must be dropped.
+   * The de-duplication one input recipe run uses.
+   *
+   * @returns Its view: keys shared with the run, its own, or none checked.
    */
-  isDuplicate(record) {
-    if (this.scope === 'off' || record.key === null) return false;
-    if (this.seen.has(record.key)) return true;
-    this.seen.add(record.key);
-    return false;
+  forRecipe() {
+    if (this.scope === 'off') return {
+      isDuplicate: () => false
+    };
+    const seen = this.scope === 'run' ? this.shared : new Set();
+    return {
+      isDuplicate: record => {
+        if (record.key === null) return false;
+        if (seen.has(record.key)) return true;
+        seen.add(record.key);
+        return false;
+      }
+    };
   }
 }
 
@@ -4354,8 +4835,9 @@ function formatFromExtension(extension) {
 
 /**
  * Sends a `request` step: renders its templates, waits for the gate's throttle,
- * sends, checks the response against the recipe's block rule, then binds it as
- * the scope's current document (and under the step id).
+ * sends (again, after a pause, while it fails in passing: `limits.retry`),
+ * checks the response against the recipe's block rule, then binds it as the
+ * scope's current document (and under the step id).
  *
  * @param step - The request step.
  * @param scope - The scope to render in and bind into.
@@ -4368,20 +4850,29 @@ function formatFromExtension(extension) {
 async function sendRequest(step, scope, client, recipe, gate, events) {
   const lookup = path => scope.lookup(path);
   const url = resolveUrl(renderText(step.url, lookup), scope.pageState?.url);
-  await gate.throttle();
+  const request = {
+    method: step.method,
+    url,
+    query: step.query === undefined ? undefined : renderMap(step.query, lookup),
+    headers: step.headers === undefined ? undefined : renderMap(step.headers, lookup),
+    body: renderDeep(step.body, lookup),
+    as: step.as,
+    encoding: step.encoding,
+    delimiter: step.delimiter,
+    scalars: step.scalars,
+    timeoutMs: recipe.limits?.timeoutMs
+  };
+  const rule = resolveRetryRule(recipe.limits?.retry);
   let response;
   try {
-    response = await client.send({
-      method: step.method,
-      url,
-      query: step.query === undefined ? undefined : renderMap(step.query, lookup),
-      headers: step.headers === undefined ? undefined : renderMap(step.headers, lookup),
-      body: renderDeep(step.body, lookup),
-      as: step.as,
-      encoding: step.encoding,
-      delimiter: step.delimiter,
-      scalars: step.scalars,
-      timeoutMs: recipe.limits?.timeoutMs
+    response = await withTransportRetry(url, {
+      run: () => client.send(request),
+      problem: outcome => 'error' in outcome ? problemOf(outcome.error, rule.statuses) : undefined
+    }, {
+      recipeId: recipe.id,
+      gate,
+      events,
+      rule
     });
   } catch (error) {
     if (!(error instanceof HttpError)) throw error;
@@ -4431,6 +4922,14 @@ async function sendRequest(step, scope, client, recipe, gate, events) {
   });
   if (step.id !== undefined) scope.set(step.id, documentValue(response.body));
 }
+/** A retry status (with the server's `Retry-After`), or a connection that failed. */
+function problemOf(error, statuses) {
+  if (error instanceof HttpError) return statuses.includes(error.status) ? {
+    reason: `HTTP ${error.status}`,
+    retryAfter: error.headers['retry-after']
+  } : undefined;
+  return transientError(error);
+}
 /** The class names of the widgets `session.captcha` solves. Checked only when a recipe declares it. */
 const CAPTCHA_MARKUP = /\b(?:g-recaptcha|h-captcha|cf-turnstile)\b/;
 function bodyText(body) {
@@ -5713,18 +6212,34 @@ function readElements(elements) {
 
 /**
  * Runs a `goto` step: renders the URL (relative to the current page), waits for
- * the gate's throttle (`delayMs`), navigates, records the page's real URL in the
- * scope, and checks the response against the recipe's block rule.
+ * the gate's throttle (`delayMs`), navigates (again, after a pause, while it
+ * fails in passing: `limits.retry`), records the page's real URL in the scope,
+ * and checks the response against the recipe's block rule.
  *
  * @throws BlockedError when the response is a block.
  */
 async function navigate(step, page, scope, recipe, gate, events) {
   const target = renderText(step.url, path => scope.lookup(path));
   const url = new URL(target, scope.pageState?.url ?? page.url()).href;
-  await gate.throttle();
-  const response = await page.goto(url, {
-    waitUntil: step.waitUntil,
-    timeout: recipe.limits?.timeoutMs
+  const rule = resolveRetryRule(recipe.limits?.retry);
+  const response = await withTransportRetry(url, {
+    run: () => page.goto(url, {
+      waitUntil: step.waitUntil,
+      timeout: recipe.limits?.timeoutMs
+    }),
+    problem: outcome => {
+      if ('error' in outcome) return transientError(outcome.error);
+      const status = outcome.value?.status();
+      return status !== undefined && rule.statuses.includes(status) ? {
+        reason: `HTTP ${status}`,
+        retryAfter: outcome.value?.headers()['retry-after']
+      } : undefined;
+    }
+  }, {
+    recipeId: recipe.id,
+    gate,
+    events,
+    rule
   });
   scope.setPage({
     url: page.url()
@@ -5874,8 +6389,13 @@ class WebStepRunner {
     const link = this.page.locator(next.selector).first();
     if (!(await appears(link, NEXT_LINK_TIMEOUT_MS))) return null;
     const before = this.page.url();
-    await link.click();
-    await this.page.waitForLoadState();
+    const release = await this.gate.request(before);
+    try {
+      await link.click();
+      await this.page.waitForLoadState();
+    } finally {
+      release();
+    }
     if (this.page.url() === before) await this.page.waitForTimeout(NEXT_LINK_TIMEOUT_MS / 4);
     this.events.emit({
       type: 'page:visit',
@@ -5889,6 +6409,23 @@ class WebStepRunner {
       url: this.page.url()
     };
   }
+  /**
+   * A runner on a new tab of the same context, for one parallel iteration:
+   * it shares cookies, the gate and the captcha guard; disposing it closes the tab only.
+   *
+   * @returns The forked runner.
+   */
+  async fork() {
+    const {
+      context
+    } = this.session;
+    const page = await context.newPage();
+    const viewport = this.recipe.session?.viewport;
+    if (viewport !== undefined) await page.setViewportSize(viewport);
+    return new WebStepRunner(new BrowserSession(context, page, async () => {
+      await page.close();
+    }), this.recipe, this.events, this.gate, this.captcha);
+  }
   async elements(selector) {
     return snapshotElements(selector, this.page);
   }
@@ -5925,16 +6462,30 @@ function accessOptions(lease, headers) {
  * The bootstrap runs through the same access lease as the crawl that follows,
  * so a login and the requests that use its cookies come from one IP.
  *
+ * With `session.browserProfile`, the bootstrap runs in that profile, and
+ * without a bootstrap the profile's own cookies and storage are the state: an
+ * api recipe picks up a login a browser left in the profile.
+ *
  * @param recipe - The input recipe.
  * @param deps - Browser, hooks, events.
  * @param lease - The recipe run's access; direct when omitted.
  * @param captcha - Solves the bootstrap's captchas (a login form's).
+ * @param owner - The recipe run, which a browser profile is held by.
  * @returns The state, or `undefined` when the recipe declares none.
  */
-async function resolveStorageState(recipe, deps, lease, captcha) {
+async function resolveStorageState(recipe, deps, lease, captcha, owner = {}) {
   const saved = await readSavedState(recipe, deps);
+  if (saved !== undefined) return saved;
   const session = recipe.session;
-  if (saved !== undefined || session?.bootstrap === undefined) return saved;
+  if (session?.browserProfile !== undefined) {
+    const browserSession = await openBrowserProfile(recipe, deps, lease, owner);
+    try {
+      return session.bootstrap === undefined ? await browserSession.storageState() : await runBootstrap(recipe, browserSession, deps, captcha);
+    } finally {
+      await browserSession.close();
+    }
+  }
+  if (session?.bootstrap === undefined) return undefined;
   const browser = await deps.browser();
   const browserSession = await browser.newSession({
     cookies: session.cookies,
@@ -5948,6 +6499,27 @@ async function resolveStorageState(recipe, deps, lease, captcha) {
     await browserSession.close();
   }
 }
+/**
+ * Opens the recipe's `session.browserProfile` with its session options and
+ * the lease's proxy.
+ *
+ * @param recipe - A recipe with `session.browserProfile`.
+ * @param deps - For `profiles`.
+ * @param lease - The access lease.
+ * @param owner - The recipe run.
+ * @returns The session in the profile.
+ */
+async function openBrowserProfile(recipe, deps, lease, owner) {
+  const session = recipe.session;
+  const name = session?.browserProfile ?? '';
+  if (deps.profiles === undefined) throw new Error(`recipe "${recipe.id}" uses browser profile "${name}", but this crawler has no profiles directory (CrawlOptions.profilesDir)`);
+  return deps.profiles.open(name, {
+    cookies: session?.cookies,
+    userAgent: session?.userAgent,
+    viewport: session?.viewport,
+    ...accessOptions(lease, session?.headers)
+  }, owner);
+}
 /**
  * The storage state saved by an earlier bootstrap (`session.storageStatePath`), if the recipe names one.
  *
@@ -5977,7 +6549,7 @@ async function runBootstrap(recipe, browserSession, deps, captcha) {
     cookies: [],
     origins: []
   };
-  const runner = new WebStepRunner(browserSession, recipe, deps.events, undefined, captcha);
+  const runner = new WebStepRunner(browserSession, recipe, deps.events, new RunGate(1, recipe.limits?.delayMs ?? 0, deps.hosts), captcha);
   const scope = new ExtractionScope();
   scope.set('vars', recipe.vars ?? {});
   scope.set('start', {
@@ -6082,6 +6654,66 @@ class RotatingRunner {
       throw error;
     }
   }
+  /**
+   * A runner for one parallel iteration, forked from whichever runner is
+   * current when it runs a step: after a rotation it forks again from the new
+   * one, since the old context is gone (or going). Blocks are noted and
+   * rotated like the main runner's.
+   *
+   * @returns The iteration's runner.
+   */
+  async fork() {
+    // `own`: a tab this iteration opened and must close; an api runner is shared and never disposed here.
+    let forked;
+    const disposeForked = async () => {
+      if (forked?.own === true) await disposeQuietly(forked.runner);
+      forked = undefined;
+    };
+    const current = async () => {
+      if (forked?.generation === this.generation) return forked.runner;
+      await disposeForked();
+      const inner = this.inner;
+      forked = inner.fork === undefined ? {
+        generation: this.generation,
+        runner: inner,
+        own: false
+      } : {
+        generation: this.generation,
+        runner: await inner.fork(),
+        own: true
+      };
+      return forked.runner;
+    };
+    return {
+      runLeaf: async (step, scope) => {
+        const generation = this.generation;
+        try {
+          const runner = await current();
+          await runner.runLeaf(step, scope);
+        } catch (error) {
+          this.note(error, generation);
+          throw error;
+        }
+      },
+      nextPage: async (next, scope) => {
+        const generation = this.generation;
+        try {
+          const runner = await current();
+          return await runner.nextPage(next, scope);
+        } catch (error) {
+          this.note(error, generation);
+          throw error;
+        }
+      },
+      elements: async (selector, scope) => {
+        const runner = await current();
+        if (runner.elements === undefined) throw new Error('forEach over selector iterates live elements and needs a browser; this recipe runs in api mode');
+        return runner.elements(selector, scope);
+      },
+      rotate: error => this.rotate(error),
+      dispose: disposeForked
+    };
+  }
   async elements(selector, scope) {
     if (this.inner.elements === undefined) throw new Error('forEach over selector iterates live elements and needs a browser; this recipe runs in api mode');
     return this.inner.elements(selector, scope);
@@ -6130,12 +6762,19 @@ class RotatingRunner {
  * the sink sees one record at a time and `maxRecords` is exact: once reached,
  * every later emit returns `stop` before mapping.
  *
- * @param input - The input recipe.
+ * @param recipe - The input recipe.
  * @param output - The output recipe it feeds.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @returns What happened.
  */
-async function runInputRecipe(input, output, deps) {
+async function runInputRecipe(recipe, output, deps) {
+  const input = {
+    ...recipe,
+    limits: {
+      ...recipe.limits,
+      retry: resolveRetryRule(recipe.limits?.retry, deps.retry)
+    }
+  };
   const started = Date.now();
   const report = {
     recipeId: input.id,
@@ -6154,8 +6793,8 @@ async function runInputRecipe(input, output, deps) {
     failed: 0
   };
   const limits = input.limits ?? {};
-  // A web recipe drives one page, so only api mode runs iterations in parallel.
-  const gate = new RunGate(input.mode === 'web' ? 1 : limits.concurrency ?? 1, limits.delayMs ?? 0);
+  // Parallel iterations: requests in api mode, tabs of the recipe's context in web mode.
+  const gate = new RunGate(limits.concurrency ?? 1, limits.delayMs ?? 0, deps.hosts);
   let stopped = false;
   let chain = Promise.resolve();
   const unsubscribe = deps.events.subscribe(event => {
@@ -6170,7 +6809,7 @@ async function runInputRecipe(input, output, deps) {
     recipeId: input.id,
     mode: input.mode
   });
-  deps.dedupe.startRecipe();
+  const dedupe = deps.dedupe.forRecipe();
   let runner;
   try {
     const onBlock = input.session?.onBlock;
@@ -6278,7 +6917,7 @@ async function runInputRecipe(input, output, deps) {
           url,
           key: record.key
         });
-      } else if (deps.dedupe.isDuplicate(record)) {
+      } else if (dedupe.isDuplicate(record)) {
         report.duplicates += 1;
         deps.events.emit({
           type: 'record:duplicate',
@@ -6370,7 +7009,8 @@ async function openRunner(input, deps, context, lease) {
     lease
   });
   if (lease.cdp !== undefined) return openRemoteRunner(input, deps, context, lease, lease.cdp);
-  const storageState = await resolveStorageState(input, deps, lease, captcha);
+  if (input.mode === 'web' && input.session?.browserProfile !== undefined) return openProfileRunner(input, deps, context, lease, captcha);
+  const storageState = await resolveStorageState(input, deps, lease, captcha, context);
   const session = input.session;
   const access = accessOptions(lease, session?.headers);
   if (input.mode === 'web') {
@@ -6394,6 +7034,28 @@ async function openRunner(input, deps, context, lease) {
   });
   return new ApiStepRunner(client, input, deps.events, gate);
 }
+/**
+ * A web runner in a persistent browser profile. The bootstrap runs in the
+ * same browser as the crawl, and what both leave behind (cookies, storage)
+ * stays in the profile for the next run.
+ */
+async function openProfileRunner(input, deps, context, lease, captcha) {
+  const saved = await readSavedState(input, deps);
+  const browserSession = await openBrowserProfile(saved === undefined ? input : {
+    ...input,
+    session: {
+      ...input.session,
+      cookies: [...saved.cookies, ...(input.session?.cookies ?? [])]
+    }
+  }, deps, lease, context);
+  try {
+    if (saved === undefined && input.session?.bootstrap !== undefined) await runBootstrap(input, browserSession, deps, captcha);
+  } catch (error) {
+    await browserSession.close();
+    throw error;
+  }
+  return new WebStepRunner(browserSession, input, deps.events, context.gate, captcha);
+}
 /**
  * A web runner in a remote browser. The bootstrap runs in the same remote
  * session as the crawl: providers tie the IP and fingerprint to the
@@ -6408,6 +7070,7 @@ async function openRemoteRunner(input, deps, context, lease, cdp) {
     lease
   });
   if (input.mode === 'api') throw new AccessConfigError(`recipe "${input.id}" runs in api mode, but access profile "${lease.profile}" is a remote browser; api recipes need a proxy profile`);
+  if (input.session?.browserProfile !== undefined) throw new AccessConfigError(`recipe "${input.id}" uses browser profile "${input.session.browserProfile}", which needs a local browser, but access profile "${lease.profile}" is a remote browser`);
   const session = input.session;
   const storageState = await readSavedState(input, deps);
   const browserSession = await BrowserClient.connectOverCDP(cdp, {
@@ -6426,27 +7089,42 @@ async function openRemoteRunner(input, deps, context, lease, cdp) {
 }
 
 /**
- * Runs every input recipe of a set, one after another, into one sink.
+ * Runs every input recipe of a set into one sink, `parallel` at a time
+ * (default one after another). Reports come back in the set's order whatever
+ * order the recipes finish in. Under `onRecipeError: 'stop'`, a failed recipe
+ * stops the ones not started yet; those already running finish.
  *
  * @param set - The bound recipes.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @param onRecipeError - Whether a failed recipe stops the run.
+ * @param parallel - How many input recipes run at once.
  * @returns The report.
  */
-async function runCrawl(set, deps, onRecipeError) {
+async function runCrawl(set, deps, onRecipeError, parallel = 1) {
   const started = Date.now();
   await deps.sink.open(set.output);
-  const recipes = [];
+  const reports = [];
   let sink;
   try {
-    for (const input of set.inputs) {
-      const report = await runInputRecipe(input, set.output, deps);
-      recipes.push(report);
-      if (onRecipeError === 'stop' && report.error !== undefined) break;
-    }
+    let next = 0;
+    let stopped = false;
+    const lane = async () => {
+      while (!stopped && next < set.inputs.length) {
+        const index = next;
+        next += 1;
+        const report = await runInputRecipe(set.inputs[index], set.output, deps);
+        reports[index] = report;
+        if (onRecipeError === 'stop' && report.error !== undefined) stopped = true;
+      }
+    };
+    const lanes = Math.max(1, Math.min(parallel, set.inputs.length));
+    await Promise.all(Array.from({
+      length: lanes
+    }, lane));
   } finally {
     sink = await deps.sink.close();
   }
+  const recipes = reports.filter(report => report !== undefined);
   return {
     outputId: set.output.id,
     recipes,
@@ -6471,6 +7149,8 @@ function createCrawler(options = {}) {
   const access = new AccessBroker(options.access, options.accessPlugins);
   const hooks = new HookRegistry(options.hooks);
   const captchaSolvers = new CaptchaSolverRegistry(options.captchaSolvers);
+  const hosts = new HostThrottle(options.throttle ?? options.access?.throttle);
+  const profiles = new BrowserProfiles(options.profilesDir ?? resolve$1(options.storageStateDir ?? '.', '.opencraw', 'profiles'), options.browser);
   const events = new EventBus(options.onEvent);
   let browser;
   const launch = () => {
@@ -6489,8 +7169,11 @@ function createCrawler(options = {}) {
       debug: options.debug === true,
       access,
       captchaSolvers,
+      hosts,
+      profiles,
+      retry: options.retry,
       ignoreHTTPSErrors: options.browser?.ignoreHTTPSErrors
-    }, options.onRecipeError ?? 'continue'),
+    }, options.onRecipeError ?? 'continue', options.parallel ?? 1),
     async close() {
       const launched = browser;
       browser = undefined;
@@ -6962,13 +7645,21 @@ const sessionSpecSchema = z.strictObject({
   access: sessionAccessSchema.optional(),
   blockedWhen: blockRuleSchema.optional(),
   onBlock: blockRotationSchema.optional(),
-  captcha: captchaSettingsSchema.optional()
+  captcha: captchaSettingsSchema.optional(),
+  browserProfile: z.string().regex(/^[\w-]+$/, 'a browser profile name is letters, digits, hyphens and underscores').optional()
+});
+const retryRuleSchema = z.strictObject({
+  attempts: z.int().min(1).max(10).optional(),
+  backoffMs: z.int().nonnegative().optional(),
+  maxDelayMs: z.int().nonnegative().optional(),
+  statuses: z.array(z.int().min(400).max(599)).optional()
 });
 const limitsSchema = z.strictObject({
   maxRecords: z.int().positive().optional(),
   delayMs: z.int().nonnegative().optional(),
   timeoutMs: z.int().positive().optional(),
-  concurrency: z.int().min(1).max(64).optional()
+  concurrency: z.int().min(1).max(64).optional(),
+  retry: retryRuleSchema.optional()
 });
 const inputRecipeSchema = z.strictObject({
   $schema: z.string().optional(),
@@ -7499,5 +8190,5 @@ function isSameOutput(document, output) {
   return recipeKindOf(document.content) === 'output' && document.content.id === output.id;
 }
 
-export { ACCESS_PRESETS, AccessBroker, AccessConfigError, BrowserClient, BrowserSession, CaptchaError, DEFAULT_CAPTCHA_SELECTOR, HttpClient, HttpError, MappingFailedError, PdfReadError, RecipeBindingError, RecipeSet, RecipeValidationError, RecordRejectedError, StepFailure, TransformError, UnknownHookError, accessConfigJsonSchema, accessConfigSchema, bindRecipeSet, createCrawler, csvWorkbook, deckText, detectChallenge, detectDelimiter, fillDown, findDeckTables, findGridTables, findTables, htmlTableSheets, inputRecipeJsonSchema, inputRecipeSchema, isDeckDocument, isWorkbookDocument, jsonLinesSink, loadAccessConfig, loadRecipeSet, loadRecipes, memorySink, outputRecipeJsonSchema, outputRecipeSchema, parseCsv, parseInputRecipe, parseOutputRecipe, pdfText, readMarkdown, readPdf, readRecipeSource, readYaml, traceLine, tryParseJson, validateBinding, workbookText };
+export { ACCESS_PRESETS, AccessBroker, AccessConfigError, BrowserClient, BrowserSession, CaptchaError, DEFAULT_CAPTCHA_SELECTOR, DEFAULT_RETRY_RULE, HostThrottle, HttpClient, HttpError, MappingFailedError, PdfReadError, RecipeBindingError, RecipeSet, RecipeValidationError, RecordRejectedError, StepFailure, TransformError, UnknownHookError, accessConfigJsonSchema, accessConfigSchema, bindRecipeSet, createCrawler, csvWorkbook, deckText, detectChallenge, detectDelimiter, fillDown, findDeckTables, findGridTables, findTables, htmlTableSheets, inputRecipeJsonSchema, inputRecipeSchema, isDeckDocument, isWorkbookDocument, jsonLinesSink, loadAccessConfig, loadRecipeSet, loadRecipes, memorySink, outputRecipeJsonSchema, outputRecipeSchema, parseCsv, parseInputRecipe, parseOutputRecipe, pdfText, readMarkdown, readPdf, readRecipeSource, readYaml, retryRuleSchema, throttleConfigSchema, traceLine, tryParseJson, validateBinding, workbookText };
 //# sourceMappingURL=index.esm.js.map