@opencraw/core 0.1.2 → 0.1.3

package/dist/src/step-flow/transport-retry.policy.d.ts

@@ -0,0 +1,76 @@
+import type { EventBus } from '../crawl-events/index.js';
+import type { RetryRule } from '../recipe-schema/index.js';
+import type { RunGate } from './run-gate.policy.js';
+/** A retry rule with every default filled in. */
+export type ResolvedRetryRule = Required<RetryRule>;
+/** Statuses a server uses for "not now": timeout, too early, too many requests, and the 5xx that pass. */
+export declare const RETRY_STATUSES: readonly number[];
+/** Three tries, one second then two apart, never a wait over 30 seconds. */
+export declare const DEFAULT_RETRY_RULE: ResolvedRetryRule;
+/** Why an attempt may be worth repeating, and the server's own `Retry-After`, if it sent one. */
+export interface Transient {
+    reason: string;
+    retryAfter?: string;
+}
+/** One try of a request, and how to tell a passing failure in what it gave. */
+export interface TransportAttempt<T> {
+    run: () => Promise<T>;
+    /** A transient problem in the outcome (a retry status, a connection error), or `undefined` when the outcome stands. */
+    problem: (outcome: {
+        value: T;
+    } | {
+        error: unknown;
+    }) => Transient | undefined;
+}
+export interface RetryContext {
+    recipeId: string;
+    gate: RunGate;
+    events: EventBus;
+    rule: ResolvedRetryRule;
+}
+/**
+ * The retry rule a recipe runs with: its `limits.retry` over the crawler's
+ * default over `DEFAULT_RETRY_RULE`.
+ *
+ * @param own - The recipe's `limits.retry`.
+ * @param crawler - The crawler's `retry` option.
+ * @returns The rule.
+ */
+export declare function resolveRetryRule(own?: RetryRule, crawler?: RetryRule): ResolvedRetryRule;
+/**
+ * Whether an error is a connection failure worth another try.
+ *
+ * @param error - What the request threw.
+ * @returns The reason, or `undefined`.
+ */
+export declare function transientError(error: unknown): Transient | undefined;
+/**
+ * How long to wait before attempt `attempt + 1`: the server's `Retry-After`
+ * when it gave one, else `backoffMs` doubling per attempt with a little
+ * jitter; `undefined` when the server asks for longer than `maxDelayMs` (it
+ * means "come back much later", which a crawl cannot wait for).
+ *
+ * @param rule - The retry rule.
+ * @param attempt - The attempt that just failed, from 1.
+ * @param retryAfter - The `Retry-After` header: seconds, or an HTTP date.
+ * @param now - The current time, for dates.
+ * @returns Milliseconds, or `undefined` for no retry.
+ */
+export declare function retryDelay(rule: ResolvedRetryRule, attempt: number, retryAfter?: string, now?: number): number | undefined;
+/**
+ * Sends a request through the gate (the recipe's rate, the site's lane), and
+ * sends it again after a pause while it fails in a passing way, up to
+ * `rule.attempts` tries in all. A `Retry-After` holds back every request to
+ * that site, not only this one. Each retry is reported as `request:retry`.
+ *
+ * Like redialling a busy number: wait a moment, dial again, give up after a
+ * few tries; and if the other end said "call back in a minute", wait that minute.
+ *
+ * @param url - Where the request goes.
+ * @param attempt - How to send it and how to judge the outcome.
+ * @param context - The recipe, gate, events and rule.
+ * @returns What the last try gave.
+ * @throws What the last try threw.
+ */
+export declare function withTransportRetry<T>(url: string, attempt: TransportAttempt<T>, context: RetryContext): Promise<T>;
+//# sourceMappingURL=transport-retry.policy.d.ts.map

package/dist/src/web-steps/navigate.use-case.d.ts

@@ -5,8 +5,9 @@ import type { GotoStep, InputRecipe } from '../recipe-schema/index.js';
 import type { RunGate } from '../step-flow/index.js';
 /**
  * Runs a `goto` step: renders the URL (relative to the current page), waits for
- * the gate's throttle (`delayMs`), navigates, records the page's real URL in the
- * scope, and checks the response against the recipe's block rule.
+ * the gate's throttle (`delayMs`), navigates (again, after a pause, while it
+ * fails in passing: `limits.retry`), records the page's real URL in the scope,
+ * and checks the response against the recipe's block rule.
  *
  * @throws BlockedError when the response is a block.
  */

package/dist/src/web-steps/run-web-step.use-case.d.ts

@@ -1,4 +1,4 @@
-import type { BrowserSession } from '../browser-session/index.js';
+import { BrowserSession } from '../browser-session/index.js';
 import type { CaptchaGuard } from '../captcha/index.js';
 import type { EventBus } from '../crawl-events/index.js';
 import type { ExtractionScope, LiveElement } from '../extraction-scope/index.js';
@@ -24,6 +24,13 @@ export declare class WebStepRunner implements StepRunner {
     private visit;
     runLeaf(step: Step, scope: ExtractionScope): Promise<void>;
     nextPage(next: PaginateNext, scope: ExtractionScope): Promise<NextPageResult>;
+    /**
+     * A runner on a new tab of the same context, for one parallel iteration:
+     * it shares cookies, the gate and the captcha guard; disposing it closes the tab only.
+     *
+     * @returns The forked runner.
+     */
+    fork(): Promise<WebStepRunner>;
     elements(selector: string): Promise<LiveElement[]>;
     dispose(): Promise<void>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/core",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",