@scrapa/sdk 0.1.0

package/src/client.ts

@@ -0,0 +1,227 @@
+// The transport under every generated method: one request, one `{ data }` unwrap, one error
+// taxonomy.
+//
+// The retry policy is the opinionated part, and it is narrow on purpose. A scrape that we
+// broke was already refunded, so repeating it costs nothing; a rate limit says when it
+// clears. But a call that timed out on YOUR side may well have answered on ours — repeating
+// it reads the page twice and charges twice — so the client hands that one back with its
+// code and lets you decide. The default timeout sits above the API's own ceiling, which is
+// what makes a client-side timeout genuinely exceptional rather than routine.
+import { ScrapaError, type ClientErrorCode } from "./errors.ts";
+import { HEADERS, type ErrorCode, type UsageMeta } from "./generated/contract.ts";
+import { DEFAULT_BASE_URL, GeneratedOperations, SDK_VERSION } from "./generated/operations.ts";
+
+export interface ScrapaOptions {
+    /** An `sc_` key from the console. */
+    apiKey: string;
+    /** Defaults to the hosted API. */
+    baseUrl?: string;
+    /** Per-attempt timeout in ms. Default 65000 — above the API's own 60s ceiling for a
+     *  scrape, so this fires only when something is genuinely wrong. */
+    timeout?: number;
+    /** Extra attempts for the failures a repeat can fix. Default 2, 0 disables. */
+    maxRetries?: number;
+    /** Swap the fetch implementation (tests, a proxy, a custom agent). */
+    fetch?: typeof globalThis.fetch;
+}
+
+/** What a generated method hands the transport. */
+export interface RequestSpec {
+    method: "GET" | "POST" | "DELETE";
+    /** `/sessions/{id}` — a `{name}` is filled from the params and dropped from the rest. */
+    path: string;
+    /** Safe to repeat: a refunded failure can be retried without charging twice. */
+    idempotent: boolean;
+}
+
+/** Per-call overrides. */
+export interface RequestOptions {
+    /** Abort this call — yours to control, and the API stops paying for egress when it fires. */
+    signal?: AbortSignal;
+    /** Override the client's timeout for one call. */
+    timeout?: number;
+    /** What the call cost and what is left, as the answer carries it. A method returns the
+     *  page itself, so this is where the meter goes for the callers who want it — per call,
+     *  rather than a mutable `lastUsage` that two concurrent scrapes would fight over. */
+    onUsage?: (usage: UsageMeta) => void;
+}
+
+/** A Retry-After we would honour beyond this is not a retry, it is a hang: the caller gets
+ *  the error (with `retryAfter` on it) and schedules it themselves. */
+const MAX_RETRY_DELAY_MS = 10_000;
+
+/** What the API answers with when it never got to answer at all. */
+function statusCode(status: number): ErrorCode {
+    if (status === 503) return "SERVICE_UNAVAILABLE";
+    if (status === 404) return "NOT_FOUND";
+    if (status === 401) return "UNAUTHORIZED";
+    return "INTERNAL_ERROR";
+}
+
+/** Seconds from `Retry-After`, or from the error's own details. */
+function retryAfterOf(header: string | null, details: unknown): number | undefined {
+    const fromDetails =
+        typeof details === "object" && details !== null
+            ? (details as { retryAfterSecs?: unknown }).retryAfterSecs
+            : undefined;
+    if (typeof fromDetails === "number" && Number.isFinite(fromDetails)) return fromDetails;
+    const seconds = header === null ? NaN : Number(header);
+    return Number.isFinite(seconds) ? seconds : undefined;
+}
+
+function numberHeader(response: Response, name: string): number | undefined {
+    const raw = response.headers.get(name);
+    if (raw === null) return undefined;
+    const value = Number(raw);
+    return Number.isFinite(value) ? value : undefined;
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+export class Scrapa extends GeneratedOperations {
+    private readonly apiKey: string;
+    private readonly baseUrl: string;
+    private readonly timeout: number;
+    private readonly maxRetries: number;
+    private readonly fetchImpl: typeof globalThis.fetch;
+
+    constructor(options: ScrapaOptions | string) {
+        super();
+        const opts = typeof options === "string" ? { apiKey: options } : options;
+        if (!opts.apiKey)
+            throw new TypeError(
+                "scrapa: an API key is required — create one at https://console.scrapa.app/keys",
+            );
+        this.apiKey = opts.apiKey;
+        // A trailing slash would double up against a path's leading one.
+        this.baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+        this.timeout = opts.timeout ?? 65_000;
+        this.maxRetries = opts.maxRetries ?? 2;
+        const impl = opts.fetch ?? globalThis.fetch;
+        if (typeof impl !== "function")
+            throw new TypeError(
+                "scrapa: no global fetch — use Node 20+, or pass `fetch` in the options.",
+            );
+        // Bound: an unbound global fetch throws "Illegal invocation" in browsers.
+        this.fetchImpl = opts.fetch ?? impl.bind(globalThis);
+    }
+
+    protected async request<T>(
+        spec: RequestSpec,
+        params: object = {},
+        opts: RequestOptions = {},
+    ): Promise<T> {
+        const retries = spec.idempotent ? this.maxRetries : 0;
+        for (let attempt = 0; ; attempt++) {
+            try {
+                return await this.attempt<T>(spec, params, opts);
+            } catch (error) {
+                if (!(error instanceof ScrapaError) || attempt >= retries) throw error;
+                const delay = this.retryDelay(error, attempt);
+                if (delay === undefined) throw error;
+                await sleep(delay);
+            }
+        }
+    }
+
+    /** How long to wait before repeating a call, or undefined to give up now. */
+    private retryDelay(error: ScrapaError, attempt: number): number | undefined {
+        // A timeout on OUR side may have answered on theirs; repeating it would read the
+        // page — and charge for it — twice. The caller gets the error and decides.
+        const worthRepeating = error.retryable && error.code !== "TIMEOUT";
+        if (!worthRepeating) return undefined;
+        if (error.retryAfter !== undefined) {
+            const ms = error.retryAfter * 1_000;
+            return ms > MAX_RETRY_DELAY_MS ? undefined : ms;
+        }
+        // Exponential with jitter, so a fleet of clients does not resynchronise onto the
+        // same second after an outage.
+        return Math.min(250 * 2 ** attempt, 4_000) * (0.5 + Math.random());
+    }
+
+    private async attempt<T>(spec: RequestSpec, params: object, opts: RequestOptions): Promise<T> {
+        const rest: Record<string, unknown> = { ...params };
+        // A templated segment is spent on the URL, not sent again as a query parameter.
+        const path = spec.path.replace(/\{(\w+)\}/g, (_, key: string) => {
+            const value = rest[key];
+            delete rest[key];
+            return encodeURIComponent(String(value ?? ""));
+        });
+        const url = new URL(this.baseUrl + path);
+        const hasBody = spec.method === "POST";
+        if (!hasBody)
+            for (const [key, value] of Object.entries(rest))
+                if (value !== undefined && value !== null) url.searchParams.set(key, String(value));
+
+        const headers: Record<string, string> = {
+            authorization: `Bearer ${this.apiKey}`,
+            accept: "application/json",
+            [HEADERS.client]: `scrapa-js/${SDK_VERSION}`,
+        };
+        if (hasBody) headers["content-type"] = "application/json";
+
+        const timeout = AbortSignal.timeout(opts.timeout ?? this.timeout);
+        let response: Response;
+        try {
+            response = await this.fetchImpl(url.toString(), {
+                method: spec.method,
+                headers,
+                ...(hasBody ? { body: JSON.stringify(rest) } : undefined),
+                signal: opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout,
+            });
+        } catch (cause) {
+            // The caller's own abort is theirs, not a failure of ours to describe.
+            if (opts.signal?.aborted) throw cause;
+            const timedOut = cause instanceof Error && cause.name === "TimeoutError";
+            throw new ScrapaError({
+                code: (timedOut ? "TIMEOUT" : "NETWORK_ERROR") satisfies ClientErrorCode,
+                status: 0,
+                message: timedOut
+                    ? `scrapa: no answer within ${opts.timeout ?? this.timeout}ms — the page may still have been read and charged.`
+                    : `scrapa: could not reach ${url.host}.`,
+                cause,
+            });
+        }
+
+        const requestId = response.headers.get(HEADERS.requestId) ?? undefined;
+        const text = await response.text();
+        let payload: unknown;
+        try {
+            payload = text ? (JSON.parse(text) as unknown) : undefined;
+        } catch {
+            payload = undefined;
+        }
+
+        if (response.ok) {
+            const answer = payload as { data: T; meta?: UsageMeta };
+            if (opts.onUsage && answer.meta) opts.onUsage(answer.meta);
+            return answer.data;
+        }
+
+        const envelope =
+            typeof payload === "object" && payload !== null
+                ? (payload as { error?: Record<string, unknown> }).error
+                : undefined;
+        const details = envelope?.details;
+        throw new ScrapaError({
+            code: (envelope?.code as ErrorCode | undefined) ?? statusCode(response.status),
+            status: response.status,
+            // A proxy or a cold start can answer HTML; say what actually came back rather
+            // than pretending the body was ours.
+            message:
+                typeof envelope?.message === "string"
+                    ? envelope.message
+                    : `scrapa: ${spec.method} ${spec.path} failed with ${response.status}.`,
+            messageKey: typeof envelope?.messageKey === "string" ? envelope.messageKey : undefined,
+            params: envelope?.params as Record<string, string | number> | undefined,
+            details: envelope ? details : text.slice(0, 500) || undefined,
+            retryAfter: retryAfterOf(response.headers.get(HEADERS.retryAfter), details),
+            requestId,
+            // Every metered answer says what it cost, refusals included: a blocked page
+            // reports 0, and that is how "blocked pages cost nothing" is verifiable rather
+            // than asserted.
+            cost: numberHeader(response, HEADERS.cost),
+            creditsRemaining: numberHeader(response, HEADERS.creditsRemaining),
+        });
+    }
+}

package/src/errors.ts

@@ -0,0 +1,83 @@
+// Every failure this client can raise, as one class.
+//
+// The API answers a refusal with a machine-readable code, and acting on the code is the
+// whole point: it says WHO said no, which is what decides the caller's next move. The site
+// blocked the page (try another engine, pool or country), our policy refused the URL
+// (retrying cannot help), a limit needs waiting out, or we broke it (retry as-is, it was
+// refunded). Collapse that into `Error("Request failed")` and the caller is left guessing.
+import { OUTCOME_OF, type ErrorCode } from "./generated/contract.ts";
+
+/** Failures that never reached the API, so they carry no server code. */
+export type ClientErrorCode = "NETWORK_ERROR" | "TIMEOUT";
+
+export interface ScrapaErrorInit {
+    code: ErrorCode | ClientErrorCode;
+    message: string;
+    /** HTTP status, or 0 when no answer arrived. */
+    status: number;
+    messageKey?: string;
+    params?: Record<string, string | number>;
+    details?: unknown;
+    /** Seconds until the call is worth making again, when the API said. */
+    retryAfter?: number;
+    /** `X-Request-ID` — quote it to support and we can find the call. */
+    requestId?: string;
+    /** What this call cost. Zero on every refusal the API charges nothing for, which is
+     *  every refusal except a page that answered. */
+    cost?: number;
+    creditsRemaining?: number;
+    cause?: unknown;
+}
+
+export class ScrapaError extends Error {
+    readonly code: ErrorCode | ClientErrorCode;
+    readonly status: number;
+    readonly messageKey?: string;
+    readonly params?: Record<string, string | number>;
+    readonly details?: unknown;
+    readonly retryAfter?: number;
+    readonly requestId?: string;
+    readonly cost?: number;
+    readonly creditsRemaining?: number;
+
+    constructor(init: ScrapaErrorInit) {
+        super(init.message, init.cause !== undefined ? { cause: init.cause } : undefined);
+        this.name = "ScrapaError";
+        this.code = init.code;
+        this.status = init.status;
+        this.messageKey = init.messageKey;
+        this.params = init.params;
+        this.details = init.details;
+        this.retryAfter = init.retryAfter;
+        this.requestId = init.requestId;
+        this.cost = init.cost;
+        this.creditsRemaining = init.creditsRemaining;
+    }
+
+    /**
+     * Who said no — the one thing that decides what to do next.
+     *
+     *   "blocked" the site refused the page. It cost you nothing: try another engine, pool
+     *             or country. The same call again will be refused again.
+     *   "denied"  our policy refused this URL (a private address, the acceptable-use list).
+     *             Retrying cannot help; change the URL.
+     *   "broken"  we broke it, and already refunded it. Worth one retry as-is.
+     *
+     * `undefined` for the refusals that were the caller's own doing — a bad field, a missing
+     * key, a spent quota — which are not outcomes of a page at all.
+     */
+    get outcome(): "blocked" | "denied" | "broken" | undefined {
+        return OUTCOME_OF[this.code as ErrorCode];
+    }
+
+    /** True when repeating the identical call could work. A limit that clears by waiting
+     *  and anything we broke, never a verdict on the page itself. */
+    get retryable(): boolean {
+        return (
+            this.code === "NETWORK_ERROR" ||
+            this.code === "TIMEOUT" ||
+            this.code === "RATE_LIMIT_EXCEEDED" ||
+            this.outcome === "broken"
+        );
+    }
+}