@scrapa/sdk 0.2.1 → 0.3.0

package/src/client.ts

@@ -116,11 +116,11 @@ export class Scrapa extends GeneratedOperations {
         const opts = typeof options === "string" ? { apiKey: options } : options;
         if (!opts.apiKey)
             throw new TypeError(
-                "scrapa: an API key is required — create one at https://console.scrapa.app/keys",
+                "scrapa: an API key is required. Create one at https://console.scrapa.app/keys",
             );
         if (typeof (opts.fetch ?? globalThis.fetch) !== "function")
             throw new TypeError(
-                "scrapa: no global fetch — use Node 20+, or pass `fetch` in the options.",
+                "scrapa: no global fetch. Use Node 20+, or pass `fetch` in the options.",
             );
         const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
         const host = new URL(baseUrl).host;

package/src/errors.ts

@@ -125,7 +125,7 @@ export function scrapaError(failure: Failure, host: string): ScrapaError {
             status: 0,
             mayHaveRun: method !== "GET",
             message: failure.timedOut
-                ? `scrapa: no answer within ${failure.timeoutMs}ms — the page may still have been read and charged.`
+                ? `scrapa: no answer within ${failure.timeoutMs}ms. The page may still have been read and charged.`
                 : `scrapa: could not reach ${host}.`,
             cause: failure.cause,
         });

package/src/generated/contract.ts

@@ -44,6 +44,10 @@ export const ERROR_CODES = [
     // or did not answer, or the managed pool has no exit in the place you asked for. Free,
     // and no colour — the fix is in the request (the proxy, the credentials, a wider place).
     "PROXY_REFUSED", // 422 — details {reason: "no_exit" | "refused" | "unreachable", place?, status?}
+    // The site sent a file bigger than `raw` hands back whole. Not 413, which tells an HTTP
+    // client its OWN request was too big. Free, and no colour, for PROXY_REFUSED's reason:
+    // the fix is in the request (another format, or download it yourself), not another road.
+    "FILE_TOO_LARGE", // 422 — details {limit}, in bytes
     "RATE_LIMIT_EXCEEDED", // 429 — the per-minute burst bucket; Retry-After + details {limit}
     "QUOTA_EXCEEDED", // 429 — a durable bound that WAITING clears; details {scope, limit, resetAt}
     "INTERNAL_ERROR", // 500 — we broke it; the body is always generic, the cause stays server-side
@@ -71,6 +75,7 @@ export const ERROR_STATUS = {
     UPSTREAM_BLOCKED: 422,
     NAV_TIMEOUT: 422,
     PROXY_REFUSED: 422,
+    FILE_TOO_LARGE: 422,
     RATE_LIMIT_EXCEEDED: 429,
     QUOTA_EXCEEDED: 429,
     INTERNAL_ERROR: 500,
@@ -79,10 +84,11 @@ export const ERROR_STATUS = {
 } as const satisfies Record<ErrorCode, number>;
 
 /**
- * The three no-charge outcomes, as DESIGN.md names them. Every code that costs the caller
- * nothing maps to exactly one — this is the map the console reads to pick the hue and the
- * word, and the SDK reads to tell a caller what to do next. A code absent from here was
- * the caller's own mistake (a bad body, a missing key) and gets no colour.
+ * The three no-charge outcomes, as DESIGN.md names them. Each code here maps to exactly one
+ * — this is the map the console reads to pick the hue and the word, and the SDK reads to
+ * tell a caller what to do next. A code absent from here gets no colour: the caller's own
+ * mistake (a bad body, a missing key), or a free refusal whose fix is in the request rather
+ * than the site (PROXY_REFUSED, FILE_TOO_LARGE).
  *
  *   blocked → the site said no    → try another engine, pool or country
  *   denied  → our policy said no  → change the URL; retrying cannot help
@@ -180,7 +186,8 @@ export type ResolvedEngine = Exclude<Engine, "auto">;
 /** What a scrape can hand back. `markdown` is the default because it is what an LLM, a
  *  notebook and a human all read; `html` is the page as fetched; `text` is prose with the
  *  markup gone; `links` every absolute anchor in document order; `extract` is the page read
- *  into the fields the caller asked for, which is the one format a model writes. */
+ *  into the fields the caller asked for, which is the one format a model writes; `raw` is the
+ *  response's own bytes, the file a browser would save. */
 export const SCRAPE_FORMATS = [
     "markdown",
     "html",
@@ -189,6 +196,7 @@ export const SCRAPE_FORMATS = [
     "screenshot",
     "pdf",
     "extract",
+    "raw",
 ] as const;
 
 export type ScrapeFormat = (typeof SCRAPE_FORMATS)[number];
@@ -210,7 +218,7 @@ export type SearchTopic = (typeof SEARCH_TOPICS)[number];
 
 /** Which front answered a search. Not a caller's choice — the ladder picks, and says which
  *  one it was, because a result set's flavour follows its index. */
-export const SEARCH_ENGINES = ["bing", "brave"] as const;
+export const SEARCH_ENGINES = ["bing", "brave", "duckduckgo"] as const;
 
 export type SearchEngine = (typeof SEARCH_ENGINES)[number];
 
@@ -277,6 +285,14 @@ export interface ScrapeMetadataDto {
     needsJs: boolean;
 }
 
+/** A file, exactly as the site sent it. */
+export interface ScrapeRawDto {
+    /** The bytes, base64-encoded. */
+    base64: string;
+    /** The response's `content-type`, media type only: `application/pdf`. */
+    contentType: string | null;
+}
+
 /** `POST /v1/scrape`'s answer. Only the requested formats are present; everything else is
  *  always there. Artifacts (`screenshot`, `pdf`) are base64 in v1. */
 export interface ScrapeResultDto {
@@ -307,6 +323,9 @@ export interface ScrapeResultDto {
     links?: string[];
     screenshot?: string;
     pdf?: string;
+    /** The response's own bytes, as a browser would save them (any compression undone), when
+     *  `raw` was in `formats`. Never cut short: a file over the limit is refused instead. */
+    raw?: ScrapeRawDto;
     /** The fields asked for in `extract`, read off the page by a model. Null when the model
      *  could not answer — that credit is handed back and the page still stands. Present only
      *  when `extract` was in `formats`. */

package/src/generated/operations.ts

@@ -34,7 +34,7 @@ import type {
 } from "./params.ts";
 
 /** The version this client reports in `x-scrapa-client`. */
-export const SDK_VERSION = "0.2.1";
+export const SDK_VERSION = "0.3.0";
 
 /** The hosted API. */
 export const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
@@ -50,12 +50,13 @@ export abstract class GeneratedOperations {
      * Scrape a page
      *
      * Read one web page and hand it back as markdown, HTML, plain text, its links, a screenshot, a
-     * PDF — or the fields you ask for, read off the page by a model. `engine: "auto"` (the
-     * default) makes a plain HTTP request first and re-reads the page in a real Chromium when the
-     * body turns out to be a JavaScript shell, so you pay for a browser only when the page needed
-     * one. Every request leaves through a residential proxy, or through your own with `proxy: {
-     * pool: "custom", server }`. The answer says which engine replied, where the page finally was
-     * and what it cost.
+     * PDF, or the fields you ask for, read off the page by a model. `engine: "auto"` (the default)
+     * makes a plain HTTP request first and re-reads the page in a real Chromium when the body
+     * turns out to be a JavaScript shell, so you pay for a browser only when the page needed one.
+     * To send a search form or a JSON query, use `method: "POST"` with a `body`; to download a
+     * file exactly as the site sent it, ask for `formats: ["raw"]`. Every request leaves through a
+     * residential proxy, or through your own with `proxy: { pool: "custom", server }`. The answer
+     * says which engine replied, where the page finally was and what it cost.
      *
      * Credits: 1 credit with the fetch engine, 5 with the browser (a screenshot or a PDF is
      * included, never a surcharge), 5 more when you ask for `extract`, plus managed-proxy
@@ -84,11 +85,11 @@ export abstract class GeneratedOperations {
      * Price a scrape
      *
      * What the same request would cost, without making it. Free, and refused by the same guards a
-     * real call meets — so a quote is never a price for a call that would be denied. `maxCost` is
+     * real call meets, so a quote is never a price for a call that would be denied. `maxCost` is
      * the ceiling: for `auto` it is the browser's price, which is only charged if the page turns
      * out to need one.
      *
-     * Credits: 0 credits — pricing a call is free.
+     * Credits: 0 credits. Pricing a call is free.
      *
      * `POST /v1/scrape/quote`
      */
@@ -111,12 +112,13 @@ export abstract class GeneratedOperations {
     /**
      * Search the web
      *
-     * Ask the web a question and get back ranked results — title, URL and the snippet the index
-     * holds — read live off a real search front through a residential exit, never a reseller's
-     * cache. `country` picks both the exit and the market, so `br` returns what somebody in Brazil
-     * would see. Narrow to your own list of hosts with `includeDomains`, ask for `topic: "news"`
-     * when you want dated journalism, and add `answer: true` when you want a paragraph written
-     * over the results instead of writing it yourself. To read a result, pass its URL to `scrape`.
+     * Ask the web a question and get back ranked results: title, URL and the snippet the index
+     * holds. They are read live off a real search front through a residential exit, never a
+     * reseller's cache. `country` picks both the exit and the market, so `br` returns what
+     * somebody in Brazil would see. Narrow to your own list of hosts with `includeDomains`, ask
+     * for `topic: "news"` when you want dated journalism, and add `answer: true` when you want a
+     * paragraph written over the results instead of writing it yourself. To read a result, pass
+     * its URL to `scrape`.
      *
      * Credits: 4 credits per 10 results, so a `limit` above 10 costs two rows, plus 1 for `answer:
      * true`. A search every front refused, and anything we broke, cost nothing; a search that ran
@@ -146,8 +148,8 @@ export abstract class GeneratedOperations {
          * Start a batch
          *
          * Hand us a list of URLs and stop holding a connection open. Every one is read with the
-         * options you pass here — the same engines, the same exit and the same meter a single
-         * `scrape` uses — and a page the site blocked costs nothing here too. Nothing is charged
+         * options you pass here. It uses the same engines, the same exit and the same meter as a
+         * single `scrape`, and a page the site blocked costs nothing here too. Nothing is charged
          * when you submit: a batch spends as it goes, so use `quote` for the ceiling. Poll `get`
          * for progress, or pass a `webhook` and we will call you instead. Results are readable for
          * 24 hours.
@@ -179,9 +181,9 @@ export abstract class GeneratedOperations {
          * What that submission would cost at most, without submitting it. `maxCost` prices every
          * URL at the ceiling one page could reach, so the real bill comes in lower by whatever the
          * web refuses for free and by every page that never needed a browser. Bandwidth is on top
-         * and cannot be quoted — it depends on what the pages weigh.
+         * and cannot be quoted, because it depends on what the pages weigh.
          *
-         * Credits: 0 credits — pricing a batch is free.
+         * Credits: 0 credits. Pricing a batch is free.
          *
          * `POST /v1/batches/quote`
          */
@@ -202,10 +204,10 @@ export abstract class GeneratedOperations {
         /**
          * List your batches
          *
-         * Your most recent batches, newest first — what each one is doing, how far it has got and
-         * what it has cost. Results are not included; ask `get` for one batch's pages.
+         * Your most recent batches, newest first, with what each one is doing, how far it has got
+         * and what it has cost. Results are not included; ask `get` for one batch's pages.
          *
-         * Credits: 0 credits — listing batches is free.
+         * Credits: 0 credits. Listing batches is free.
          *
          * `GET /v1/batches`
          */
@@ -218,13 +220,13 @@ export abstract class GeneratedOperations {
          *
          * Where a batch got to: whether it is still running, how many of its URLs now have a
          * verdict, how many of those the web or our policy refused, how many were never attempted,
-         * and what it has spent so far. Ask for `items` to get the results themselves — the pages
+         * and what it has spent so far. Ask for `items` to get the results themselves: the pages
          * and the refusals, in the order you submitted them, each one the same answer a single
-         * `scrape` would have given — up to 200 at a time, from wherever `from` says. A batch
-         * whose results have expired still answers with everything else, because somebody who paid
-         * for those pages must never be told the batch does not exist.
+         * `scrape` would have given. You get up to 200 at a time, from wherever `from` says. A
+         * batch whose results have expired still answers with everything else, because somebody
+         * who paid for those pages must never be told the batch does not exist.
          *
-         * Credits: 0 credits — reading a batch is free.
+         * Credits: 0 credits. Reading a batch is free.
          *
          * `GET /v1/batches/{id}`
          */
@@ -235,12 +237,12 @@ export abstract class GeneratedOperations {
         /**
          * Cancel a batch
          *
-         * Stop a batch. A URL already being read finishes and is charged — it is a page we are
-         * already paying for — and every URL not yet started becomes `skipped`, which costs
-         * nothing and is not a verdict about the URL: submit those positions again whenever you
-         * like. Safe to repeat; a batch that has already ended answers with how it ended.
+         * Stop a batch. A URL already being read finishes and is charged, because it is a page we
+         * are already paying for. Every URL not yet started becomes `skipped`, which costs nothing
+         * and is not a verdict about the URL: submit those positions again whenever you like. Safe
+         * to repeat; a batch that has already ended answers with how it ended.
          *
-         * Credits: 0 credits — cancelling is free, and it stops the spending.
+         * Credits: 0 credits. Cancelling is free, and it stops the spending.
          *
          * `DELETE /v1/batches/{id}`
          */
@@ -264,10 +266,10 @@ export abstract class GeneratedOperations {
          * `puppeteer.connect({ browserWSEndpoint: connectUrl })` in Puppeteer. The browser is
          * yours alone for the whole window, every byte it pulls leaves through the exit you
          * picked, and the socket survives our deploys. The URL carries the session's token and is
-         * shown once — it cannot be rebuilt from the id.
+         * shown once. It cannot be rebuilt from the id.
          *
          * Credits: 2 credits per minute, reserved for the whole window when the session starts and
-         * settled by the minutes actually used when it closes — so closing early gives the rest
+         * settled by the minutes actually used when it closes, so closing early gives the rest
          * back, and a browser that never came up costs nothing at all. Managed-proxy bandwidth
          * past the first megabyte is added at close; your own proxy is never metered.
          *
@@ -291,10 +293,10 @@ export abstract class GeneratedOperations {
          * Price a session
          *
          * What holding a browser for that long would cost, without holding one. `maxCost` is the
-         * whole window at the per-minute rate — a ceiling, not a bill, because the minutes you do
-         * not use come back when the session closes.
+         * whole window at the per-minute rate. It is a ceiling, not a bill, because the minutes
+         * you do not use come back when the session closes.
          *
-         * Credits: 0 credits — pricing a session is free.
+         * Credits: 0 credits. Pricing a session is free.
          *
          * `POST /v1/sessions/quote`
          */
@@ -315,10 +317,10 @@ export abstract class GeneratedOperations {
         /**
          * List your sessions
          *
-         * Your most recent sessions, newest first — the live ones and the ones that have ended,
+         * Your most recent sessions, newest first: the live ones and the ones that have ended,
          * with what each of them cost.
          *
-         * Credits: 0 credits — listing sessions is free.
+         * Credits: 0 credits. Listing sessions is free.
          *
          * `GET /v1/sessions`
          */
@@ -331,10 +333,10 @@ export abstract class GeneratedOperations {
          * Read one session
          *
          * Where a session got to: whether it is still live, when it expires, what it has pulled
-         * and — once it has ended — what it finally cost. The connect URL is not repeated here; it
+         * and, once it has ended, what it finally cost. The connect URL is not repeated here; it
          * is shown once, when the session starts.
          *
-         * Credits: 0 credits — reading a session is free.
+         * Credits: 0 credits. Reading a session is free.
          *
          * `GET /v1/sessions/{id}`
          */
@@ -347,8 +349,8 @@ export abstract class GeneratedOperations {
          * Close a session
          *
          * Stop the browser and settle the bill: the minutes you held it are kept and the rest of
-         * the reservation goes back. Safe to repeat — a session that has already ended answers
-         * with how it ended.
+         * the reservation goes back. Safe to repeat: a session that has already ended answers with
+         * how it ended.
          *
          * Credits: Closing is free. It is where the session's own bill lands: the unused minutes
          * are refunded and the managed pool's bandwidth is charged.