@scrapa/sdk 0.3.0 → 0.5.0

package/src/generated/params.ts

@@ -14,9 +14,10 @@ export interface ScrapeParams {
      *  always use "fetch", so leave it out for them. */
     engine?: "auto" | "fetch" | "browser";
     /** Whether to try again a different way when the first attempt is blocked. On by default.
-     *  `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
-     *  how you bound a call: with `auto`, trying again is what charges the browser price instead of
-     *  the fetch one. A POST is sent once, so it is off for a POST. */
+     *  `false` stops after one try. The site's refusal is your answer, and it costs nothing, unless
+     *  your own `browser.actions` script ran in the page first. It is how you bound a call: with
+     *  `auto`, trying again is what charges the browser price instead of the fetch one. A POST is
+     *  sent once, so it is off for a POST. */
     escalate?: boolean;
     /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
      *  sent once: never retried and never sent again another way, because sending it twice could
@@ -31,15 +32,35 @@ export interface ScrapeParams {
     /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
      *  default) takes the site's first no as the answer. A blocked page costs nothing, so a try
      *  that is blocked again is free, but it takes as long as the first one did, so the call can
-     *  run that many times longer. */
+     *  run that many times longer. Once your `browser.actions` steps click, type, press a key or
+     *  run a script on a page, it is not tried again, so nothing is done twice. If one of your
+     *  scripts ran in it, the block is charged. */
     retry?: number;
     /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
      *  asking for both costs no more than one. "extract" is the page read into fields by a model,
-     *  and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
-     *  it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
-     *  fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
-     *  cut short. */
+     *  and needs an `extract` block saying which ones. A link to a PDF or a Word (.docx) file is
+     *  read into text, so the text formats work on it too. We read files up to 25 MB, and from a
+     *  PDF the first 200 pages and the first 5 million characters. Reading a file costs what the
+     *  engine costs: 1 credit with fetch, 5 with the browser. A Word file that holds more than 32
+     *  MB once unzipped answers FILE_TOO_LARGE, and a file link that answers a short web page
+     *  instead of the file answers UPSTREAM_BLOCKED, both at no charge. A dead link is the
+     *  exception: its 404 or 410 page is the site's answer, and comes back with that status at the
+     *  fetch price. "raw" is the file exactly as the site sent it, as base64 with its content type:
+     *  a PDF report, an image, a spreadsheet. It uses the fetch engine and holds up to 5 MB; a
+     *  larger file answers FILE_TOO_LARGE at no charge, never cut short. */
     formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
+    /** Read only the page's main content into `markdown`, `text` and `extract`: the article or the
+     *  product, without the menus, ads and related links around it. On by default. Set `false` for
+     *  the whole page. When the main content is under 50 words, you get the whole page instead.
+     *  `html` and `links` always hold the whole page. */
+    onlyMainContent?: boolean;
+    /** CSS selectors for the parts of the page to keep, such as `["h1", ".price"]`. `markdown`,
+     *  `text` and `extract` then hold only what they match, in page order, and `onlyMainContent` no
+     *  longer applies. Hidden parts are never kept. Up to 20 selectors, 256 characters each. */
+    includeTags?: string[];
+    /** CSS selectors for the parts of the page to leave out of `markdown`, `text` and `extract`,
+     *  such as `[".comments", "#newsletter"]`. Up to 20 selectors, 256 characters each. */
+    excludeTags?: string[];
     /** What the page should answer with. Set `"json"` when you are reading an API: anything else
      *  that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
      *  not your answer. It costs nothing, and we try again from another address instead of handing
@@ -68,8 +89,10 @@ export interface ScrapeParams {
         server?: string;
         /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
          *  and the managed pool leaves from that country. Leave it out and the page sees a US
-         *  browser, from an address we pick. The browser's clock and languages follow the country
-         *  on every pool. */
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
         country?: string;
         /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
          *  only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -82,7 +105,7 @@ export interface ScrapeParams {
         city?: string;
     };
     /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
-     *  turns out to need one. */
+     *  turns out to need one, except `actions`, which always use the browser. */
     browser?: {
         /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
          *  "domcontentloaded" is the fastest and stops at the HTML; "networkidle" waits until the
@@ -105,13 +128,89 @@ export interface ScrapeParams {
         /** Screenshot the whole scrollable page instead of just the window. A long page makes a
          *  large answer: the image comes back as base64 inside the response. */
         fullPage?: boolean;
+        /** Steps to run in the page before it is read, in the order you list them. A step can wait,
+         *  click, type, press a key, scroll, take a screenshot, keep the HTML or run a script.
+         *  Steps always use the browser engine, at the browser price. The formats you ask for read
+         *  the page as the last step left it. Up to 50 steps, at most 10 of them screenshots or
+         *  scrapes. If one fails, the call answers ACTION_FAILED, which names the step and why.
+         *  That costs nothing, unless one of your scripts started, the one that failed included:
+         *  then the call is charged. */
+        actions?: (
+            | {
+                  type: "wait";
+                  /** How long to wait, in milliseconds. All the waits together have to fit inside
+                   *  `timeoutMs`. */
+                  milliseconds?: number;
+                  /** Wait until this CSS selector is on the page. If it never shows up, the step fails. */
+                  selector?: string;
+              }
+            | {
+                  type: "click";
+                  /** The CSS selector of what to click. The first match gets the click, and the step
+                   *  waits for it to show up. */
+                  selector: string;
+                  /** Click every visible match, not just the first. A selector that matches nothing is
+                   *  then not an error, which suits a cookie banner only some visitors see. */
+                  all?: boolean;
+              }
+            | {
+                  type: "write";
+                  /** What to type, up to 1,000 characters. */
+                  text: string;
+              }
+            | {
+                  type: "press";
+                  /** The key's name: "Enter", "Tab", "Escape", "ArrowDown", or two together such as
+                   *  "Control+A". */
+                  key: string;
+              }
+            | {
+                  type: "scroll";
+                  direction: "up" | "down";
+                  /** Scroll this element instead of the whole page, such as a list with its own scroll
+                   *  bar. */
+                  selector?: string;
+              }
+            | {
+                  type: "screenshot";
+                  /** Take the whole scrollable page instead of just the window. */
+                  fullPage?: boolean;
+              }
+            | {
+                  type: "scrape";
+              }
+            | {
+                  type: "executeJavascript";
+                  /** The body of an async function, up to 8,000 characters, such as `return
+                   *  document.title`. It runs in the page, beside the page's own scripts. What it returns
+                   *  comes back in `actions.javascriptReturns` as JSON, or as null when JSON cannot hold
+                   *  it. All the scripts in one call can return 64 KB together. Once a script starts, the
+                   *  call is charged even if that script throws, a later step fails, the page crashes or
+                   *  times out, the site blocks it, the script leads to an address we refuse, your own
+                   *  proxy says no, or an `includeTags` or `excludeTags` selector is too slow for the
+                   *  page it built, because the script may already have read the page. A failure on our
+                   *  side (a 5xx) is still refunded. */
+                  script: string;
+              }
+        )[];
     };
     /** Extra request headers to send to the page, such as an authorization token, a cookie or a
      *  custom user agent. The connection's own headers cannot be set. */
     headers?: Record<string, string>;
-    /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
-     *  free outcome. */
+    /** How long to give the page before giving up, in milliseconds. A page that never arrives costs
+     *  nothing, unless your own `browser.actions` script ran in the page first. */
     timeoutMs?: number;
+    /** The oldest copy of the page you accept, in milliseconds, up to 3600000 (one hour). If we
+     *  still hold a copy of the same URL that your account read within that time, from the same
+     *  country and with the same engine, you get it back for 0 credits and the answer says `cached:
+     *  true`. If not, we read the page now and keep a copy for up to an hour, unless the page still
+     *  needs JavaScript (`metadata.needsJs`). We may drop a copy sooner, to make room or on a
+     *  deploy, and a batch keeps its own copies, apart from single calls. `0` always reads the page
+     *  now, and keeps the copy for the calls after it. `extract` is still charged, because the
+     *  model reads the copy again. Leave `maxAge` out to read the page fresh every time. A POST, or
+     *  a call with `headers`, your own proxy, `browser` options, a screenshot, a PDF, `raw` or
+     *  `escalate: false`, is always read fresh. */
+    maxAge?: number;
 }
 
 /** Arguments for `quote()`. */
@@ -124,9 +223,10 @@ export interface QuoteScrapeParams {
      *  always use "fetch", so leave it out for them. */
     engine?: "auto" | "fetch" | "browser";
     /** Whether to try again a different way when the first attempt is blocked. On by default.
-     *  `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
-     *  how you bound a call: with `auto`, trying again is what charges the browser price instead of
-     *  the fetch one. A POST is sent once, so it is off for a POST. */
+     *  `false` stops after one try. The site's refusal is your answer, and it costs nothing, unless
+     *  your own `browser.actions` script ran in the page first. It is how you bound a call: with
+     *  `auto`, trying again is what charges the browser price instead of the fetch one. A POST is
+     *  sent once, so it is off for a POST. */
     escalate?: boolean;
     /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
      *  sent once: never retried and never sent again another way, because sending it twice could
@@ -141,15 +241,35 @@ export interface QuoteScrapeParams {
     /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
      *  default) takes the site's first no as the answer. A blocked page costs nothing, so a try
      *  that is blocked again is free, but it takes as long as the first one did, so the call can
-     *  run that many times longer. */
+     *  run that many times longer. Once your `browser.actions` steps click, type, press a key or
+     *  run a script on a page, it is not tried again, so nothing is done twice. If one of your
+     *  scripts ran in it, the block is charged. */
     retry?: number;
     /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
      *  asking for both costs no more than one. "extract" is the page read into fields by a model,
-     *  and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
-     *  it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
-     *  fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
-     *  cut short. */
+     *  and needs an `extract` block saying which ones. A link to a PDF or a Word (.docx) file is
+     *  read into text, so the text formats work on it too. We read files up to 25 MB, and from a
+     *  PDF the first 200 pages and the first 5 million characters. Reading a file costs what the
+     *  engine costs: 1 credit with fetch, 5 with the browser. A Word file that holds more than 32
+     *  MB once unzipped answers FILE_TOO_LARGE, and a file link that answers a short web page
+     *  instead of the file answers UPSTREAM_BLOCKED, both at no charge. A dead link is the
+     *  exception: its 404 or 410 page is the site's answer, and comes back with that status at the
+     *  fetch price. "raw" is the file exactly as the site sent it, as base64 with its content type:
+     *  a PDF report, an image, a spreadsheet. It uses the fetch engine and holds up to 5 MB; a
+     *  larger file answers FILE_TOO_LARGE at no charge, never cut short. */
     formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
+    /** Read only the page's main content into `markdown`, `text` and `extract`: the article or the
+     *  product, without the menus, ads and related links around it. On by default. Set `false` for
+     *  the whole page. When the main content is under 50 words, you get the whole page instead.
+     *  `html` and `links` always hold the whole page. */
+    onlyMainContent?: boolean;
+    /** CSS selectors for the parts of the page to keep, such as `["h1", ".price"]`. `markdown`,
+     *  `text` and `extract` then hold only what they match, in page order, and `onlyMainContent` no
+     *  longer applies. Hidden parts are never kept. Up to 20 selectors, 256 characters each. */
+    includeTags?: string[];
+    /** CSS selectors for the parts of the page to leave out of `markdown`, `text` and `extract`,
+     *  such as `[".comments", "#newsletter"]`. Up to 20 selectors, 256 characters each. */
+    excludeTags?: string[];
     /** What the page should answer with. Set `"json"` when you are reading an API: anything else
      *  that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
      *  not your answer. It costs nothing, and we try again from another address instead of handing
@@ -178,8 +298,10 @@ export interface QuoteScrapeParams {
         server?: string;
         /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
          *  and the managed pool leaves from that country. Leave it out and the page sees a US
-         *  browser, from an address we pick. The browser's clock and languages follow the country
-         *  on every pool. */
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
         country?: string;
         /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
          *  only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -192,7 +314,7 @@ export interface QuoteScrapeParams {
         city?: string;
     };
     /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
-     *  turns out to need one. */
+     *  turns out to need one, except `actions`, which always use the browser. */
     browser?: {
         /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
          *  "domcontentloaded" is the fastest and stops at the HTML; "networkidle" waits until the
@@ -215,13 +337,155 @@ export interface QuoteScrapeParams {
         /** Screenshot the whole scrollable page instead of just the window. A long page makes a
          *  large answer: the image comes back as base64 inside the response. */
         fullPage?: boolean;
+        /** Steps to run in the page before it is read, in the order you list them. A step can wait,
+         *  click, type, press a key, scroll, take a screenshot, keep the HTML or run a script.
+         *  Steps always use the browser engine, at the browser price. The formats you ask for read
+         *  the page as the last step left it. Up to 50 steps, at most 10 of them screenshots or
+         *  scrapes. If one fails, the call answers ACTION_FAILED, which names the step and why.
+         *  That costs nothing, unless one of your scripts started, the one that failed included:
+         *  then the call is charged. */
+        actions?: (
+            | {
+                  type: "wait";
+                  /** How long to wait, in milliseconds. All the waits together have to fit inside
+                   *  `timeoutMs`. */
+                  milliseconds?: number;
+                  /** Wait until this CSS selector is on the page. If it never shows up, the step fails. */
+                  selector?: string;
+              }
+            | {
+                  type: "click";
+                  /** The CSS selector of what to click. The first match gets the click, and the step
+                   *  waits for it to show up. */
+                  selector: string;
+                  /** Click every visible match, not just the first. A selector that matches nothing is
+                   *  then not an error, which suits a cookie banner only some visitors see. */
+                  all?: boolean;
+              }
+            | {
+                  type: "write";
+                  /** What to type, up to 1,000 characters. */
+                  text: string;
+              }
+            | {
+                  type: "press";
+                  /** The key's name: "Enter", "Tab", "Escape", "ArrowDown", or two together such as
+                   *  "Control+A". */
+                  key: string;
+              }
+            | {
+                  type: "scroll";
+                  direction: "up" | "down";
+                  /** Scroll this element instead of the whole page, such as a list with its own scroll
+                   *  bar. */
+                  selector?: string;
+              }
+            | {
+                  type: "screenshot";
+                  /** Take the whole scrollable page instead of just the window. */
+                  fullPage?: boolean;
+              }
+            | {
+                  type: "scrape";
+              }
+            | {
+                  type: "executeJavascript";
+                  /** The body of an async function, up to 8,000 characters, such as `return
+                   *  document.title`. It runs in the page, beside the page's own scripts. What it returns
+                   *  comes back in `actions.javascriptReturns` as JSON, or as null when JSON cannot hold
+                   *  it. All the scripts in one call can return 64 KB together. Once a script starts, the
+                   *  call is charged even if that script throws, a later step fails, the page crashes or
+                   *  times out, the site blocks it, the script leads to an address we refuse, your own
+                   *  proxy says no, or an `includeTags` or `excludeTags` selector is too slow for the
+                   *  page it built, because the script may already have read the page. A failure on our
+                   *  side (a 5xx) is still refunded. */
+                  script: string;
+              }
+        )[];
     };
     /** Extra request headers to send to the page, such as an authorization token, a cookie or a
      *  custom user agent. The connection's own headers cannot be set. */
     headers?: Record<string, string>;
-    /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
-     *  free outcome. */
+    /** How long to give the page before giving up, in milliseconds. A page that never arrives costs
+     *  nothing, unless your own `browser.actions` script ran in the page first. */
     timeoutMs?: number;
+    /** The oldest copy of the page you accept, in milliseconds, up to 3600000 (one hour). If we
+     *  still hold a copy of the same URL that your account read within that time, from the same
+     *  country and with the same engine, you get it back for 0 credits and the answer says `cached:
+     *  true`. If not, we read the page now and keep a copy for up to an hour, unless the page still
+     *  needs JavaScript (`metadata.needsJs`). We may drop a copy sooner, to make room or on a
+     *  deploy, and a batch keeps its own copies, apart from single calls. `0` always reads the page
+     *  now, and keeps the copy for the calls after it. `extract` is still charged, because the
+     *  model reads the copy again. Leave `maxAge` out to read the page fresh every time. A POST, or
+     *  a call with `headers`, your own proxy, `browser` options, a screenshot, a PDF, `raw` or
+     *  `escalate: false`, is always read fresh. */
+    maxAge?: number;
+}
+
+/** Arguments for `map()`. */
+export interface MapSiteParams {
+    /** The site to map, as any http(s) URL on it. Private and loopback hosts are refused. */
+    url: string;
+    /** How many URLs to hand back at most, 1 to 30,000. The price is the same whatever you ask for. */
+    limit?: number;
+    /** Only URLs that contain this text, upper or lower case alike: `blog` keeps `/blog/` and
+     *  `/Blog-post`. */
+    search?: string;
+    /** Also keep URLs on other hosts of the same domain, like `blog.example.com` for `example.com`.
+     *  Off by default: only the host you named, with or without `www.`. */
+    includeSubdomains?: boolean;
+    /** Where to look. "include" reads the site's sitemaps and the links on the page you named;
+     *  "only" reads the sitemaps alone; "skip" reads the page's links alone. */
+    sitemap?: "include" | "only" | "skip";
+    /** Where the map's requests leave from. Every file it reads goes through the same exit. */
+    proxy?: {
+        /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
+         *  "custom" is your own proxy, free on every plan, bytes never billed. */
+        pool?: "residential" | "custom";
+        /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
+         *  for the managed one. */
+        server?: string;
+        /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
+         *  and the managed pool leaves from that country. Leave it out and the page sees a US
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
+        country?: string;
+    };
+}
+
+/** Arguments for `quoteMap()`. */
+export interface QuoteMapParams {
+    /** The site to map, as any http(s) URL on it. Private and loopback hosts are refused. */
+    url: string;
+    /** How many URLs to hand back at most, 1 to 30,000. The price is the same whatever you ask for. */
+    limit?: number;
+    /** Only URLs that contain this text, upper or lower case alike: `blog` keeps `/blog/` and
+     *  `/Blog-post`. */
+    search?: string;
+    /** Also keep URLs on other hosts of the same domain, like `blog.example.com` for `example.com`.
+     *  Off by default: only the host you named, with or without `www.`. */
+    includeSubdomains?: boolean;
+    /** Where to look. "include" reads the site's sitemaps and the links on the page you named;
+     *  "only" reads the sitemaps alone; "skip" reads the page's links alone. */
+    sitemap?: "include" | "only" | "skip";
+    /** Where the map's requests leave from. Every file it reads goes through the same exit. */
+    proxy?: {
+        /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
+         *  "custom" is your own proxy, free on every plan, bytes never billed. */
+        pool?: "residential" | "custom";
+        /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
+         *  for the managed one. */
+        server?: string;
+        /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
+         *  and the managed pool leaves from that country. Leave it out and the page sees a US
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
+        country?: string;
+    };
 }
 
 /** Arguments for `search()`. */
@@ -233,13 +497,15 @@ export interface SearchParams {
      *  `includeDomains` filters what it returns. */
     limit?: number;
     /** What kind of results. "general" is the web; "news" is dated journalism, and the only topic
-     *  that reliably carries `publishedAt`. */
-    topic?: "general" | "news";
+     *  that reliably carries `publishedAt`; "images" is pictures: each result is the page that
+     *  shows one, with the picture's own address, a thumbnail and its size in `image`. Pictures the
+     *  engine marks as adult are left out. */
+    topic?: "general" | "news" | "images";
     /** Only pages from the last day, week, month or year. Absent is any time. */
     timeRange?: "day" | "week" | "month" | "year";
     /** Only results on these hosts, bare (`linkedin.com`). Subdomains are included. Asked of the
      *  search front AND enforced on the answer, so what comes back is on your list whatever the
-     *  front did with it. */
+     *  front did with it. With `topic: "images"` it takes one host. */
     includeDomains?: string[];
     /** Also write a short paragraph answering the query from the results, in the query's language.
      *  Costs one extra credit; leave it off when your own model reads the snippets. */
@@ -255,8 +521,10 @@ export interface SearchParams {
         server?: string;
         /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
          *  and the managed pool leaves from that country. Leave it out and the page sees a US
-         *  browser, from an address we pick. The browser's clock and languages follow the country
-         *  on every pool. */
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
         country?: string;
     };
 }
@@ -269,22 +537,43 @@ export interface CreateBatchParams {
      *  always use "fetch", so leave it out for them. */
     engine?: "auto" | "fetch" | "browser";
     /** Whether to try again a different way when the first attempt is blocked. On by default.
-     *  `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
-     *  how you bound a call: with `auto`, trying again is what charges the browser price instead of
-     *  the fetch one. A POST is sent once, so it is off for a POST. */
+     *  `false` stops after one try. The site's refusal is your answer, and it costs nothing, unless
+     *  your own `browser.actions` script ran in the page first. It is how you bound a call: with
+     *  `auto`, trying again is what charges the browser price instead of the fetch one. A POST is
+     *  sent once, so it is off for a POST. */
     escalate?: boolean;
     /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
      *  default) takes the site's first no as the answer. A blocked page costs nothing, so a try
      *  that is blocked again is free, but it takes as long as the first one did, so the call can
-     *  run that many times longer. */
+     *  run that many times longer. Once your `browser.actions` steps click, type, press a key or
+     *  run a script on a page, it is not tried again, so nothing is done twice. If one of your
+     *  scripts ran in it, the block is charged. */
     retry?: number;
     /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
      *  asking for both costs no more than one. "extract" is the page read into fields by a model,
-     *  and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
-     *  it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
-     *  fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
-     *  cut short. */
+     *  and needs an `extract` block saying which ones. A link to a PDF or a Word (.docx) file is
+     *  read into text, so the text formats work on it too. We read files up to 25 MB, and from a
+     *  PDF the first 200 pages and the first 5 million characters. Reading a file costs what the
+     *  engine costs: 1 credit with fetch, 5 with the browser. A Word file that holds more than 32
+     *  MB once unzipped answers FILE_TOO_LARGE, and a file link that answers a short web page
+     *  instead of the file answers UPSTREAM_BLOCKED, both at no charge. A dead link is the
+     *  exception: its 404 or 410 page is the site's answer, and comes back with that status at the
+     *  fetch price. "raw" is the file exactly as the site sent it, as base64 with its content type:
+     *  a PDF report, an image, a spreadsheet. It uses the fetch engine and holds up to 5 MB; a
+     *  larger file answers FILE_TOO_LARGE at no charge, never cut short. */
     formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
+    /** Read only the page's main content into `markdown`, `text` and `extract`: the article or the
+     *  product, without the menus, ads and related links around it. On by default. Set `false` for
+     *  the whole page. When the main content is under 50 words, you get the whole page instead.
+     *  `html` and `links` always hold the whole page. */
+    onlyMainContent?: boolean;
+    /** CSS selectors for the parts of the page to keep, such as `["h1", ".price"]`. `markdown`,
+     *  `text` and `extract` then hold only what they match, in page order, and `onlyMainContent` no
+     *  longer applies. Hidden parts are never kept. Up to 20 selectors, 256 characters each. */
+    includeTags?: string[];
+    /** CSS selectors for the parts of the page to leave out of `markdown`, `text` and `extract`,
+     *  such as `[".comments", "#newsletter"]`. Up to 20 selectors, 256 characters each. */
+    excludeTags?: string[];
     /** What the page should answer with. Set `"json"` when you are reading an API: anything else
      *  that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
      *  not your answer. It costs nothing, and we try again from another address instead of handing
@@ -313,8 +602,10 @@ export interface CreateBatchParams {
         server?: string;
         /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
          *  and the managed pool leaves from that country. Leave it out and the page sees a US
-         *  browser, from an address we pick. The browser's clock and languages follow the country
-         *  on every pool. */
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
         country?: string;
         /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
          *  only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -327,7 +618,7 @@ export interface CreateBatchParams {
         city?: string;
     };
     /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
-     *  turns out to need one. */
+     *  turns out to need one, except `actions`, which always use the browser. */
     browser?: {
         /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
          *  "domcontentloaded" is the fastest and stops at the HTML; "networkidle" waits until the
@@ -350,13 +641,89 @@ export interface CreateBatchParams {
         /** Screenshot the whole scrollable page instead of just the window. A long page makes a
          *  large answer: the image comes back as base64 inside the response. */
         fullPage?: boolean;
+        /** Steps to run in the page before it is read, in the order you list them. A step can wait,
+         *  click, type, press a key, scroll, take a screenshot, keep the HTML or run a script.
+         *  Steps always use the browser engine, at the browser price. The formats you ask for read
+         *  the page as the last step left it. Up to 50 steps, at most 10 of them screenshots or
+         *  scrapes. If one fails, the call answers ACTION_FAILED, which names the step and why.
+         *  That costs nothing, unless one of your scripts started, the one that failed included:
+         *  then the call is charged. */
+        actions?: (
+            | {
+                  type: "wait";
+                  /** How long to wait, in milliseconds. All the waits together have to fit inside
+                   *  `timeoutMs`. */
+                  milliseconds?: number;
+                  /** Wait until this CSS selector is on the page. If it never shows up, the step fails. */
+                  selector?: string;
+              }
+            | {
+                  type: "click";
+                  /** The CSS selector of what to click. The first match gets the click, and the step
+                   *  waits for it to show up. */
+                  selector: string;
+                  /** Click every visible match, not just the first. A selector that matches nothing is
+                   *  then not an error, which suits a cookie banner only some visitors see. */
+                  all?: boolean;
+              }
+            | {
+                  type: "write";
+                  /** What to type, up to 1,000 characters. */
+                  text: string;
+              }
+            | {
+                  type: "press";
+                  /** The key's name: "Enter", "Tab", "Escape", "ArrowDown", or two together such as
+                   *  "Control+A". */
+                  key: string;
+              }
+            | {
+                  type: "scroll";
+                  direction: "up" | "down";
+                  /** Scroll this element instead of the whole page, such as a list with its own scroll
+                   *  bar. */
+                  selector?: string;
+              }
+            | {
+                  type: "screenshot";
+                  /** Take the whole scrollable page instead of just the window. */
+                  fullPage?: boolean;
+              }
+            | {
+                  type: "scrape";
+              }
+            | {
+                  type: "executeJavascript";
+                  /** The body of an async function, up to 8,000 characters, such as `return
+                   *  document.title`. It runs in the page, beside the page's own scripts. What it returns
+                   *  comes back in `actions.javascriptReturns` as JSON, or as null when JSON cannot hold
+                   *  it. All the scripts in one call can return 64 KB together. Once a script starts, the
+                   *  call is charged even if that script throws, a later step fails, the page crashes or
+                   *  times out, the site blocks it, the script leads to an address we refuse, your own
+                   *  proxy says no, or an `includeTags` or `excludeTags` selector is too slow for the
+                   *  page it built, because the script may already have read the page. A failure on our
+                   *  side (a 5xx) is still refunded. */
+                  script: string;
+              }
+        )[];
     };
     /** Extra request headers to send to the page, such as an authorization token, a cookie or a
      *  custom user agent. The connection's own headers cannot be set. */
     headers?: Record<string, string>;
-    /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
-     *  free outcome. */
+    /** How long to give the page before giving up, in milliseconds. A page that never arrives costs
+     *  nothing, unless your own `browser.actions` script ran in the page first. */
     timeoutMs?: number;
+    /** The oldest copy of the page you accept, in milliseconds, up to 3600000 (one hour). If we
+     *  still hold a copy of the same URL that your account read within that time, from the same
+     *  country and with the same engine, you get it back for 0 credits and the answer says `cached:
+     *  true`. If not, we read the page now and keep a copy for up to an hour, unless the page still
+     *  needs JavaScript (`metadata.needsJs`). We may drop a copy sooner, to make room or on a
+     *  deploy, and a batch keeps its own copies, apart from single calls. `0` always reads the page
+     *  now, and keeps the copy for the calls after it. `extract` is still charged, because the
+     *  model reads the copy again. Leave `maxAge` out to read the page fresh every time. A POST, or
+     *  a call with `headers`, your own proxy, `browser` options, a screenshot, a PDF, `raw` or
+     *  `escalate: false`, is always read fresh. */
+    maxAge?: number;
     /** The pages to read, in the order you want them back. How many one batch may carry depends on
      *  your plan. Duplicates are kept: position, not URL, is what lines results up against what you
      *  submitted. */
@@ -383,22 +750,43 @@ export interface QuoteBatchParams {
      *  always use "fetch", so leave it out for them. */
     engine?: "auto" | "fetch" | "browser";
     /** Whether to try again a different way when the first attempt is blocked. On by default.
-     *  `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
-     *  how you bound a call: with `auto`, trying again is what charges the browser price instead of
-     *  the fetch one. A POST is sent once, so it is off for a POST. */
+     *  `false` stops after one try. The site's refusal is your answer, and it costs nothing, unless
+     *  your own `browser.actions` script ran in the page first. It is how you bound a call: with
+     *  `auto`, trying again is what charges the browser price instead of the fetch one. A POST is
+     *  sent once, so it is off for a POST. */
     escalate?: boolean;
     /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
      *  default) takes the site's first no as the answer. A blocked page costs nothing, so a try
      *  that is blocked again is free, but it takes as long as the first one did, so the call can
-     *  run that many times longer. */
+     *  run that many times longer. Once your `browser.actions` steps click, type, press a key or
+     *  run a script on a page, it is not tried again, so nothing is done twice. If one of your
+     *  scripts ran in it, the block is charged. */
     retry?: number;
     /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
      *  asking for both costs no more than one. "extract" is the page read into fields by a model,
-     *  and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
-     *  it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
-     *  fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
-     *  cut short. */
+     *  and needs an `extract` block saying which ones. A link to a PDF or a Word (.docx) file is
+     *  read into text, so the text formats work on it too. We read files up to 25 MB, and from a
+     *  PDF the first 200 pages and the first 5 million characters. Reading a file costs what the
+     *  engine costs: 1 credit with fetch, 5 with the browser. A Word file that holds more than 32
+     *  MB once unzipped answers FILE_TOO_LARGE, and a file link that answers a short web page
+     *  instead of the file answers UPSTREAM_BLOCKED, both at no charge. A dead link is the
+     *  exception: its 404 or 410 page is the site's answer, and comes back with that status at the
+     *  fetch price. "raw" is the file exactly as the site sent it, as base64 with its content type:
+     *  a PDF report, an image, a spreadsheet. It uses the fetch engine and holds up to 5 MB; a
+     *  larger file answers FILE_TOO_LARGE at no charge, never cut short. */
     formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
+    /** Read only the page's main content into `markdown`, `text` and `extract`: the article or the
+     *  product, without the menus, ads and related links around it. On by default. Set `false` for
+     *  the whole page. When the main content is under 50 words, you get the whole page instead.
+     *  `html` and `links` always hold the whole page. */
+    onlyMainContent?: boolean;
+    /** CSS selectors for the parts of the page to keep, such as `["h1", ".price"]`. `markdown`,
+     *  `text` and `extract` then hold only what they match, in page order, and `onlyMainContent` no
+     *  longer applies. Hidden parts are never kept. Up to 20 selectors, 256 characters each. */
+    includeTags?: string[];
+    /** CSS selectors for the parts of the page to leave out of `markdown`, `text` and `extract`,
+     *  such as `[".comments", "#newsletter"]`. Up to 20 selectors, 256 characters each. */
+    excludeTags?: string[];
     /** What the page should answer with. Set `"json"` when you are reading an API: anything else
      *  that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
      *  not your answer. It costs nothing, and we try again from another address instead of handing
@@ -427,8 +815,10 @@ export interface QuoteBatchParams {
         server?: string;
         /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
          *  and the managed pool leaves from that country. Leave it out and the page sees a US
-         *  browser, from an address we pick. The browser's clock and languages follow the country
-         *  on every pool. */
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
         country?: string;
         /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
          *  only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -441,7 +831,7 @@ export interface QuoteBatchParams {
         city?: string;
     };
     /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
-     *  turns out to need one. */
+     *  turns out to need one, except `actions`, which always use the browser. */
     browser?: {
         /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
          *  "domcontentloaded" is the fastest and stops at the HTML; "networkidle" waits until the
@@ -464,13 +854,89 @@ export interface QuoteBatchParams {
         /** Screenshot the whole scrollable page instead of just the window. A long page makes a
          *  large answer: the image comes back as base64 inside the response. */
         fullPage?: boolean;
+        /** Steps to run in the page before it is read, in the order you list them. A step can wait,
+         *  click, type, press a key, scroll, take a screenshot, keep the HTML or run a script.
+         *  Steps always use the browser engine, at the browser price. The formats you ask for read
+         *  the page as the last step left it. Up to 50 steps, at most 10 of them screenshots or
+         *  scrapes. If one fails, the call answers ACTION_FAILED, which names the step and why.
+         *  That costs nothing, unless one of your scripts started, the one that failed included:
+         *  then the call is charged. */
+        actions?: (
+            | {
+                  type: "wait";
+                  /** How long to wait, in milliseconds. All the waits together have to fit inside
+                   *  `timeoutMs`. */
+                  milliseconds?: number;
+                  /** Wait until this CSS selector is on the page. If it never shows up, the step fails. */
+                  selector?: string;
+              }
+            | {
+                  type: "click";
+                  /** The CSS selector of what to click. The first match gets the click, and the step
+                   *  waits for it to show up. */
+                  selector: string;
+                  /** Click every visible match, not just the first. A selector that matches nothing is
+                   *  then not an error, which suits a cookie banner only some visitors see. */
+                  all?: boolean;
+              }
+            | {
+                  type: "write";
+                  /** What to type, up to 1,000 characters. */
+                  text: string;
+              }
+            | {
+                  type: "press";
+                  /** The key's name: "Enter", "Tab", "Escape", "ArrowDown", or two together such as
+                   *  "Control+A". */
+                  key: string;
+              }
+            | {
+                  type: "scroll";
+                  direction: "up" | "down";
+                  /** Scroll this element instead of the whole page, such as a list with its own scroll
+                   *  bar. */
+                  selector?: string;
+              }
+            | {
+                  type: "screenshot";
+                  /** Take the whole scrollable page instead of just the window. */
+                  fullPage?: boolean;
+              }
+            | {
+                  type: "scrape";
+              }
+            | {
+                  type: "executeJavascript";
+                  /** The body of an async function, up to 8,000 characters, such as `return
+                   *  document.title`. It runs in the page, beside the page's own scripts. What it returns
+                   *  comes back in `actions.javascriptReturns` as JSON, or as null when JSON cannot hold
+                   *  it. All the scripts in one call can return 64 KB together. Once a script starts, the
+                   *  call is charged even if that script throws, a later step fails, the page crashes or
+                   *  times out, the site blocks it, the script leads to an address we refuse, your own
+                   *  proxy says no, or an `includeTags` or `excludeTags` selector is too slow for the
+                   *  page it built, because the script may already have read the page. A failure on our
+                   *  side (a 5xx) is still refunded. */
+                  script: string;
+              }
+        )[];
     };
     /** Extra request headers to send to the page, such as an authorization token, a cookie or a
      *  custom user agent. The connection's own headers cannot be set. */
     headers?: Record<string, string>;
-    /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
-     *  free outcome. */
+    /** How long to give the page before giving up, in milliseconds. A page that never arrives costs
+     *  nothing, unless your own `browser.actions` script ran in the page first. */
     timeoutMs?: number;
+    /** The oldest copy of the page you accept, in milliseconds, up to 3600000 (one hour). If we
+     *  still hold a copy of the same URL that your account read within that time, from the same
+     *  country and with the same engine, you get it back for 0 credits and the answer says `cached:
+     *  true`. If not, we read the page now and keep a copy for up to an hour, unless the page still
+     *  needs JavaScript (`metadata.needsJs`). We may drop a copy sooner, to make room or on a
+     *  deploy, and a batch keeps its own copies, apart from single calls. `0` always reads the page
+     *  now, and keeps the copy for the calls after it. `extract` is still charged, because the
+     *  model reads the copy again. Leave `maxAge` out to read the page fresh every time. A POST, or
+     *  a call with `headers`, your own proxy, `browser` options, a screenshot, a PDF, `raw` or
+     *  `escalate: false`, is always read fresh. */
+    maxAge?: number;
     /** The pages to read, in the order you want them back. How many one batch may carry depends on
      *  your plan. Duplicates are kept: position, not URL, is what lines results up against what you
      *  submitted. */
@@ -530,8 +996,10 @@ export interface CreateSessionParams {
         server?: string;
         /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
          *  and the managed pool leaves from that country. Leave it out and the page sees a US
-         *  browser, from an address we pick. The browser's clock and languages follow the country
-         *  on every pool. */
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
         country?: string;
         /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
          *  only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -562,8 +1030,10 @@ export interface QuoteSessionParams {
         server?: string;
         /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
          *  and the managed pool leaves from that country. Leave it out and the page sees a US
-         *  browser, from an address we pick. The browser's clock and languages follow the country
-         *  on every pool. */
+         *  browser, from an address we pick. A scrape or a map on the managed pool makes one
+         *  exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
+         *  `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
+         *  the country on every pool. */
         country?: string;
         /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
          *  only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and