@scrapa/sdk 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,9 +8,10 @@ export interface ScrapeParams {
8
8
  * always use "fetch", so leave it out for them. */
9
9
  engine?: "auto" | "fetch" | "browser";
10
10
  /** Whether to try again a different way when the first attempt is blocked. On by default.
11
- * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
12
- * how you bound a call: with `auto`, trying again is what charges the browser price instead of
13
- * the fetch one. A POST is sent once, so it is off for a POST. */
11
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing, unless
12
+ * your own `browser.actions` script ran in the page first. It is how you bound a call: with
13
+ * `auto`, trying again is what charges the browser price instead of the fetch one. A POST is
14
+ * sent once, so it is off for a POST. */
14
15
  escalate?: boolean;
15
16
  /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
16
17
  * sent once: never retried and never sent again another way, because sending it twice could
@@ -25,15 +26,22 @@ export interface ScrapeParams {
25
26
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
26
27
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
27
28
  * that is blocked again is free, but it takes as long as the first one did, so the call can
28
- * run that many times longer. */
29
+ * run that many times longer. Once your `browser.actions` steps click, type, press a key or
30
+ * run a script on a page, it is not tried again, so nothing is done twice. If one of your
31
+ * scripts ran in it, the block is charged. */
29
32
  retry?: number;
30
33
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
31
34
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
32
35
  * and needs an `extract` block saying which ones. A link to a PDF or a Word (.docx) file is
33
- * read into text, so the text formats work on it too. We read files up to 25 MB and the first
34
- * 200 pages, at the fetch price. "raw" is the file exactly as the site sent it, as base64 with
35
- * its content type: a PDF report, an image, a spreadsheet. It uses the fetch engine and holds
36
- * up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never cut short. */
36
+ * read into text, so the text formats work on it too. We read files up to 25 MB, and from a
37
+ * PDF the first 200 pages and the first 5 million characters. Reading a file costs what the
38
+ * engine costs: 1 credit with fetch, 5 with the browser. A Word file that holds more than 32
39
+ * MB once unzipped answers FILE_TOO_LARGE, and a file link that answers a short web page
40
+ * instead of the file answers UPSTREAM_BLOCKED, both at no charge. A dead link is the
41
+ * exception: its 404 or 410 page is the site's answer, and comes back with that status at the
42
+ * fetch price. "raw" is the file exactly as the site sent it, as base64 with its content type:
43
+ * a PDF report, an image, a spreadsheet. It uses the fetch engine and holds up to 5 MB; a
44
+ * larger file answers FILE_TOO_LARGE at no charge, never cut short. */
37
45
  formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
38
46
  /** Read only the page's main content into `markdown`, `text` and `extract`: the article or the
39
47
  * product, without the menus, ads and related links around it. On by default. Set `false` for
@@ -75,8 +83,10 @@ export interface ScrapeParams {
75
83
  server?: string;
76
84
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
77
85
  * and the managed pool leaves from that country. Leave it out and the page sees a US
78
- * browser, from an address we pick. The browser's clock and languages follow the country
79
- * on every pool. */
86
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
87
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
88
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
89
+ * the country on every pool. */
80
90
  country?: string;
81
91
  /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
82
92
  * only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -89,7 +99,7 @@ export interface ScrapeParams {
89
99
  city?: string;
90
100
  };
91
101
  /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
92
- * turns out to need one. */
102
+ * turns out to need one, except `actions`, which always use the browser. */
93
103
  browser?: {
94
104
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
95
105
  * "domcontentloaded" is the fastest and stops at the HTML; "networkidle" waits until the
@@ -112,13 +122,80 @@ export interface ScrapeParams {
112
122
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
113
123
  * large answer: the image comes back as base64 inside the response. */
114
124
  fullPage?: boolean;
125
+ /** Steps to run in the page before it is read, in the order you list them. A step can wait,
126
+ * click, type, press a key, scroll, take a screenshot, keep the HTML or run a script.
127
+ * Steps always use the browser engine, at the browser price. The formats you ask for read
128
+ * the page as the last step left it. Up to 50 steps, at most 10 of them screenshots or
129
+ * scrapes. If one fails, the call answers ACTION_FAILED, which names the step and why.
130
+ * That costs nothing, unless one of your scripts started, the one that failed included:
131
+ * then the call is charged. */
132
+ actions?: ({
133
+ type: "wait";
134
+ /** How long to wait, in milliseconds. All the waits together have to fit inside
135
+ * `timeoutMs`. */
136
+ milliseconds?: number;
137
+ /** Wait until this CSS selector is on the page. If it never shows up, the step fails. */
138
+ selector?: string;
139
+ } | {
140
+ type: "click";
141
+ /** The CSS selector of what to click. The first match gets the click, and the step
142
+ * waits for it to show up. */
143
+ selector: string;
144
+ /** Click every visible match, not just the first. A selector that matches nothing is
145
+ * then not an error, which suits a cookie banner only some visitors see. */
146
+ all?: boolean;
147
+ } | {
148
+ type: "write";
149
+ /** What to type, up to 1,000 characters. */
150
+ text: string;
151
+ } | {
152
+ type: "press";
153
+ /** The key's name: "Enter", "Tab", "Escape", "ArrowDown", or two together such as
154
+ * "Control+A". */
155
+ key: string;
156
+ } | {
157
+ type: "scroll";
158
+ direction: "up" | "down";
159
+ /** Scroll this element instead of the whole page, such as a list with its own scroll
160
+ * bar. */
161
+ selector?: string;
162
+ } | {
163
+ type: "screenshot";
164
+ /** Take the whole scrollable page instead of just the window. */
165
+ fullPage?: boolean;
166
+ } | {
167
+ type: "scrape";
168
+ } | {
169
+ type: "executeJavascript";
170
+ /** The body of an async function, up to 8,000 characters, such as `return
171
+ * document.title`. It runs in the page, beside the page's own scripts. What it returns
172
+ * comes back in `actions.javascriptReturns` as JSON, or as null when JSON cannot hold
173
+ * it. All the scripts in one call can return 64 KB together. Once a script starts, the
174
+ * call is charged even if that script throws, a later step fails, the page crashes or
175
+ * times out, the site blocks it, the script leads to an address we refuse, your own
176
+ * proxy says no, or an `includeTags` or `excludeTags` selector is too slow for the
177
+ * page it built, because the script may already have read the page. A failure on our
178
+ * side (a 5xx) is still refunded. */
179
+ script: string;
180
+ })[];
115
181
  };
116
182
  /** Extra request headers to send to the page, such as an authorization token, a cookie or a
117
183
  * custom user agent. The connection's own headers cannot be set. */
118
184
  headers?: Record<string, string>;
119
- /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
120
- * free outcome. */
185
+ /** How long to give the page before giving up, in milliseconds. A page that never arrives costs
186
+ * nothing, unless your own `browser.actions` script ran in the page first. */
121
187
  timeoutMs?: number;
188
+ /** The oldest copy of the page you accept, in milliseconds, up to 3600000 (one hour). If we
189
+ * still hold a copy of the same URL that your account read within that time, from the same
190
+ * country and with the same engine, you get it back for 0 credits and the answer says `cached:
191
+ * true`. If not, we read the page now and keep a copy for up to an hour, unless the page still
192
+ * needs JavaScript (`metadata.needsJs`). We may drop a copy sooner, to make room or on a
193
+ * deploy, and a batch keeps its own copies, apart from single calls. `0` always reads the page
194
+ * now, and keeps the copy for the calls after it. `extract` is still charged, because the
195
+ * model reads the copy again. Leave `maxAge` out to read the page fresh every time. A POST, or
196
+ * a call with `headers`, your own proxy, `browser` options, a screenshot, a PDF, `raw` or
197
+ * `escalate: false`, is always read fresh. */
198
+ maxAge?: number;
122
199
  }
123
200
  /** Arguments for `quote()`. */
124
201
  export interface QuoteScrapeParams {
@@ -130,9 +207,10 @@ export interface QuoteScrapeParams {
130
207
  * always use "fetch", so leave it out for them. */
131
208
  engine?: "auto" | "fetch" | "browser";
132
209
  /** Whether to try again a different way when the first attempt is blocked. On by default.
133
- * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
134
- * how you bound a call: with `auto`, trying again is what charges the browser price instead of
135
- * the fetch one. A POST is sent once, so it is off for a POST. */
210
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing, unless
211
+ * your own `browser.actions` script ran in the page first. It is how you bound a call: with
212
+ * `auto`, trying again is what charges the browser price instead of the fetch one. A POST is
213
+ * sent once, so it is off for a POST. */
136
214
  escalate?: boolean;
137
215
  /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
138
216
  * sent once: never retried and never sent again another way, because sending it twice could
@@ -147,15 +225,22 @@ export interface QuoteScrapeParams {
147
225
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
148
226
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
149
227
  * that is blocked again is free, but it takes as long as the first one did, so the call can
150
- * run that many times longer. */
228
+ * run that many times longer. Once your `browser.actions` steps click, type, press a key or
229
+ * run a script on a page, it is not tried again, so nothing is done twice. If one of your
230
+ * scripts ran in it, the block is charged. */
151
231
  retry?: number;
152
232
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
153
233
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
154
234
  * and needs an `extract` block saying which ones. A link to a PDF or a Word (.docx) file is
155
- * read into text, so the text formats work on it too. We read files up to 25 MB and the first
156
- * 200 pages, at the fetch price. "raw" is the file exactly as the site sent it, as base64 with
157
- * its content type: a PDF report, an image, a spreadsheet. It uses the fetch engine and holds
158
- * up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never cut short. */
235
+ * read into text, so the text formats work on it too. We read files up to 25 MB, and from a
236
+ * PDF the first 200 pages and the first 5 million characters. Reading a file costs what the
237
+ * engine costs: 1 credit with fetch, 5 with the browser. A Word file that holds more than 32
238
+ * MB once unzipped answers FILE_TOO_LARGE, and a file link that answers a short web page
239
+ * instead of the file answers UPSTREAM_BLOCKED, both at no charge. A dead link is the
240
+ * exception: its 404 or 410 page is the site's answer, and comes back with that status at the
241
+ * fetch price. "raw" is the file exactly as the site sent it, as base64 with its content type:
242
+ * a PDF report, an image, a spreadsheet. It uses the fetch engine and holds up to 5 MB; a
243
+ * larger file answers FILE_TOO_LARGE at no charge, never cut short. */
159
244
  formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
160
245
  /** Read only the page's main content into `markdown`, `text` and `extract`: the article or the
161
246
  * product, without the menus, ads and related links around it. On by default. Set `false` for
@@ -197,8 +282,10 @@ export interface QuoteScrapeParams {
197
282
  server?: string;
198
283
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
199
284
  * and the managed pool leaves from that country. Leave it out and the page sees a US
200
- * browser, from an address we pick. The browser's clock and languages follow the country
201
- * on every pool. */
285
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
286
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
287
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
288
+ * the country on every pool. */
202
289
  country?: string;
203
290
  /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
204
291
  * only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -211,7 +298,7 @@ export interface QuoteScrapeParams {
211
298
  city?: string;
212
299
  };
213
300
  /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
214
- * turns out to need one. */
301
+ * turns out to need one, except `actions`, which always use the browser. */
215
302
  browser?: {
216
303
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
217
304
  * "domcontentloaded" is the fastest and stops at the HTML; "networkidle" waits until the
@@ -234,13 +321,80 @@ export interface QuoteScrapeParams {
234
321
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
235
322
  * large answer: the image comes back as base64 inside the response. */
236
323
  fullPage?: boolean;
324
+ /** Steps to run in the page before it is read, in the order you list them. A step can wait,
325
+ * click, type, press a key, scroll, take a screenshot, keep the HTML or run a script.
326
+ * Steps always use the browser engine, at the browser price. The formats you ask for read
327
+ * the page as the last step left it. Up to 50 steps, at most 10 of them screenshots or
328
+ * scrapes. If one fails, the call answers ACTION_FAILED, which names the step and why.
329
+ * That costs nothing, unless one of your scripts started, the one that failed included:
330
+ * then the call is charged. */
331
+ actions?: ({
332
+ type: "wait";
333
+ /** How long to wait, in milliseconds. All the waits together have to fit inside
334
+ * `timeoutMs`. */
335
+ milliseconds?: number;
336
+ /** Wait until this CSS selector is on the page. If it never shows up, the step fails. */
337
+ selector?: string;
338
+ } | {
339
+ type: "click";
340
+ /** The CSS selector of what to click. The first match gets the click, and the step
341
+ * waits for it to show up. */
342
+ selector: string;
343
+ /** Click every visible match, not just the first. A selector that matches nothing is
344
+ * then not an error, which suits a cookie banner only some visitors see. */
345
+ all?: boolean;
346
+ } | {
347
+ type: "write";
348
+ /** What to type, up to 1,000 characters. */
349
+ text: string;
350
+ } | {
351
+ type: "press";
352
+ /** The key's name: "Enter", "Tab", "Escape", "ArrowDown", or two together such as
353
+ * "Control+A". */
354
+ key: string;
355
+ } | {
356
+ type: "scroll";
357
+ direction: "up" | "down";
358
+ /** Scroll this element instead of the whole page, such as a list with its own scroll
359
+ * bar. */
360
+ selector?: string;
361
+ } | {
362
+ type: "screenshot";
363
+ /** Take the whole scrollable page instead of just the window. */
364
+ fullPage?: boolean;
365
+ } | {
366
+ type: "scrape";
367
+ } | {
368
+ type: "executeJavascript";
369
+ /** The body of an async function, up to 8,000 characters, such as `return
370
+ * document.title`. It runs in the page, beside the page's own scripts. What it returns
371
+ * comes back in `actions.javascriptReturns` as JSON, or as null when JSON cannot hold
372
+ * it. All the scripts in one call can return 64 KB together. Once a script starts, the
373
+ * call is charged even if that script throws, a later step fails, the page crashes or
374
+ * times out, the site blocks it, the script leads to an address we refuse, your own
375
+ * proxy says no, or an `includeTags` or `excludeTags` selector is too slow for the
376
+ * page it built, because the script may already have read the page. A failure on our
377
+ * side (a 5xx) is still refunded. */
378
+ script: string;
379
+ })[];
237
380
  };
238
381
  /** Extra request headers to send to the page, such as an authorization token, a cookie or a
239
382
  * custom user agent. The connection's own headers cannot be set. */
240
383
  headers?: Record<string, string>;
241
- /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
242
- * free outcome. */
384
+ /** How long to give the page before giving up, in milliseconds. A page that never arrives costs
385
+ * nothing, unless your own `browser.actions` script ran in the page first. */
243
386
  timeoutMs?: number;
387
+ /** The oldest copy of the page you accept, in milliseconds, up to 3600000 (one hour). If we
388
+ * still hold a copy of the same URL that your account read within that time, from the same
389
+ * country and with the same engine, you get it back for 0 credits and the answer says `cached:
390
+ * true`. If not, we read the page now and keep a copy for up to an hour, unless the page still
391
+ * needs JavaScript (`metadata.needsJs`). We may drop a copy sooner, to make room or on a
392
+ * deploy, and a batch keeps its own copies, apart from single calls. `0` always reads the page
393
+ * now, and keeps the copy for the calls after it. `extract` is still charged, because the
394
+ * model reads the copy again. Leave `maxAge` out to read the page fresh every time. A POST, or
395
+ * a call with `headers`, your own proxy, `browser` options, a screenshot, a PDF, `raw` or
396
+ * `escalate: false`, is always read fresh. */
397
+ maxAge?: number;
244
398
  }
245
399
  /** Arguments for `map()`. */
246
400
  export interface MapSiteParams {
@@ -267,8 +421,10 @@ export interface MapSiteParams {
267
421
  server?: string;
268
422
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
269
423
  * and the managed pool leaves from that country. Leave it out and the page sees a US
270
- * browser, from an address we pick. The browser's clock and languages follow the country
271
- * on every pool. */
424
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
425
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
426
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
427
+ * the country on every pool. */
272
428
  country?: string;
273
429
  };
274
430
  }
@@ -297,8 +453,10 @@ export interface QuoteMapParams {
297
453
  server?: string;
298
454
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
299
455
  * and the managed pool leaves from that country. Leave it out and the page sees a US
300
- * browser, from an address we pick. The browser's clock and languages follow the country
301
- * on every pool. */
456
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
457
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
458
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
459
+ * the country on every pool. */
302
460
  country?: string;
303
461
  };
304
462
  }
@@ -311,13 +469,15 @@ export interface SearchParams {
311
469
  * `includeDomains` filters what it returns. */
312
470
  limit?: number;
313
471
  /** What kind of results. "general" is the web; "news" is dated journalism, and the only topic
314
- * that reliably carries `publishedAt`. */
315
- topic?: "general" | "news";
472
+ * that reliably carries `publishedAt`; "images" is pictures: each result is the page that
473
+ * shows one, with the picture's own address, a thumbnail and its size in `image`. Pictures the
474
+ * engine marks as adult are left out. */
475
+ topic?: "general" | "news" | "images";
316
476
  /** Only pages from the last day, week, month or year. Absent is any time. */
317
477
  timeRange?: "day" | "week" | "month" | "year";
318
478
  /** Only results on these hosts, bare (`linkedin.com`). Subdomains are included. Asked of the
319
479
  * search front AND enforced on the answer, so what comes back is on your list whatever the
320
- * front did with it. */
480
+ * front did with it. With `topic: "images"` it takes one host. */
321
481
  includeDomains?: string[];
322
482
  /** Also write a short paragraph answering the query from the results, in the query's language.
323
483
  * Costs one extra credit; leave it off when your own model reads the snippets. */
@@ -333,8 +493,10 @@ export interface SearchParams {
333
493
  server?: string;
334
494
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
335
495
  * and the managed pool leaves from that country. Leave it out and the page sees a US
336
- * browser, from an address we pick. The browser's clock and languages follow the country
337
- * on every pool. */
496
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
497
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
498
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
499
+ * the country on every pool. */
338
500
  country?: string;
339
501
  };
340
502
  }
@@ -346,22 +508,30 @@ export interface CreateBatchParams {
346
508
  * always use "fetch", so leave it out for them. */
347
509
  engine?: "auto" | "fetch" | "browser";
348
510
  /** Whether to try again a different way when the first attempt is blocked. On by default.
349
- * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
350
- * how you bound a call: with `auto`, trying again is what charges the browser price instead of
351
- * the fetch one. A POST is sent once, so it is off for a POST. */
511
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing, unless
512
+ * your own `browser.actions` script ran in the page first. It is how you bound a call: with
513
+ * `auto`, trying again is what charges the browser price instead of the fetch one. A POST is
514
+ * sent once, so it is off for a POST. */
352
515
  escalate?: boolean;
353
516
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
354
517
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
355
518
  * that is blocked again is free, but it takes as long as the first one did, so the call can
356
- * run that many times longer. */
519
+ * run that many times longer. Once your `browser.actions` steps click, type, press a key or
520
+ * run a script on a page, it is not tried again, so nothing is done twice. If one of your
521
+ * scripts ran in it, the block is charged. */
357
522
  retry?: number;
358
523
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
359
524
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
360
525
  * and needs an `extract` block saying which ones. A link to a PDF or a Word (.docx) file is
361
- * read into text, so the text formats work on it too. We read files up to 25 MB and the first
362
- * 200 pages, at the fetch price. "raw" is the file exactly as the site sent it, as base64 with
363
- * its content type: a PDF report, an image, a spreadsheet. It uses the fetch engine and holds
364
- * up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never cut short. */
526
+ * read into text, so the text formats work on it too. We read files up to 25 MB, and from a
527
+ * PDF the first 200 pages and the first 5 million characters. Reading a file costs what the
528
+ * engine costs: 1 credit with fetch, 5 with the browser. A Word file that holds more than 32
529
+ * MB once unzipped answers FILE_TOO_LARGE, and a file link that answers a short web page
530
+ * instead of the file answers UPSTREAM_BLOCKED, both at no charge. A dead link is the
531
+ * exception: its 404 or 410 page is the site's answer, and comes back with that status at the
532
+ * fetch price. "raw" is the file exactly as the site sent it, as base64 with its content type:
533
+ * a PDF report, an image, a spreadsheet. It uses the fetch engine and holds up to 5 MB; a
534
+ * larger file answers FILE_TOO_LARGE at no charge, never cut short. */
365
535
  formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
366
536
  /** Read only the page's main content into `markdown`, `text` and `extract`: the article or the
367
537
  * product, without the menus, ads and related links around it. On by default. Set `false` for
@@ -403,8 +573,10 @@ export interface CreateBatchParams {
403
573
  server?: string;
404
574
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
405
575
  * and the managed pool leaves from that country. Leave it out and the page sees a US
406
- * browser, from an address we pick. The browser's clock and languages follow the country
407
- * on every pool. */
576
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
577
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
578
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
579
+ * the country on every pool. */
408
580
  country?: string;
409
581
  /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
410
582
  * only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -417,7 +589,7 @@ export interface CreateBatchParams {
417
589
  city?: string;
418
590
  };
419
591
  /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
420
- * turns out to need one. */
592
+ * turns out to need one, except `actions`, which always use the browser. */
421
593
  browser?: {
422
594
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
423
595
  * "domcontentloaded" is the fastest and stops at the HTML; "networkidle" waits until the
@@ -440,13 +612,80 @@ export interface CreateBatchParams {
440
612
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
441
613
  * large answer: the image comes back as base64 inside the response. */
442
614
  fullPage?: boolean;
615
+ /** Steps to run in the page before it is read, in the order you list them. A step can wait,
616
+ * click, type, press a key, scroll, take a screenshot, keep the HTML or run a script.
617
+ * Steps always use the browser engine, at the browser price. The formats you ask for read
618
+ * the page as the last step left it. Up to 50 steps, at most 10 of them screenshots or
619
+ * scrapes. If one fails, the call answers ACTION_FAILED, which names the step and why.
620
+ * That costs nothing, unless one of your scripts started, the one that failed included:
621
+ * then the call is charged. */
622
+ actions?: ({
623
+ type: "wait";
624
+ /** How long to wait, in milliseconds. All the waits together have to fit inside
625
+ * `timeoutMs`. */
626
+ milliseconds?: number;
627
+ /** Wait until this CSS selector is on the page. If it never shows up, the step fails. */
628
+ selector?: string;
629
+ } | {
630
+ type: "click";
631
+ /** The CSS selector of what to click. The first match gets the click, and the step
632
+ * waits for it to show up. */
633
+ selector: string;
634
+ /** Click every visible match, not just the first. A selector that matches nothing is
635
+ * then not an error, which suits a cookie banner only some visitors see. */
636
+ all?: boolean;
637
+ } | {
638
+ type: "write";
639
+ /** What to type, up to 1,000 characters. */
640
+ text: string;
641
+ } | {
642
+ type: "press";
643
+ /** The key's name: "Enter", "Tab", "Escape", "ArrowDown", or two together such as
644
+ * "Control+A". */
645
+ key: string;
646
+ } | {
647
+ type: "scroll";
648
+ direction: "up" | "down";
649
+ /** Scroll this element instead of the whole page, such as a list with its own scroll
650
+ * bar. */
651
+ selector?: string;
652
+ } | {
653
+ type: "screenshot";
654
+ /** Take the whole scrollable page instead of just the window. */
655
+ fullPage?: boolean;
656
+ } | {
657
+ type: "scrape";
658
+ } | {
659
+ type: "executeJavascript";
660
+ /** The body of an async function, up to 8,000 characters, such as `return
661
+ * document.title`. It runs in the page, beside the page's own scripts. What it returns
662
+ * comes back in `actions.javascriptReturns` as JSON, or as null when JSON cannot hold
663
+ * it. All the scripts in one call can return 64 KB together. Once a script starts, the
664
+ * call is charged even if that script throws, a later step fails, the page crashes or
665
+ * times out, the site blocks it, the script leads to an address we refuse, your own
666
+ * proxy says no, or an `includeTags` or `excludeTags` selector is too slow for the
667
+ * page it built, because the script may already have read the page. A failure on our
668
+ * side (a 5xx) is still refunded. */
669
+ script: string;
670
+ })[];
443
671
  };
444
672
  /** Extra request headers to send to the page, such as an authorization token, a cookie or a
445
673
  * custom user agent. The connection's own headers cannot be set. */
446
674
  headers?: Record<string, string>;
447
- /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
448
- * free outcome. */
675
+ /** How long to give the page before giving up, in milliseconds. A page that never arrives costs
676
+ * nothing, unless your own `browser.actions` script ran in the page first. */
449
677
  timeoutMs?: number;
678
+ /** The oldest copy of the page you accept, in milliseconds, up to 3600000 (one hour). If we
679
+ * still hold a copy of the same URL that your account read within that time, from the same
680
+ * country and with the same engine, you get it back for 0 credits and the answer says `cached:
681
+ * true`. If not, we read the page now and keep a copy for up to an hour, unless the page still
682
+ * needs JavaScript (`metadata.needsJs`). We may drop a copy sooner, to make room or on a
683
+ * deploy, and a batch keeps its own copies, apart from single calls. `0` always reads the page
684
+ * now, and keeps the copy for the calls after it. `extract` is still charged, because the
685
+ * model reads the copy again. Leave `maxAge` out to read the page fresh every time. A POST, or
686
+ * a call with `headers`, your own proxy, `browser` options, a screenshot, a PDF, `raw` or
687
+ * `escalate: false`, is always read fresh. */
688
+ maxAge?: number;
450
689
  /** The pages to read, in the order you want them back. How many one batch may carry depends on
451
690
  * your plan. Duplicates are kept: position, not URL, is what lines results up against what you
452
691
  * submitted. */
@@ -472,22 +711,30 @@ export interface QuoteBatchParams {
472
711
  * always use "fetch", so leave it out for them. */
473
712
  engine?: "auto" | "fetch" | "browser";
474
713
  /** Whether to try again a different way when the first attempt is blocked. On by default.
475
- * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
476
- * how you bound a call: with `auto`, trying again is what charges the browser price instead of
477
- * the fetch one. A POST is sent once, so it is off for a POST. */
714
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing, unless
715
+ * your own `browser.actions` script ran in the page first. It is how you bound a call: with
716
+ * `auto`, trying again is what charges the browser price instead of the fetch one. A POST is
717
+ * sent once, so it is off for a POST. */
478
718
  escalate?: boolean;
479
719
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
480
720
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
481
721
  * that is blocked again is free, but it takes as long as the first one did, so the call can
482
- * run that many times longer. */
722
+ * run that many times longer. Once your `browser.actions` steps click, type, press a key or
723
+ * run a script on a page, it is not tried again, so nothing is done twice. If one of your
724
+ * scripts ran in it, the block is charged. */
483
725
  retry?: number;
484
726
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
485
727
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
486
728
  * and needs an `extract` block saying which ones. A link to a PDF or a Word (.docx) file is
487
- * read into text, so the text formats work on it too. We read files up to 25 MB and the first
488
- * 200 pages, at the fetch price. "raw" is the file exactly as the site sent it, as base64 with
489
- * its content type: a PDF report, an image, a spreadsheet. It uses the fetch engine and holds
490
- * up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never cut short. */
729
+ * read into text, so the text formats work on it too. We read files up to 25 MB, and from a
730
+ * PDF the first 200 pages and the first 5 million characters. Reading a file costs what the
731
+ * engine costs: 1 credit with fetch, 5 with the browser. A Word file that holds more than 32
732
+ * MB once unzipped answers FILE_TOO_LARGE, and a file link that answers a short web page
733
+ * instead of the file answers UPSTREAM_BLOCKED, both at no charge. A dead link is the
734
+ * exception: its 404 or 410 page is the site's answer, and comes back with that status at the
735
+ * fetch price. "raw" is the file exactly as the site sent it, as base64 with its content type:
736
+ * a PDF report, an image, a spreadsheet. It uses the fetch engine and holds up to 5 MB; a
737
+ * larger file answers FILE_TOO_LARGE at no charge, never cut short. */
491
738
  formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
492
739
  /** Read only the page's main content into `markdown`, `text` and `extract`: the article or the
493
740
  * product, without the menus, ads and related links around it. On by default. Set `false` for
@@ -529,8 +776,10 @@ export interface QuoteBatchParams {
529
776
  server?: string;
530
777
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
531
778
  * and the managed pool leaves from that country. Leave it out and the page sees a US
532
- * browser, from an address we pick. The browser's clock and languages follow the country
533
- * on every pool. */
779
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
780
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
781
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
782
+ * the country on every pool. */
534
783
  country?: string;
535
784
  /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
536
785
  * only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -543,7 +792,7 @@ export interface QuoteBatchParams {
543
792
  city?: string;
544
793
  };
545
794
  /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
546
- * turns out to need one. */
795
+ * turns out to need one, except `actions`, which always use the browser. */
547
796
  browser?: {
548
797
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
549
798
  * "domcontentloaded" is the fastest and stops at the HTML; "networkidle" waits until the
@@ -566,13 +815,80 @@ export interface QuoteBatchParams {
566
815
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
567
816
  * large answer: the image comes back as base64 inside the response. */
568
817
  fullPage?: boolean;
818
+ /** Steps to run in the page before it is read, in the order you list them. A step can wait,
819
+ * click, type, press a key, scroll, take a screenshot, keep the HTML or run a script.
820
+ * Steps always use the browser engine, at the browser price. The formats you ask for read
821
+ * the page as the last step left it. Up to 50 steps, at most 10 of them screenshots or
822
+ * scrapes. If one fails, the call answers ACTION_FAILED, which names the step and why.
823
+ * That costs nothing, unless one of your scripts started, the one that failed included:
824
+ * then the call is charged. */
825
+ actions?: ({
826
+ type: "wait";
827
+ /** How long to wait, in milliseconds. All the waits together have to fit inside
828
+ * `timeoutMs`. */
829
+ milliseconds?: number;
830
+ /** Wait until this CSS selector is on the page. If it never shows up, the step fails. */
831
+ selector?: string;
832
+ } | {
833
+ type: "click";
834
+ /** The CSS selector of what to click. The first match gets the click, and the step
835
+ * waits for it to show up. */
836
+ selector: string;
837
+ /** Click every visible match, not just the first. A selector that matches nothing is
838
+ * then not an error, which suits a cookie banner only some visitors see. */
839
+ all?: boolean;
840
+ } | {
841
+ type: "write";
842
+ /** What to type, up to 1,000 characters. */
843
+ text: string;
844
+ } | {
845
+ type: "press";
846
+ /** The key's name: "Enter", "Tab", "Escape", "ArrowDown", or two together such as
847
+ * "Control+A". */
848
+ key: string;
849
+ } | {
850
+ type: "scroll";
851
+ direction: "up" | "down";
852
+ /** Scroll this element instead of the whole page, such as a list with its own scroll
853
+ * bar. */
854
+ selector?: string;
855
+ } | {
856
+ type: "screenshot";
857
+ /** Take the whole scrollable page instead of just the window. */
858
+ fullPage?: boolean;
859
+ } | {
860
+ type: "scrape";
861
+ } | {
862
+ type: "executeJavascript";
863
+ /** The body of an async function, up to 8,000 characters, such as `return
864
+ * document.title`. It runs in the page, beside the page's own scripts. What it returns
865
+ * comes back in `actions.javascriptReturns` as JSON, or as null when JSON cannot hold
866
+ * it. All the scripts in one call can return 64 KB together. Once a script starts, the
867
+ * call is charged even if that script throws, a later step fails, the page crashes or
868
+ * times out, the site blocks it, the script leads to an address we refuse, your own
869
+ * proxy says no, or an `includeTags` or `excludeTags` selector is too slow for the
870
+ * page it built, because the script may already have read the page. A failure on our
871
+ * side (a 5xx) is still refunded. */
872
+ script: string;
873
+ })[];
569
874
  };
570
875
  /** Extra request headers to send to the page, such as an authorization token, a cookie or a
571
876
  * custom user agent. The connection's own headers cannot be set. */
572
877
  headers?: Record<string, string>;
573
- /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
574
- * free outcome. */
878
+ /** How long to give the page before giving up, in milliseconds. A page that never arrives costs
879
+ * nothing, unless your own `browser.actions` script ran in the page first. */
575
880
  timeoutMs?: number;
881
+ /** The oldest copy of the page you accept, in milliseconds, up to 3600000 (one hour). If we
882
+ * still hold a copy of the same URL that your account read within that time, from the same
883
+ * country and with the same engine, you get it back for 0 credits and the answer says `cached:
884
+ * true`. If not, we read the page now and keep a copy for up to an hour, unless the page still
885
+ * needs JavaScript (`metadata.needsJs`). We may drop a copy sooner, to make room or on a
886
+ * deploy, and a batch keeps its own copies, apart from single calls. `0` always reads the page
887
+ * now, and keeps the copy for the calls after it. `extract` is still charged, because the
888
+ * model reads the copy again. Leave `maxAge` out to read the page fresh every time. A POST, or
889
+ * a call with `headers`, your own proxy, `browser` options, a screenshot, a PDF, `raw` or
890
+ * `escalate: false`, is always read fresh. */
891
+ maxAge?: number;
576
892
  /** The pages to read, in the order you want them back. How many one batch may carry depends on
577
893
  * your plan. Duplicates are kept: position, not URL, is what lines results up against what you
578
894
  * submitted. */
@@ -628,8 +944,10 @@ export interface CreateSessionParams {
628
944
  server?: string;
629
945
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
630
946
  * and the managed pool leaves from that country. Leave it out and the page sees a US
631
- * browser, from an address we pick. The browser's clock and languages follow the country
632
- * on every pool. */
947
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
948
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
949
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
950
+ * the country on every pool. */
633
951
  country?: string;
634
952
  /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
635
953
  * only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and
@@ -659,8 +977,10 @@ export interface QuoteSessionParams {
659
977
  server?: string;
660
978
  /** Where the call should look like it comes from, ISO 3166-1 alpha-2 (`us`, `br`). Name one
661
979
  * and the managed pool leaves from that country. Leave it out and the page sees a US
662
- * browser, from an address we pick. The browser's clock and languages follow the country
663
- * on every pool. */
980
+ * browser, from an address we pick. A scrape or a map on the managed pool makes one
981
+ * exception: a site whose address ends in `.br`, `.de`, `.es`, `.fr`, `.it`, `.jp`, `.nl`,
982
+ * `.pl` or `.uk` is read from its own country. The browser's clock and languages follow
983
+ * the country on every pool. */
664
984
  country?: string;
665
985
  /** A state or region inside `country`, by name: `california`, `são paulo`. Managed pool
666
986
  * only. Spell the name out, since an abbreviation like `sp` finds nobody. In the US and