@scrapa/sdk 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,35 +1,50 @@
1
1
  /** Arguments for `scrape()`. */
2
2
  export interface ScrapeParams {
3
- /** The page to read. An http(s) URL — private and loopback hosts are refused. */
3
+ /** The page to read. An http(s) URL. Private and loopback hosts are refused. */
4
4
  url: string;
5
5
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
6
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
7
- * JavaScript shell, billing whichever engine answered. */
6
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
7
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
8
+ * always use "fetch", so leave it out for them. */
8
9
  engine?: "auto" | "fetch" | "browser";
9
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
10
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
11
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
10
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
11
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
12
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
13
+ * the fetch one. A POST is sent once, so it is off for a POST. */
12
14
  escalate?: boolean;
15
+ /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
16
+ * sent once: never retried and never sent again another way, because sending it twice could
17
+ * repeat what it did. */
18
+ method?: "GET" | "POST";
19
+ /** What a POST sends, at most 64 KB. Set its type in `headers`: `{"content-type":
20
+ * "application/json"}` for JSON, or `application/x-www-form-urlencoded` for a form. If the
21
+ * site redirects, a 301, 302 or 303 reads the next page with GET and without the body; a 307
22
+ * or 308 sends it again only if the protocol, host and port stay the same, and otherwise stops
23
+ * and names the new address. */
24
+ body?: string;
13
25
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
14
26
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
15
- * that is blocked again is free — but it takes as long as the first one did, so the call can
27
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
16
28
  * run that many times longer. */
17
29
  retry?: number;
18
30
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
19
31
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
20
- * and needs an `extract` block saying which ones. */
21
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
32
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
33
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
34
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
35
+ * cut short. */
36
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
22
37
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
23
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
24
- * then not your answer. It costs nothing, and we try again from another address instead of
25
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
38
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
39
+ * not your answer. It costs nothing, and we try again from another address instead of handing
40
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
26
41
  expect?: "json";
27
42
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
28
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
43
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
29
44
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
30
45
  * code. */
31
46
  extract?: {
32
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
47
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
33
48
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
34
49
  schema?: Record<string, unknown>;
35
50
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -40,7 +55,7 @@ export interface ScrapeParams {
40
55
  /** Where the request leaves from. Defaults to the managed residential pool. */
41
56
  proxy?: {
42
57
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
43
- * "custom" is your own proxy — free on every plan, bytes never billed. */
58
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
44
59
  pool?: "residential" | "custom";
45
60
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
46
61
  * for the managed one. */
@@ -60,7 +75,7 @@ export interface ScrapeParams {
60
75
  * charge; ask for the state or the country alone. */
61
76
  city?: string;
62
77
  };
63
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
78
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
64
79
  * turns out to need one. */
65
80
  browser?: {
66
81
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -68,9 +83,9 @@ export interface ScrapeParams {
68
83
  * page stops asking for things. Every page gets a short settle after this whichever you
69
84
  * pick. */
70
85
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
71
- /** Wait until this CSS selector is on the page — the dependable way to read something a
72
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
73
- * than the half-built page. */
86
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
87
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
88
+ * rather than the half-built page. */
74
89
  waitForSelector?: string;
75
90
  /** Wait this many milliseconds after the page settles, for the scripts no selector
76
91
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -82,11 +97,11 @@ export interface ScrapeParams {
82
97
  height: number;
83
98
  };
84
99
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
85
- * large answer — the image comes back as base64 inside the response. */
100
+ * large answer: the image comes back as base64 inside the response. */
86
101
  fullPage?: boolean;
87
102
  };
88
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
89
- * agent. The connection's own headers cannot be set. */
103
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
104
+ * custom user agent. The connection's own headers cannot be set. */
90
105
  headers?: Record<string, string>;
91
106
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
92
107
  * free outcome. */
@@ -94,36 +109,51 @@ export interface ScrapeParams {
94
109
  }
95
110
  /** Arguments for `quote()`. */
96
111
  export interface QuoteScrapeParams {
97
- /** The page to read. An http(s) URL — private and loopback hosts are refused. */
112
+ /** The page to read. An http(s) URL. Private and loopback hosts are refused. */
98
113
  url: string;
99
114
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
100
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
101
- * JavaScript shell, billing whichever engine answered. */
115
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
116
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
117
+ * always use "fetch", so leave it out for them. */
102
118
  engine?: "auto" | "fetch" | "browser";
103
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
104
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
105
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
119
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
120
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
121
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
122
+ * the fetch one. A POST is sent once, so it is off for a POST. */
106
123
  escalate?: boolean;
124
+ /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
125
+ * sent once: never retried and never sent again another way, because sending it twice could
126
+ * repeat what it did. */
127
+ method?: "GET" | "POST";
128
+ /** What a POST sends, at most 64 KB. Set its type in `headers`: `{"content-type":
129
+ * "application/json"}` for JSON, or `application/x-www-form-urlencoded` for a form. If the
130
+ * site redirects, a 301, 302 or 303 reads the next page with GET and without the body; a 307
131
+ * or 308 sends it again only if the protocol, host and port stay the same, and otherwise stops
132
+ * and names the new address. */
133
+ body?: string;
107
134
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
108
135
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
109
- * that is blocked again is free — but it takes as long as the first one did, so the call can
136
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
110
137
  * run that many times longer. */
111
138
  retry?: number;
112
139
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
113
140
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
114
- * and needs an `extract` block saying which ones. */
115
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
141
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
142
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
143
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
144
+ * cut short. */
145
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
116
146
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
117
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
118
- * then not your answer. It costs nothing, and we try again from another address instead of
119
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
147
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
148
+ * not your answer. It costs nothing, and we try again from another address instead of handing
149
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
120
150
  expect?: "json";
121
151
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
122
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
152
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
123
153
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
124
154
  * code. */
125
155
  extract?: {
126
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
156
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
127
157
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
128
158
  schema?: Record<string, unknown>;
129
159
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -134,7 +164,7 @@ export interface QuoteScrapeParams {
134
164
  /** Where the request leaves from. Defaults to the managed residential pool. */
135
165
  proxy?: {
136
166
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
137
- * "custom" is your own proxy — free on every plan, bytes never billed. */
167
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
138
168
  pool?: "residential" | "custom";
139
169
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
140
170
  * for the managed one. */
@@ -154,7 +184,7 @@ export interface QuoteScrapeParams {
154
184
  * charge; ask for the state or the country alone. */
155
185
  city?: string;
156
186
  };
157
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
187
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
158
188
  * turns out to need one. */
159
189
  browser?: {
160
190
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -162,9 +192,9 @@ export interface QuoteScrapeParams {
162
192
  * page stops asking for things. Every page gets a short settle after this whichever you
163
193
  * pick. */
164
194
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
165
- /** Wait until this CSS selector is on the page — the dependable way to read something a
166
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
167
- * than the half-built page. */
195
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
196
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
197
+ * rather than the half-built page. */
168
198
  waitForSelector?: string;
169
199
  /** Wait this many milliseconds after the page settles, for the scripts no selector
170
200
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -176,11 +206,11 @@ export interface QuoteScrapeParams {
176
206
  height: number;
177
207
  };
178
208
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
179
- * large answer — the image comes back as base64 inside the response. */
209
+ * large answer: the image comes back as base64 inside the response. */
180
210
  fullPage?: boolean;
181
211
  };
182
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
183
- * agent. The connection's own headers cannot be set. */
212
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
213
+ * custom user agent. The connection's own headers cannot be set. */
184
214
  headers?: Record<string, string>;
185
215
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
186
216
  * free outcome. */
@@ -188,10 +218,10 @@ export interface QuoteScrapeParams {
188
218
  }
189
219
  /** Arguments for `search()`. */
190
220
  export interface SearchParams {
191
- /** What to search for, in plain words — the same thing you would type. */
221
+ /** What to search for, in plain words: the same thing you would type. */
192
222
  query: string;
193
223
  /** How many results to hand back at most, 1–20. Billed per 10, so asking for more than 10 costs
194
- * a second row. Fewer can come back than you asked for — an index has what it has, and
224
+ * a second row. Fewer can come back than you asked for: an index has what it has, and
195
225
  * `includeDomains` filters what it returns. */
196
226
  limit?: number;
197
227
  /** What kind of results. "general" is the web; "news" is dated journalism, and the only topic
@@ -199,18 +229,18 @@ export interface SearchParams {
199
229
  topic?: "general" | "news";
200
230
  /** Only pages from the last day, week, month or year. Absent is any time. */
201
231
  timeRange?: "day" | "week" | "month" | "year";
202
- /** Only results on these hosts, bare (`linkedin.com`) — subdomains included. Asked of the
232
+ /** Only results on these hosts, bare (`linkedin.com`). Subdomains are included. Asked of the
203
233
  * search front AND enforced on the answer, so what comes back is on your list whatever the
204
234
  * front did with it. */
205
235
  includeDomains?: string[];
206
236
  /** Also write a short paragraph answering the query from the results, in the query's language.
207
237
  * Costs one extra credit; leave it off when your own model reads the snippets. */
208
238
  answer?: boolean;
209
- /** Where the search leaves from. `country` picks the exit AND the market the front answers for
210
- * — `br` gets Brazilian results in Portuguese. */
239
+ /** Where the search leaves from. `country` picks the exit AND the market the front answers for.
240
+ * `br` gets Brazilian results in Portuguese. */
211
241
  proxy?: {
212
242
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
213
- * "custom" is your own proxy — free on every plan, bytes never billed. */
243
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
214
244
  pool?: "residential" | "custom";
215
245
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
216
246
  * for the managed one. */
@@ -225,33 +255,38 @@ export interface SearchParams {
225
255
  /** Arguments for `batches.create()`. */
226
256
  export interface CreateBatchParams {
227
257
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
228
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
229
- * JavaScript shell, billing whichever engine answered. */
258
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
259
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
260
+ * always use "fetch", so leave it out for them. */
230
261
  engine?: "auto" | "fetch" | "browser";
231
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
232
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
233
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
262
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
263
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
264
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
265
+ * the fetch one. A POST is sent once, so it is off for a POST. */
234
266
  escalate?: boolean;
235
267
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
236
268
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
237
- * that is blocked again is free — but it takes as long as the first one did, so the call can
269
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
238
270
  * run that many times longer. */
239
271
  retry?: number;
240
272
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
241
273
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
242
- * and needs an `extract` block saying which ones. */
243
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
274
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
275
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
276
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
277
+ * cut short. */
278
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
244
279
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
245
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
246
- * then not your answer. It costs nothing, and we try again from another address instead of
247
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
280
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
281
+ * not your answer. It costs nothing, and we try again from another address instead of handing
282
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
248
283
  expect?: "json";
249
284
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
250
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
285
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
251
286
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
252
287
  * code. */
253
288
  extract?: {
254
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
289
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
255
290
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
256
291
  schema?: Record<string, unknown>;
257
292
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -262,7 +297,7 @@ export interface CreateBatchParams {
262
297
  /** Where the request leaves from. Defaults to the managed residential pool. */
263
298
  proxy?: {
264
299
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
265
- * "custom" is your own proxy — free on every plan, bytes never billed. */
300
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
266
301
  pool?: "residential" | "custom";
267
302
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
268
303
  * for the managed one. */
@@ -282,7 +317,7 @@ export interface CreateBatchParams {
282
317
  * charge; ask for the state or the country alone. */
283
318
  city?: string;
284
319
  };
285
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
320
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
286
321
  * turns out to need one. */
287
322
  browser?: {
288
323
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -290,9 +325,9 @@ export interface CreateBatchParams {
290
325
  * page stops asking for things. Every page gets a short settle after this whichever you
291
326
  * pick. */
292
327
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
293
- /** Wait until this CSS selector is on the page — the dependable way to read something a
294
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
295
- * than the half-built page. */
328
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
329
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
330
+ * rather than the half-built page. */
296
331
  waitForSelector?: string;
297
332
  /** Wait this many milliseconds after the page settles, for the scripts no selector
298
333
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -304,11 +339,11 @@ export interface CreateBatchParams {
304
339
  height: number;
305
340
  };
306
341
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
307
- * large answer — the image comes back as base64 inside the response. */
342
+ * large answer: the image comes back as base64 inside the response. */
308
343
  fullPage?: boolean;
309
344
  };
310
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
311
- * agent. The connection's own headers cannot be set. */
345
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
346
+ * custom user agent. The connection's own headers cannot be set. */
312
347
  headers?: Record<string, string>;
313
348
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
314
349
  * free outcome. */
@@ -323,42 +358,48 @@ export interface CreateBatchParams {
323
358
  /** Where to POST when something happens. An https URL you control; private and loopback
324
359
  * hosts are refused, the same as any other URL we fetch. */
325
360
  url: string;
326
- /** What to call about. "batch.completed" fires once, when every URL has a verdict.
327
- * "batch.item.completed" fires per URL — one request from us per item, which is what makes
328
- * a long batch usable before it ends, so ask for it deliberately. */
361
+ /** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
362
+ * we could not finish the batch. `batch.status` says which: `completed` or `failed`.
363
+ * "batch.item.completed" fires per URL, with one request from us per item. That makes a
364
+ * long batch usable before it ends, so ask for it deliberately. */
329
365
  events?: ("batch.completed" | "batch.item.completed")[];
330
366
  };
331
367
  }
332
368
  /** Arguments for `batches.quote()`. */
333
369
  export interface QuoteBatchParams {
334
370
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
335
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
336
- * JavaScript shell, billing whichever engine answered. */
371
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
372
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
373
+ * always use "fetch", so leave it out for them. */
337
374
  engine?: "auto" | "fetch" | "browser";
338
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
339
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
340
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
375
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
376
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
377
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
378
+ * the fetch one. A POST is sent once, so it is off for a POST. */
341
379
  escalate?: boolean;
342
380
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
343
381
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
344
- * that is blocked again is free — but it takes as long as the first one did, so the call can
382
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
345
383
  * run that many times longer. */
346
384
  retry?: number;
347
385
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
348
386
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
349
- * and needs an `extract` block saying which ones. */
350
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
387
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
388
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
389
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
390
+ * cut short. */
391
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
351
392
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
352
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
353
- * then not your answer. It costs nothing, and we try again from another address instead of
354
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
393
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
394
+ * not your answer. It costs nothing, and we try again from another address instead of handing
395
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
355
396
  expect?: "json";
356
397
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
357
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
398
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
358
399
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
359
400
  * code. */
360
401
  extract?: {
361
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
402
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
362
403
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
363
404
  schema?: Record<string, unknown>;
364
405
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -369,7 +410,7 @@ export interface QuoteBatchParams {
369
410
  /** Where the request leaves from. Defaults to the managed residential pool. */
370
411
  proxy?: {
371
412
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
372
- * "custom" is your own proxy — free on every plan, bytes never billed. */
413
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
373
414
  pool?: "residential" | "custom";
374
415
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
375
416
  * for the managed one. */
@@ -389,7 +430,7 @@ export interface QuoteBatchParams {
389
430
  * charge; ask for the state or the country alone. */
390
431
  city?: string;
391
432
  };
392
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
433
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
393
434
  * turns out to need one. */
394
435
  browser?: {
395
436
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -397,9 +438,9 @@ export interface QuoteBatchParams {
397
438
  * page stops asking for things. Every page gets a short settle after this whichever you
398
439
  * pick. */
399
440
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
400
- /** Wait until this CSS selector is on the page — the dependable way to read something a
401
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
402
- * than the half-built page. */
441
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
442
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
443
+ * rather than the half-built page. */
403
444
  waitForSelector?: string;
404
445
  /** Wait this many milliseconds after the page settles, for the scripts no selector
405
446
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -411,11 +452,11 @@ export interface QuoteBatchParams {
411
452
  height: number;
412
453
  };
413
454
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
414
- * large answer — the image comes back as base64 inside the response. */
455
+ * large answer: the image comes back as base64 inside the response. */
415
456
  fullPage?: boolean;
416
457
  };
417
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
418
- * agent. The connection's own headers cannot be set. */
458
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
459
+ * custom user agent. The connection's own headers cannot be set. */
419
460
  headers?: Record<string, string>;
420
461
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
421
462
  * free outcome. */
@@ -430,9 +471,10 @@ export interface QuoteBatchParams {
430
471
  /** Where to POST when something happens. An https URL you control; private and loopback
431
472
  * hosts are refused, the same as any other URL we fetch. */
432
473
  url: string;
433
- /** What to call about. "batch.completed" fires once, when every URL has a verdict.
434
- * "batch.item.completed" fires per URL — one request from us per item, which is what makes
435
- * a long batch usable before it ends, so ask for it deliberately. */
474
+ /** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
475
+ * we could not finish the batch. `batch.status` says which: `completed` or `failed`.
476
+ * "batch.item.completed" fires per URL, with one request from us per item. That makes a
477
+ * long batch usable before it ends, so ask for it deliberately. */
436
478
  events?: ("batch.completed" | "batch.item.completed")[];
437
479
  };
438
480
  }
@@ -467,7 +509,7 @@ export interface CreateSessionParams {
467
509
  * pool. */
468
510
  proxy?: {
469
511
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
470
- * "custom" is your own proxy — free on every plan, bytes never billed. */
512
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
471
513
  pool?: "residential" | "custom";
472
514
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
473
515
  * for the managed one. */
@@ -498,7 +540,7 @@ export interface QuoteSessionParams {
498
540
  * pool. */
499
541
  proxy?: {
500
542
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
501
- * "custom" is your own proxy — free on every plan, bytes never billed. */
543
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
502
544
  pool?: "residential" | "custom";
503
545
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
504
546
  * for the managed one. */
@@ -1 +1 @@
1
- {"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,iFAAiF;IACjF,GAAG,EAAE,MAAM,CAAC;IACZ;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+BAA+B;AAC/B,MAAM,WAAW,iBAAiB;IAC9B,iFAAiF;IACjF,GAAG,EAAE,MAAM,CAAC;IACZ;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd;;oDAEgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;+CAC2C;IAC3C,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;6BAEyB;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;uFACmF;IACnF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;uDACmD;IACnD,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;8EAEsE;QACtE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC7B;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;8EAEsE;QACtE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC3B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX;kEAC8D;IAC9D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;uFACmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAChC;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IAC/B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC7B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd"}
1
+ {"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;8BAE0B;IAC1B,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB;;;;qCAIiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+BAA+B;AAC/B,MAAM,WAAW,iBAAiB;IAC9B,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;8BAE0B;IAC1B,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB;;;;qCAIiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,yEAAyE;IACzE,KAAK,EAAE,MAAM,CAAC;IACd;;oDAEgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;+CAC2C;IAC3C,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;6BAEyB;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;uFACmF;IACnF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;qDACiD;IACjD,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;4EAGoE;QACpE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC7B;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;4EAGoE;QACpE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC3B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX;kEAC8D;IAC9D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;uFACmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAChC;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IAC/B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC7B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd"}
@@ -36,9 +36,9 @@ export interface RetryOptions {
36
36
  /**
37
37
  * Should this failure be retried?
38
38
  *
39
- * if (attempt < 3 && shouldRetry(error, { repeatable: method === "GET" || !!key }))
39
+ * if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
40
40
  */
41
- export declare function shouldRetry(error: unknown, options: RetryOptions): boolean;
41
+ export declare function isRetryable(error: unknown, options: RetryOptions): boolean;
42
42
  /**
43
43
  * How long before retry number `attemptIndex` (0 for the first): the server's stated wait when
44
44
  * there is one, an exponential backoff otherwise, and never less than the backoff. A
@@ -9,9 +9,9 @@ const DEFAULT_MAX_WAIT_SECS = 10;
9
9
  /**
10
10
  * Should this failure be retried?
11
11
  *
12
- * if (attempt < 3 && shouldRetry(error, { repeatable: method === "GET" || !!key }))
12
+ * if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
13
13
  */
14
- export function shouldRetry(error, options) {
14
+ export function isRetryable(error, options) {
15
15
  const { repeatable, durableCodes = [], maxWaitSecs = DEFAULT_MAX_WAIT_SECS } = options;
16
16
  const status = statusOf(error);
17
17
  // No answer at all: nothing about it is a refusal, but a write may have landed before the
@@ -104,7 +104,7 @@ export function parseRetryAfter(value) {
104
104
  * frees when another job ends. Present-and-`null` is the whole test; an absent key said nothing.
105
105
  *
106
106
  * Not exported. It is not "is this refusal durable?": it knows nothing of `durableCodes` or of a
107
- * wait too long to hold, so a screen branching on it would disagree with `shouldRetry`.
107
+ * wait too long to hold, so a screen branching on it would disagree with `isRetryable`.
108
108
  */
109
109
  function waitingNeverHelps(error) {
110
110
  const details = field(error, "details");
@@ -23,7 +23,7 @@
23
23
  * - **The header is read first, in both its forms.** Two copies read the body's wait before the
24
24
  * `Retry-After` header, and read the header only as seconds, so an HTTP date was no wait at all.
25
25
  */
26
- import { parseRetryAfter, retryDelayMs, shouldRetry } from "./retry.js";
26
+ import { parseRetryAfter, retryDelayMs, isRetryable } from "./retry.js";
27
27
  const DEFAULT_TIMEOUT_MS = 30_000;
28
28
  const DEFAULT_MAX_RETRIES = 2;
29
29
  /**
@@ -75,7 +75,7 @@ export async function send(transport, spec, params = {}, opts = {}) {
75
75
  retryAfterSecs: failure.retryAfterSecs,
76
76
  };
77
77
  const again = attempt < maxRetries &&
78
- shouldRetry(answer, { repeatable, durableCodes: transport.durableCodes });
78
+ isRetryable(answer, { repeatable, durableCodes: transport.durableCodes });
79
79
  if (!again)
80
80
  throw transport.error(failure);
81
81
  await sleep(retryDelayMs(attempt, answer), signal);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@scrapa/sdk",
3
- "version": "0.2.1",
4
- "description": "Official TypeScript client for scrapa.app — scrape any web page as markdown, HTML, text, links, a screenshot or a PDF, search the web, and drive a real browser over CDP.",
3
+ "version": "0.3.0",
4
+ "description": "Official TypeScript client for scrapa.app. Scrape any web page as markdown, HTML, text, links, a screenshot or a PDF, search the web, and drive a real browser over CDP.",
5
5
  "license": "MIT",
6
6
  "homepage": "https://scrapa.app",
7
7
  "keywords": [