@scrapa/sdk 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,35 +1,50 @@
1
1
  /** Arguments for `scrape()`. */
2
2
  export interface ScrapeParams {
3
- /** The page to read. An http(s) URL — private and loopback hosts are refused. */
3
+ /** The page to read. An http(s) URL. Private and loopback hosts are refused. */
4
4
  url: string;
5
5
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
6
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
7
- * JavaScript shell, billing whichever engine answered. */
6
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
7
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
8
+ * always use "fetch", so leave it out for them. */
8
9
  engine?: "auto" | "fetch" | "browser";
9
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
10
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
11
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
10
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
11
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
12
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
13
+ * the fetch one. A POST is sent once, so it is off for a POST. */
12
14
  escalate?: boolean;
15
+ /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
16
+ * sent once: never retried and never sent again another way, because sending it twice could
17
+ * repeat what it did. */
18
+ method?: "GET" | "POST";
19
+ /** What a POST sends, at most 64 KB. Set its type in `headers`: `{"content-type":
20
+ * "application/json"}` for JSON, or `application/x-www-form-urlencoded` for a form. If the
21
+ * site redirects, a 301, 302 or 303 reads the next page with GET and without the body; a 307
22
+ * or 308 sends it again only if the protocol, host and port stay the same, and otherwise stops
23
+ * and names the new address. */
24
+ body?: string;
13
25
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
14
26
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
15
- * that is blocked again is free — but it takes as long as the first one did, so the call can
27
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
16
28
  * run that many times longer. */
17
29
  retry?: number;
18
30
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
19
31
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
20
- * and needs an `extract` block saying which ones. */
21
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
32
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
33
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
34
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
35
+ * cut short. */
36
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
22
37
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
23
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
24
- * then not your answer. It costs nothing, and we try again from another address instead of
25
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
38
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
39
+ * not your answer. It costs nothing, and we try again from another address instead of handing
40
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
26
41
  expect?: "json";
27
42
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
28
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
43
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
29
44
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
30
45
  * code. */
31
46
  extract?: {
32
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
47
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
33
48
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
34
49
  schema?: Record<string, unknown>;
35
50
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -40,7 +55,7 @@ export interface ScrapeParams {
40
55
  /** Where the request leaves from. Defaults to the managed residential pool. */
41
56
  proxy?: {
42
57
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
43
- * "custom" is your own proxy — free on every plan, bytes never billed. */
58
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
44
59
  pool?: "residential" | "custom";
45
60
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
46
61
  * for the managed one. */
@@ -60,7 +75,7 @@ export interface ScrapeParams {
60
75
  * charge; ask for the state or the country alone. */
61
76
  city?: string;
62
77
  };
63
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
78
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
64
79
  * turns out to need one. */
65
80
  browser?: {
66
81
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -68,9 +83,9 @@ export interface ScrapeParams {
68
83
  * page stops asking for things. Every page gets a short settle after this whichever you
69
84
  * pick. */
70
85
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
71
- /** Wait until this CSS selector is on the page — the dependable way to read something a
72
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
73
- * than the half-built page. */
86
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
87
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
88
+ * rather than the half-built page. */
74
89
  waitForSelector?: string;
75
90
  /** Wait this many milliseconds after the page settles, for the scripts no selector
76
91
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -82,11 +97,11 @@ export interface ScrapeParams {
82
97
  height: number;
83
98
  };
84
99
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
85
- * large answer — the image comes back as base64 inside the response. */
100
+ * large answer: the image comes back as base64 inside the response. */
86
101
  fullPage?: boolean;
87
102
  };
88
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
89
- * agent. The connection's own headers cannot be set. */
103
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
104
+ * custom user agent. The connection's own headers cannot be set. */
90
105
  headers?: Record<string, string>;
91
106
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
92
107
  * free outcome. */
@@ -94,36 +109,51 @@ export interface ScrapeParams {
94
109
  }
95
110
  /** Arguments for `quote()`. */
96
111
  export interface QuoteScrapeParams {
97
- /** The page to read. An http(s) URL — private and loopback hosts are refused. */
112
+ /** The page to read. An http(s) URL. Private and loopback hosts are refused. */
98
113
  url: string;
99
114
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
100
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
101
- * JavaScript shell, billing whichever engine answered. */
115
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
116
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
117
+ * always use "fetch", so leave it out for them. */
102
118
  engine?: "auto" | "fetch" | "browser";
103
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
104
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
105
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
119
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
120
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
121
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
122
+ * the fetch one. A POST is sent once, so it is off for a POST. */
106
123
  escalate?: boolean;
124
+ /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
125
+ * sent once: never retried and never sent again another way, because sending it twice could
126
+ * repeat what it did. */
127
+ method?: "GET" | "POST";
128
+ /** What a POST sends, at most 64 KB. Set its type in `headers`: `{"content-type":
129
+ * "application/json"}` for JSON, or `application/x-www-form-urlencoded` for a form. If the
130
+ * site redirects, a 301, 302 or 303 reads the next page with GET and without the body; a 307
131
+ * or 308 sends it again only if the protocol, host and port stay the same, and otherwise stops
132
+ * and names the new address. */
133
+ body?: string;
107
134
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
108
135
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
109
- * that is blocked again is free — but it takes as long as the first one did, so the call can
136
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
110
137
  * run that many times longer. */
111
138
  retry?: number;
112
139
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
113
140
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
114
- * and needs an `extract` block saying which ones. */
115
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
141
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
142
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
143
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
144
+ * cut short. */
145
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
116
146
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
117
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
118
- * then not your answer. It costs nothing, and we try again from another address instead of
119
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
147
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
148
+ * not your answer. It costs nothing, and we try again from another address instead of handing
149
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
120
150
  expect?: "json";
121
151
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
122
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
152
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
123
153
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
124
154
  * code. */
125
155
  extract?: {
126
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
156
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
127
157
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
128
158
  schema?: Record<string, unknown>;
129
159
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -134,7 +164,7 @@ export interface QuoteScrapeParams {
134
164
  /** Where the request leaves from. Defaults to the managed residential pool. */
135
165
  proxy?: {
136
166
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
137
- * "custom" is your own proxy — free on every plan, bytes never billed. */
167
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
138
168
  pool?: "residential" | "custom";
139
169
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
140
170
  * for the managed one. */
@@ -154,7 +184,7 @@ export interface QuoteScrapeParams {
154
184
  * charge; ask for the state or the country alone. */
155
185
  city?: string;
156
186
  };
157
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
187
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
158
188
  * turns out to need one. */
159
189
  browser?: {
160
190
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -162,9 +192,9 @@ export interface QuoteScrapeParams {
162
192
  * page stops asking for things. Every page gets a short settle after this whichever you
163
193
  * pick. */
164
194
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
165
- /** Wait until this CSS selector is on the page — the dependable way to read something a
166
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
167
- * than the half-built page. */
195
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
196
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
197
+ * rather than the half-built page. */
168
198
  waitForSelector?: string;
169
199
  /** Wait this many milliseconds after the page settles, for the scripts no selector
170
200
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -176,11 +206,11 @@ export interface QuoteScrapeParams {
176
206
  height: number;
177
207
  };
178
208
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
179
- * large answer — the image comes back as base64 inside the response. */
209
+ * large answer: the image comes back as base64 inside the response. */
180
210
  fullPage?: boolean;
181
211
  };
182
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
183
- * agent. The connection's own headers cannot be set. */
212
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
213
+ * custom user agent. The connection's own headers cannot be set. */
184
214
  headers?: Record<string, string>;
185
215
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
186
216
  * free outcome. */
@@ -188,10 +218,10 @@ export interface QuoteScrapeParams {
188
218
  }
189
219
  /** Arguments for `search()`. */
190
220
  export interface SearchParams {
191
- /** What to search for, in plain words — the same thing you would type. */
221
+ /** What to search for, in plain words: the same thing you would type. */
192
222
  query: string;
193
223
  /** How many results to hand back at most, 1–20. Billed per 10, so asking for more than 10 costs
194
- * a second row. Fewer can come back than you asked for — an index has what it has, and
224
+ * a second row. Fewer can come back than you asked for: an index has what it has, and
195
225
  * `includeDomains` filters what it returns. */
196
226
  limit?: number;
197
227
  /** What kind of results. "general" is the web; "news" is dated journalism, and the only topic
@@ -199,18 +229,18 @@ export interface SearchParams {
199
229
  topic?: "general" | "news";
200
230
  /** Only pages from the last day, week, month or year. Absent is any time. */
201
231
  timeRange?: "day" | "week" | "month" | "year";
202
- /** Only results on these hosts, bare (`linkedin.com`) — subdomains included. Asked of the
232
+ /** Only results on these hosts, bare (`linkedin.com`). Subdomains are included. Asked of the
203
233
  * search front AND enforced on the answer, so what comes back is on your list whatever the
204
234
  * front did with it. */
205
235
  includeDomains?: string[];
206
236
  /** Also write a short paragraph answering the query from the results, in the query's language.
207
237
  * Costs one extra credit; leave it off when your own model reads the snippets. */
208
238
  answer?: boolean;
209
- /** Where the search leaves from. `country` picks the exit AND the market the front answers for
210
- * — `br` gets Brazilian results in Portuguese. */
239
+ /** Where the search leaves from. `country` picks the exit AND the market the front answers for.
240
+ * `br` gets Brazilian results in Portuguese. */
211
241
  proxy?: {
212
242
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
213
- * "custom" is your own proxy — free on every plan, bytes never billed. */
243
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
214
244
  pool?: "residential" | "custom";
215
245
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
216
246
  * for the managed one. */
@@ -225,33 +255,38 @@ export interface SearchParams {
225
255
  /** Arguments for `batches.create()`. */
226
256
  export interface CreateBatchParams {
227
257
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
228
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
229
- * JavaScript shell, billing whichever engine answered. */
258
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
259
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
260
+ * always use "fetch", so leave it out for them. */
230
261
  engine?: "auto" | "fetch" | "browser";
231
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
232
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
233
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
262
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
263
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
264
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
265
+ * the fetch one. A POST is sent once, so it is off for a POST. */
234
266
  escalate?: boolean;
235
267
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
236
268
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
237
- * that is blocked again is free — but it takes as long as the first one did, so the call can
269
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
238
270
  * run that many times longer. */
239
271
  retry?: number;
240
272
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
241
273
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
242
- * and needs an `extract` block saying which ones. */
243
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
274
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
275
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
276
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
277
+ * cut short. */
278
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
244
279
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
245
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
246
- * then not your answer. It costs nothing, and we try again from another address instead of
247
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
280
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
281
+ * not your answer. It costs nothing, and we try again from another address instead of handing
282
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
248
283
  expect?: "json";
249
284
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
250
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
285
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
251
286
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
252
287
  * code. */
253
288
  extract?: {
254
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
289
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
255
290
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
256
291
  schema?: Record<string, unknown>;
257
292
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -262,7 +297,7 @@ export interface CreateBatchParams {
262
297
  /** Where the request leaves from. Defaults to the managed residential pool. */
263
298
  proxy?: {
264
299
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
265
- * "custom" is your own proxy — free on every plan, bytes never billed. */
300
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
266
301
  pool?: "residential" | "custom";
267
302
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
268
303
  * for the managed one. */
@@ -282,7 +317,7 @@ export interface CreateBatchParams {
282
317
  * charge; ask for the state or the country alone. */
283
318
  city?: string;
284
319
  };
285
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
320
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
286
321
  * turns out to need one. */
287
322
  browser?: {
288
323
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -290,9 +325,9 @@ export interface CreateBatchParams {
290
325
  * page stops asking for things. Every page gets a short settle after this whichever you
291
326
  * pick. */
292
327
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
293
- /** Wait until this CSS selector is on the page — the dependable way to read something a
294
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
295
- * than the half-built page. */
328
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
329
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
330
+ * rather than the half-built page. */
296
331
  waitForSelector?: string;
297
332
  /** Wait this many milliseconds after the page settles, for the scripts no selector
298
333
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -304,11 +339,11 @@ export interface CreateBatchParams {
304
339
  height: number;
305
340
  };
306
341
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
307
- * large answer — the image comes back as base64 inside the response. */
342
+ * large answer: the image comes back as base64 inside the response. */
308
343
  fullPage?: boolean;
309
344
  };
310
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
311
- * agent. The connection's own headers cannot be set. */
345
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
346
+ * custom user agent. The connection's own headers cannot be set. */
312
347
  headers?: Record<string, string>;
313
348
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
314
349
  * free outcome. */
@@ -325,41 +360,46 @@ export interface CreateBatchParams {
325
360
  url: string;
326
361
  /** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
327
362
  * we could not finish the batch. `batch.status` says which: `completed` or `failed`.
328
- * "batch.item.completed" fires per URL — one request from us per item, which is what makes
329
- * a long batch usable before it ends, so ask for it deliberately. */
363
+ * "batch.item.completed" fires per URL, with one request from us per item. That makes a
364
+ * long batch usable before it ends, so ask for it deliberately. */
330
365
  events?: ("batch.completed" | "batch.item.completed")[];
331
366
  };
332
367
  }
333
368
  /** Arguments for `batches.quote()`. */
334
369
  export interface QuoteBatchParams {
335
370
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
336
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
337
- * JavaScript shell, billing whichever engine answered. */
371
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
372
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
373
+ * always use "fetch", so leave it out for them. */
338
374
  engine?: "auto" | "fetch" | "browser";
339
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
340
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
341
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
375
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
376
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
377
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
378
+ * the fetch one. A POST is sent once, so it is off for a POST. */
342
379
  escalate?: boolean;
343
380
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
344
381
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
345
- * that is blocked again is free — but it takes as long as the first one did, so the call can
382
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
346
383
  * run that many times longer. */
347
384
  retry?: number;
348
385
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
349
386
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
350
- * and needs an `extract` block saying which ones. */
351
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
387
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
388
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
389
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
390
+ * cut short. */
391
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
352
392
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
353
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
354
- * then not your answer. It costs nothing, and we try again from another address instead of
355
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
393
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
394
+ * not your answer. It costs nothing, and we try again from another address instead of handing
395
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
356
396
  expect?: "json";
357
397
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
358
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
398
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
359
399
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
360
400
  * code. */
361
401
  extract?: {
362
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
402
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
363
403
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
364
404
  schema?: Record<string, unknown>;
365
405
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -370,7 +410,7 @@ export interface QuoteBatchParams {
370
410
  /** Where the request leaves from. Defaults to the managed residential pool. */
371
411
  proxy?: {
372
412
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
373
- * "custom" is your own proxy — free on every plan, bytes never billed. */
413
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
374
414
  pool?: "residential" | "custom";
375
415
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
376
416
  * for the managed one. */
@@ -390,7 +430,7 @@ export interface QuoteBatchParams {
390
430
  * charge; ask for the state or the country alone. */
391
431
  city?: string;
392
432
  };
393
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
433
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
394
434
  * turns out to need one. */
395
435
  browser?: {
396
436
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -398,9 +438,9 @@ export interface QuoteBatchParams {
398
438
  * page stops asking for things. Every page gets a short settle after this whichever you
399
439
  * pick. */
400
440
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
401
- /** Wait until this CSS selector is on the page — the dependable way to read something a
402
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
403
- * than the half-built page. */
441
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
442
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
443
+ * rather than the half-built page. */
404
444
  waitForSelector?: string;
405
445
  /** Wait this many milliseconds after the page settles, for the scripts no selector
406
446
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -412,11 +452,11 @@ export interface QuoteBatchParams {
412
452
  height: number;
413
453
  };
414
454
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
415
- * large answer — the image comes back as base64 inside the response. */
455
+ * large answer: the image comes back as base64 inside the response. */
416
456
  fullPage?: boolean;
417
457
  };
418
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
419
- * agent. The connection's own headers cannot be set. */
458
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
459
+ * custom user agent. The connection's own headers cannot be set. */
420
460
  headers?: Record<string, string>;
421
461
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
422
462
  * free outcome. */
@@ -433,8 +473,8 @@ export interface QuoteBatchParams {
433
473
  url: string;
434
474
  /** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
435
475
  * we could not finish the batch. `batch.status` says which: `completed` or `failed`.
436
- * "batch.item.completed" fires per URL — one request from us per item, which is what makes
437
- * a long batch usable before it ends, so ask for it deliberately. */
476
+ * "batch.item.completed" fires per URL, with one request from us per item. That makes a
477
+ * long batch usable before it ends, so ask for it deliberately. */
438
478
  events?: ("batch.completed" | "batch.item.completed")[];
439
479
  };
440
480
  }
@@ -469,7 +509,7 @@ export interface CreateSessionParams {
469
509
  * pool. */
470
510
  proxy?: {
471
511
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
472
- * "custom" is your own proxy — free on every plan, bytes never billed. */
512
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
473
513
  pool?: "residential" | "custom";
474
514
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
475
515
  * for the managed one. */
@@ -500,7 +540,7 @@ export interface QuoteSessionParams {
500
540
  * pool. */
501
541
  proxy?: {
502
542
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
503
- * "custom" is your own proxy — free on every plan, bytes never billed. */
543
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
504
544
  pool?: "residential" | "custom";
505
545
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
506
546
  * for the managed one. */
@@ -1 +1 @@
1
- {"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,iFAAiF;IACjF,GAAG,EAAE,MAAM,CAAC;IACZ;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+BAA+B;AAC/B,MAAM,WAAW,iBAAiB;IAC9B,iFAAiF;IACjF,GAAG,EAAE,MAAM,CAAC;IACZ;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd;;oDAEgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;+CAC2C;IAC3C,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;6BAEyB;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;uFACmF;IACnF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;uDACmD;IACnD,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;8EAGsE;QACtE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC7B;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;8EAGsE;QACtE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC3B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX;kEAC8D;IAC9D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;uFACmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAChC;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IAC/B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC7B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd"}
1
+ {"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;8BAE0B;IAC1B,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB;;;;qCAIiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+BAA+B;AAC/B,MAAM,WAAW,iBAAiB;IAC9B,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;8BAE0B;IAC1B,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB;;;;qCAIiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,yEAAyE;IACzE,KAAK,EAAE,MAAM,CAAC;IACd;;oDAEgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;+CAC2C;IAC3C,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;6BAEyB;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;uFACmF;IACnF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;qDACiD;IACjD,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;4EAGoE;QACpE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC7B;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;4EAGoE;QACpE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC3B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX;kEAC8D;IAC9D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;uFACmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAChC;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IAC/B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC7B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd"}
@@ -36,9 +36,9 @@ export interface RetryOptions {
36
36
  /**
37
37
  * Should this failure be retried?
38
38
  *
39
- * if (attempt < 3 && shouldRetry(error, { repeatable: method === "GET" || !!key }))
39
+ * if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
40
40
  */
41
- export declare function shouldRetry(error: unknown, options: RetryOptions): boolean;
41
+ export declare function isRetryable(error: unknown, options: RetryOptions): boolean;
42
42
  /**
43
43
  * How long before retry number `attemptIndex` (0 for the first): the server's stated wait when
44
44
  * there is one, an exponential backoff otherwise, and never less than the backoff. A
@@ -9,9 +9,9 @@ const DEFAULT_MAX_WAIT_SECS = 10;
9
9
  /**
10
10
  * Should this failure be retried?
11
11
  *
12
- * if (attempt < 3 && shouldRetry(error, { repeatable: method === "GET" || !!key }))
12
+ * if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
13
13
  */
14
- export function shouldRetry(error, options) {
14
+ export function isRetryable(error, options) {
15
15
  const { repeatable, durableCodes = [], maxWaitSecs = DEFAULT_MAX_WAIT_SECS } = options;
16
16
  const status = statusOf(error);
17
17
  // No answer at all: nothing about it is a refusal, but a write may have landed before the
@@ -104,7 +104,7 @@ export function parseRetryAfter(value) {
104
104
  * frees when another job ends. Present-and-`null` is the whole test; an absent key said nothing.
105
105
  *
106
106
  * Not exported. It is not "is this refusal durable?": it knows nothing of `durableCodes` or of a
107
- * wait too long to hold, so a screen branching on it would disagree with `shouldRetry`.
107
+ * wait too long to hold, so a screen branching on it would disagree with `isRetryable`.
108
108
  */
109
109
  function waitingNeverHelps(error) {
110
110
  const details = field(error, "details");
@@ -23,7 +23,7 @@
23
23
  * - **The header is read first, in both its forms.** Two copies read the body's wait before the
24
24
  * `Retry-After` header, and read the header only as seconds, so an HTTP date was no wait at all.
25
25
  */
26
- import { parseRetryAfter, retryDelayMs, shouldRetry } from "./retry.js";
26
+ import { parseRetryAfter, retryDelayMs, isRetryable } from "./retry.js";
27
27
  const DEFAULT_TIMEOUT_MS = 30_000;
28
28
  const DEFAULT_MAX_RETRIES = 2;
29
29
  /**
@@ -75,7 +75,7 @@ export async function send(transport, spec, params = {}, opts = {}) {
75
75
  retryAfterSecs: failure.retryAfterSecs,
76
76
  };
77
77
  const again = attempt < maxRetries &&
78
- shouldRetry(answer, { repeatable, durableCodes: transport.durableCodes });
78
+ isRetryable(answer, { repeatable, durableCodes: transport.durableCodes });
79
79
  if (!again)
80
80
  throw transport.error(failure);
81
81
  await sleep(retryDelayMs(attempt, answer), signal);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@scrapa/sdk",
3
- "version": "0.2.2",
4
- "description": "Official TypeScript client for scrapa.app — scrape any web page as markdown, HTML, text, links, a screenshot or a PDF, search the web, and drive a real browser over CDP.",
3
+ "version": "0.3.0",
4
+ "description": "Official TypeScript client for scrapa.app. Scrape any web page as markdown, HTML, text, links, a screenshot or a PDF, search the web, and drive a real browser over CDP.",
5
5
  "license": "MIT",
6
6
  "homepage": "https://scrapa.app",
7
7
  "keywords": [
package/src/client.ts CHANGED
@@ -116,11 +116,11 @@ export class Scrapa extends GeneratedOperations {
116
116
  const opts = typeof options === "string" ? { apiKey: options } : options;
117
117
  if (!opts.apiKey)
118
118
  throw new TypeError(
119
- "scrapa: an API key is required — create one at https://console.scrapa.app/keys",
119
+ "scrapa: an API key is required. Create one at https://console.scrapa.app/keys",
120
120
  );
121
121
  if (typeof (opts.fetch ?? globalThis.fetch) !== "function")
122
122
  throw new TypeError(
123
- "scrapa: no global fetch — use Node 20+, or pass `fetch` in the options.",
123
+ "scrapa: no global fetch. Use Node 20+, or pass `fetch` in the options.",
124
124
  );
125
125
  const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
126
126
  const host = new URL(baseUrl).host;