@scrapa/sdk 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,36 +6,51 @@
6
6
 
7
7
  /** Arguments for `scrape()`. */
8
8
  export interface ScrapeParams {
9
- /** The page to read. An http(s) URL — private and loopback hosts are refused. */
9
+ /** The page to read. An http(s) URL. Private and loopback hosts are refused. */
10
10
  url: string;
11
11
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
12
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
13
- * JavaScript shell, billing whichever engine answered. */
12
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
13
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
14
+ * always use "fetch", so leave it out for them. */
14
15
  engine?: "auto" | "fetch" | "browser";
15
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
16
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
17
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
16
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
17
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
18
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
19
+ * the fetch one. A POST is sent once, so it is off for a POST. */
18
20
  escalate?: boolean;
21
+ /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
22
+ * sent once: never retried and never sent again another way, because sending it twice could
23
+ * repeat what it did. */
24
+ method?: "GET" | "POST";
25
+ /** What a POST sends, at most 64 KB. Set its type in `headers`: `{"content-type":
26
+ * "application/json"}` for JSON, or `application/x-www-form-urlencoded` for a form. If the
27
+ * site redirects, a 301, 302 or 303 reads the next page with GET and without the body; a 307
28
+ * or 308 sends it again only if the protocol, host and port stay the same, and otherwise stops
29
+ * and names the new address. */
30
+ body?: string;
19
31
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
20
32
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
21
- * that is blocked again is free — but it takes as long as the first one did, so the call can
33
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
22
34
  * run that many times longer. */
23
35
  retry?: number;
24
36
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
25
37
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
26
- * and needs an `extract` block saying which ones. */
27
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
38
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
39
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
40
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
41
+ * cut short. */
42
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
28
43
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
29
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
30
- * then not your answer. It costs nothing, and we try again from another address instead of
31
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
44
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
45
+ * not your answer. It costs nothing, and we try again from another address instead of handing
46
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
32
47
  expect?: "json";
33
48
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
34
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
49
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
35
50
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
36
51
  * code. */
37
52
  extract?: {
38
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
53
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
39
54
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
40
55
  schema?: Record<string, unknown>;
41
56
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -46,7 +61,7 @@ export interface ScrapeParams {
46
61
  /** Where the request leaves from. Defaults to the managed residential pool. */
47
62
  proxy?: {
48
63
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
49
- * "custom" is your own proxy — free on every plan, bytes never billed. */
64
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
50
65
  pool?: "residential" | "custom";
51
66
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
52
67
  * for the managed one. */
@@ -66,7 +81,7 @@ export interface ScrapeParams {
66
81
  * charge; ask for the state or the country alone. */
67
82
  city?: string;
68
83
  };
69
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
84
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
70
85
  * turns out to need one. */
71
86
  browser?: {
72
87
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -74,9 +89,9 @@ export interface ScrapeParams {
74
89
  * page stops asking for things. Every page gets a short settle after this whichever you
75
90
  * pick. */
76
91
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
77
- /** Wait until this CSS selector is on the page — the dependable way to read something a
78
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
79
- * than the half-built page. */
92
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
93
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
94
+ * rather than the half-built page. */
80
95
  waitForSelector?: string;
81
96
  /** Wait this many milliseconds after the page settles, for the scripts no selector
82
97
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -88,11 +103,11 @@ export interface ScrapeParams {
88
103
  height: number;
89
104
  };
90
105
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
91
- * large answer — the image comes back as base64 inside the response. */
106
+ * large answer: the image comes back as base64 inside the response. */
92
107
  fullPage?: boolean;
93
108
  };
94
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
95
- * agent. The connection's own headers cannot be set. */
109
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
110
+ * custom user agent. The connection's own headers cannot be set. */
96
111
  headers?: Record<string, string>;
97
112
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
98
113
  * free outcome. */
@@ -101,36 +116,51 @@ export interface ScrapeParams {
101
116
 
102
117
  /** Arguments for `quote()`. */
103
118
  export interface QuoteScrapeParams {
104
- /** The page to read. An http(s) URL — private and loopback hosts are refused. */
119
+ /** The page to read. An http(s) URL. Private and loopback hosts are refused. */
105
120
  url: string;
106
121
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
107
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
108
- * JavaScript shell, billing whichever engine answered. */
122
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
123
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
124
+ * always use "fetch", so leave it out for them. */
109
125
  engine?: "auto" | "fetch" | "browser";
110
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
111
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
112
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
126
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
127
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
128
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
129
+ * the fetch one. A POST is sent once, so it is off for a POST. */
113
130
  escalate?: boolean;
131
+ /** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
132
+ * sent once: never retried and never sent again another way, because sending it twice could
133
+ * repeat what it did. */
134
+ method?: "GET" | "POST";
135
+ /** What a POST sends, at most 64 KB. Set its type in `headers`: `{"content-type":
136
+ * "application/json"}` for JSON, or `application/x-www-form-urlencoded` for a form. If the
137
+ * site redirects, a 301, 302 or 303 reads the next page with GET and without the body; a 307
138
+ * or 308 sends it again only if the protocol, host and port stay the same, and otherwise stops
139
+ * and names the new address. */
140
+ body?: string;
114
141
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
115
142
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
116
- * that is blocked again is free — but it takes as long as the first one did, so the call can
143
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
117
144
  * run that many times longer. */
118
145
  retry?: number;
119
146
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
120
147
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
121
- * and needs an `extract` block saying which ones. */
122
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
148
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
149
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
150
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
151
+ * cut short. */
152
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
123
153
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
124
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
125
- * then not your answer. It costs nothing, and we try again from another address instead of
126
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
154
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
155
+ * not your answer. It costs nothing, and we try again from another address instead of handing
156
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
127
157
  expect?: "json";
128
158
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
129
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
159
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
130
160
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
131
161
  * code. */
132
162
  extract?: {
133
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
163
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
134
164
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
135
165
  schema?: Record<string, unknown>;
136
166
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -141,7 +171,7 @@ export interface QuoteScrapeParams {
141
171
  /** Where the request leaves from. Defaults to the managed residential pool. */
142
172
  proxy?: {
143
173
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
144
- * "custom" is your own proxy — free on every plan, bytes never billed. */
174
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
145
175
  pool?: "residential" | "custom";
146
176
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
147
177
  * for the managed one. */
@@ -161,7 +191,7 @@ export interface QuoteScrapeParams {
161
191
  * charge; ask for the state or the country alone. */
162
192
  city?: string;
163
193
  };
164
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
194
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
165
195
  * turns out to need one. */
166
196
  browser?: {
167
197
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -169,9 +199,9 @@ export interface QuoteScrapeParams {
169
199
  * page stops asking for things. Every page gets a short settle after this whichever you
170
200
  * pick. */
171
201
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
172
- /** Wait until this CSS selector is on the page — the dependable way to read something a
173
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
174
- * than the half-built page. */
202
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
203
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
204
+ * rather than the half-built page. */
175
205
  waitForSelector?: string;
176
206
  /** Wait this many milliseconds after the page settles, for the scripts no selector
177
207
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -183,11 +213,11 @@ export interface QuoteScrapeParams {
183
213
  height: number;
184
214
  };
185
215
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
186
- * large answer — the image comes back as base64 inside the response. */
216
+ * large answer: the image comes back as base64 inside the response. */
187
217
  fullPage?: boolean;
188
218
  };
189
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
190
- * agent. The connection's own headers cannot be set. */
219
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
220
+ * custom user agent. The connection's own headers cannot be set. */
191
221
  headers?: Record<string, string>;
192
222
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
193
223
  * free outcome. */
@@ -196,10 +226,10 @@ export interface QuoteScrapeParams {
196
226
 
197
227
  /** Arguments for `search()`. */
198
228
  export interface SearchParams {
199
- /** What to search for, in plain words — the same thing you would type. */
229
+ /** What to search for, in plain words: the same thing you would type. */
200
230
  query: string;
201
231
  /** How many results to hand back at most, 1–20. Billed per 10, so asking for more than 10 costs
202
- * a second row. Fewer can come back than you asked for — an index has what it has, and
232
+ * a second row. Fewer can come back than you asked for: an index has what it has, and
203
233
  * `includeDomains` filters what it returns. */
204
234
  limit?: number;
205
235
  /** What kind of results. "general" is the web; "news" is dated journalism, and the only topic
@@ -207,18 +237,18 @@ export interface SearchParams {
207
237
  topic?: "general" | "news";
208
238
  /** Only pages from the last day, week, month or year. Absent is any time. */
209
239
  timeRange?: "day" | "week" | "month" | "year";
210
- /** Only results on these hosts, bare (`linkedin.com`) — subdomains included. Asked of the
240
+ /** Only results on these hosts, bare (`linkedin.com`). Subdomains are included. Asked of the
211
241
  * search front AND enforced on the answer, so what comes back is on your list whatever the
212
242
  * front did with it. */
213
243
  includeDomains?: string[];
214
244
  /** Also write a short paragraph answering the query from the results, in the query's language.
215
245
  * Costs one extra credit; leave it off when your own model reads the snippets. */
216
246
  answer?: boolean;
217
- /** Where the search leaves from. `country` picks the exit AND the market the front answers for
218
- * — `br` gets Brazilian results in Portuguese. */
247
+ /** Where the search leaves from. `country` picks the exit AND the market the front answers for.
248
+ * `br` gets Brazilian results in Portuguese. */
219
249
  proxy?: {
220
250
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
221
- * "custom" is your own proxy — free on every plan, bytes never billed. */
251
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
222
252
  pool?: "residential" | "custom";
223
253
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
224
254
  * for the managed one. */
@@ -234,33 +264,38 @@ export interface SearchParams {
234
264
  /** Arguments for `batches.create()`. */
235
265
  export interface CreateBatchParams {
236
266
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
237
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
238
- * JavaScript shell, billing whichever engine answered. */
267
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
268
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
269
+ * always use "fetch", so leave it out for them. */
239
270
  engine?: "auto" | "fetch" | "browser";
240
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
241
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
242
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
271
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
272
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
273
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
274
+ * the fetch one. A POST is sent once, so it is off for a POST. */
243
275
  escalate?: boolean;
244
276
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
245
277
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
246
- * that is blocked again is free — but it takes as long as the first one did, so the call can
278
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
247
279
  * run that many times longer. */
248
280
  retry?: number;
249
281
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
250
282
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
251
- * and needs an `extract` block saying which ones. */
252
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
283
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
284
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
285
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
286
+ * cut short. */
287
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
253
288
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
254
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
255
- * then not your answer. It costs nothing, and we try again from another address instead of
256
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
289
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
290
+ * not your answer. It costs nothing, and we try again from another address instead of handing
291
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
257
292
  expect?: "json";
258
293
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
259
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
294
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
260
295
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
261
296
  * code. */
262
297
  extract?: {
263
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
298
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
264
299
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
265
300
  schema?: Record<string, unknown>;
266
301
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -271,7 +306,7 @@ export interface CreateBatchParams {
271
306
  /** Where the request leaves from. Defaults to the managed residential pool. */
272
307
  proxy?: {
273
308
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
274
- * "custom" is your own proxy — free on every plan, bytes never billed. */
309
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
275
310
  pool?: "residential" | "custom";
276
311
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
277
312
  * for the managed one. */
@@ -291,7 +326,7 @@ export interface CreateBatchParams {
291
326
  * charge; ask for the state or the country alone. */
292
327
  city?: string;
293
328
  };
294
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
329
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
295
330
  * turns out to need one. */
296
331
  browser?: {
297
332
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -299,9 +334,9 @@ export interface CreateBatchParams {
299
334
  * page stops asking for things. Every page gets a short settle after this whichever you
300
335
  * pick. */
301
336
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
302
- /** Wait until this CSS selector is on the page — the dependable way to read something a
303
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
304
- * than the half-built page. */
337
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
338
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
339
+ * rather than the half-built page. */
305
340
  waitForSelector?: string;
306
341
  /** Wait this many milliseconds after the page settles, for the scripts no selector
307
342
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -313,11 +348,11 @@ export interface CreateBatchParams {
313
348
  height: number;
314
349
  };
315
350
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
316
- * large answer — the image comes back as base64 inside the response. */
351
+ * large answer: the image comes back as base64 inside the response. */
317
352
  fullPage?: boolean;
318
353
  };
319
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
320
- * agent. The connection's own headers cannot be set. */
354
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
355
+ * custom user agent. The connection's own headers cannot be set. */
321
356
  headers?: Record<string, string>;
322
357
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
323
358
  * free outcome. */
@@ -334,8 +369,8 @@ export interface CreateBatchParams {
334
369
  url: string;
335
370
  /** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
336
371
  * we could not finish the batch. `batch.status` says which: `completed` or `failed`.
337
- * "batch.item.completed" fires per URL — one request from us per item, which is what makes
338
- * a long batch usable before it ends, so ask for it deliberately. */
372
+ * "batch.item.completed" fires per URL, with one request from us per item. That makes a
373
+ * long batch usable before it ends, so ask for it deliberately. */
339
374
  events?: ("batch.completed" | "batch.item.completed")[];
340
375
  };
341
376
  }
@@ -343,33 +378,38 @@ export interface CreateBatchParams {
343
378
  /** Arguments for `batches.quote()`. */
344
379
  export interface QuoteBatchParams {
345
380
  /** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
346
- * Chromium; "auto" fetches first and escalates to the browser when the body turns out to be a
347
- * JavaScript shell, billing whichever engine answered. */
381
+ * Chromium; "auto" (the default) fetches first and escalates to the browser when the body
382
+ * turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
383
+ * always use "fetch", so leave it out for them. */
348
384
  engine?: "auto" | "fetch" | "browser";
349
- /** Whether to try again a different way when the first attempt is blocked. `false` stops after
350
- * one try — the site's refusal is your answer, and it costs nothing. It is how you bound a
351
- * call: with `auto`, trying again is what charges the browser price instead of the fetch one. */
385
+ /** Whether to try again a different way when the first attempt is blocked. On by default.
386
+ * `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
387
+ * how you bound a call: with `auto`, trying again is what charges the browser price instead of
388
+ * the fetch one. A POST is sent once, so it is off for a POST. */
352
389
  escalate?: boolean;
353
390
  /** How many extra tries to give a page the site blocks, each from a different address. 0 (the
354
391
  * default) takes the site's first no as the answer. A blocked page costs nothing, so a try
355
- * that is blocked again is free — but it takes as long as the first one did, so the call can
392
+ * that is blocked again is free, but it takes as long as the first one did, so the call can
356
393
  * run that many times longer. */
357
394
  retry?: number;
358
395
  /** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
359
396
  * asking for both costs no more than one. "extract" is the page read into fields by a model,
360
- * and needs an `extract` block saying which ones. */
361
- formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract")[];
397
+ * and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
398
+ * it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
399
+ * fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
400
+ * cut short. */
401
+ formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
362
402
  /** What the page should answer with. Set `"json"` when you are reading an API: anything else
363
- * that comes back — a maintenance notice, a "checking your browser" page, a login form — is
364
- * then not your answer. It costs nothing, and we try again from another address instead of
365
- * handing you a page you cannot parse. Without it, whatever the page returns is your answer. */
403
+ * that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
404
+ * not your answer. It costs nothing, and we try again from another address instead of handing
405
+ * you a page you cannot parse. Without it, whatever the page returns is your answer. */
366
406
  expect?: "json";
367
407
  /** What to pull out of the page, for the "extract" format. A model reads the page and fills in
368
- * the shape you asked for — give it a JSON Schema, a sentence, or both. It reads what the page
408
+ * the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
369
409
  * says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
370
410
  * code. */
371
411
  extract?: {
372
- /** The shape you want back, as JSON Schema — an object whose properties are the fields you
412
+ /** The shape you want back, as JSON Schema: an object whose properties are the fields you
373
413
  * need. A field the page does not mention comes back null; nothing is invented to fill it. */
374
414
  schema?: Record<string, unknown>;
375
415
  /** What to pull out, in your own words: `the price, the author and the date it was
@@ -380,7 +420,7 @@ export interface QuoteBatchParams {
380
420
  /** Where the request leaves from. Defaults to the managed residential pool. */
381
421
  proxy?: {
382
422
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
383
- * "custom" is your own proxy — free on every plan, bytes never billed. */
423
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
384
424
  pool?: "residential" | "custom";
385
425
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
386
426
  * for the managed one. */
@@ -400,7 +440,7 @@ export interface QuoteBatchParams {
400
440
  * charge; ask for the state or the country alone. */
401
441
  city?: string;
402
442
  };
403
- /** How to render it. Only the browser engine reads these — with `auto` they apply if the page
443
+ /** How to render it. Only the browser engine reads these. With `auto` they apply if the page
404
444
  * turns out to need one. */
405
445
  browser?: {
406
446
  /** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
@@ -408,9 +448,9 @@ export interface QuoteBatchParams {
408
448
  * page stops asking for things. Every page gets a short settle after this whichever you
409
449
  * pick. */
410
450
  waitUntil?: "load" | "domcontentloaded" | "networkidle";
411
- /** Wait until this CSS selector is on the page — the dependable way to read something a
412
- * script draws. If it never shows up the call comes back as a timeout at no charge, rather
413
- * than the half-built page. */
451
+ /** Wait until this CSS selector is on the page. It is the dependable way to read something
452
+ * a script draws. If it never shows up, the call comes back as a timeout at no charge,
453
+ * rather than the half-built page. */
414
454
  waitForSelector?: string;
415
455
  /** Wait this many milliseconds after the page settles, for the scripts no selector
416
456
  * describes. It comes out of `timeoutMs`, never on top of it. */
@@ -422,11 +462,11 @@ export interface QuoteBatchParams {
422
462
  height: number;
423
463
  };
424
464
  /** Screenshot the whole scrollable page instead of just the window. A long page makes a
425
- * large answer — the image comes back as base64 inside the response. */
465
+ * large answer: the image comes back as base64 inside the response. */
426
466
  fullPage?: boolean;
427
467
  };
428
- /** Extra request headers to send to the page — an authorization token, a cookie, a custom user
429
- * agent. The connection's own headers cannot be set. */
468
+ /** Extra request headers to send to the page, such as an authorization token, a cookie or a
469
+ * custom user agent. The connection's own headers cannot be set. */
430
470
  headers?: Record<string, string>;
431
471
  /** How long to give the page before giving up, in milliseconds. A page that never arrives is a
432
472
  * free outcome. */
@@ -443,8 +483,8 @@ export interface QuoteBatchParams {
443
483
  url: string;
444
484
  /** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
445
485
  * we could not finish the batch. `batch.status` says which: `completed` or `failed`.
446
- * "batch.item.completed" fires per URL — one request from us per item, which is what makes
447
- * a long batch usable before it ends, so ask for it deliberately. */
486
+ * "batch.item.completed" fires per URL, with one request from us per item. That makes a
487
+ * long batch usable before it ends, so ask for it deliberately. */
448
488
  events?: ("batch.completed" | "batch.item.completed")[];
449
489
  };
450
490
  }
@@ -483,7 +523,7 @@ export interface CreateSessionParams {
483
523
  * pool. */
484
524
  proxy?: {
485
525
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
486
- * "custom" is your own proxy — free on every plan, bytes never billed. */
526
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
487
527
  pool?: "residential" | "custom";
488
528
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
489
529
  * for the managed one. */
@@ -515,7 +555,7 @@ export interface QuoteSessionParams {
515
555
  * pool. */
516
556
  proxy?: {
517
557
  /** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
518
- * "custom" is your own proxy — free on every plan, bytes never billed. */
558
+ * "custom" is your own proxy, free on every plan, bytes never billed. */
519
559
  pool?: "residential" | "custom";
520
560
  /** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
521
561
  * for the managed one. */
@@ -49,9 +49,9 @@ const DEFAULT_MAX_WAIT_SECS = 10;
49
49
  /**
50
50
  * Should this failure be retried?
51
51
  *
52
- * if (attempt < 3 && shouldRetry(error, { repeatable: method === "GET" || !!key }))
52
+ * if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
53
53
  */
54
- export function shouldRetry(error: unknown, options: RetryOptions): boolean {
54
+ export function isRetryable(error: unknown, options: RetryOptions): boolean {
55
55
  const { repeatable, durableCodes = [], maxWaitSecs = DEFAULT_MAX_WAIT_SECS } = options;
56
56
  const status = statusOf(error);
57
57
  // No answer at all: nothing about it is a refusal, but a write may have landed before the
@@ -145,7 +145,7 @@ export function parseRetryAfter(value: string | null | undefined): number | unde
145
145
  * frees when another job ends. Present-and-`null` is the whole test; an absent key said nothing.
146
146
  *
147
147
  * Not exported. It is not "is this refusal durable?": it knows nothing of `durableCodes` or of a
148
- * wait too long to hold, so a screen branching on it would disagree with `shouldRetry`.
148
+ * wait too long to hold, so a screen branching on it would disagree with `isRetryable`.
149
149
  */
150
150
  function waitingNeverHelps(error: unknown): boolean {
151
151
  const details = field(error, "details");
@@ -24,7 +24,7 @@
24
24
  * - **The header is read first, in both its forms.** Two copies read the body's wait before the
25
25
  * `Retry-After` header, and read the header only as seconds, so an HTTP date was no wait at all.
26
26
  */
27
- import { parseRetryAfter, retryDelayMs, shouldRetry } from "./retry.ts";
27
+ import { parseRetryAfter, retryDelayMs, isRetryable } from "./retry.ts";
28
28
 
29
29
  /** What a generated method tells the transport about its operation. */
30
30
  export interface RequestSpec {
@@ -193,7 +193,7 @@ export async function send<T = unknown, M = unknown>(
193
193
  };
194
194
  const again =
195
195
  attempt < maxRetries &&
196
- shouldRetry(answer, { repeatable, durableCodes: transport.durableCodes });
196
+ isRetryable(answer, { repeatable, durableCodes: transport.durableCodes });
197
197
  if (!again) throw transport.error(failure);
198
198
  await sleep(retryDelayMs(attempt, answer), signal);
199
199
  }