@scrapa/sdk 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/dist/client.js +2 -2
- package/dist/client.js.map +1 -1
- package/dist/errors.js +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/generated/contract.d.ts +21 -8
- package/dist/generated/contract.d.ts.map +1 -1
- package/dist/generated/contract.js +14 -6
- package/dist/generated/contract.js.map +1 -1
- package/dist/generated/operations.d.ts +45 -43
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +45 -43
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/params.d.ts +142 -100
- package/dist/generated/params.d.ts.map +1 -1
- package/dist/generated/retry.d.ts +2 -2
- package/dist/generated/retry.js +3 -3
- package/dist/generated/transport.js +2 -2
- package/package.json +2 -2
- package/src/client.ts +2 -2
- package/src/errors.ts +1 -1
- package/src/generated/contract.ts +25 -6
- package/src/generated/operations.ts +45 -43
- package/src/generated/params.ts +142 -100
- package/src/generated/retry.ts +3 -3
- package/src/generated/transport.ts +2 -2
|
@@ -1,35 +1,50 @@
|
|
|
1
1
|
/** Arguments for `scrape()`. */
|
|
2
2
|
export interface ScrapeParams {
|
|
3
|
-
/** The page to read. An http(s) URL
|
|
3
|
+
/** The page to read. An http(s) URL. Private and loopback hosts are refused. */
|
|
4
4
|
url: string;
|
|
5
5
|
/** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
|
|
6
|
-
* Chromium; "auto" fetches first and escalates to the browser when the body
|
|
7
|
-
* JavaScript shell, billing whichever engine answered.
|
|
6
|
+
* Chromium; "auto" (the default) fetches first and escalates to the browser when the body
|
|
7
|
+
* turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
|
|
8
|
+
* always use "fetch", so leave it out for them. */
|
|
8
9
|
engine?: "auto" | "fetch" | "browser";
|
|
9
|
-
/** Whether to try again a different way when the first attempt is blocked.
|
|
10
|
-
* one try
|
|
11
|
-
* call: with `auto`, trying again is what charges the browser price instead of
|
|
10
|
+
/** Whether to try again a different way when the first attempt is blocked. On by default.
|
|
11
|
+
* `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
|
|
12
|
+
* how you bound a call: with `auto`, trying again is what charges the browser price instead of
|
|
13
|
+
* the fetch one. A POST is sent once, so it is off for a POST. */
|
|
12
14
|
escalate?: boolean;
|
|
15
|
+
/** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
|
|
16
|
+
* sent once: never retried and never sent again another way, because sending it twice could
|
|
17
|
+
* repeat what it did. */
|
|
18
|
+
method?: "GET" | "POST";
|
|
19
|
+
/** What a POST sends, at most 64 KB. Set its type in `headers`: `{"content-type":
|
|
20
|
+
* "application/json"}` for JSON, or `application/x-www-form-urlencoded` for a form. If the
|
|
21
|
+
* site redirects, a 301, 302 or 303 reads the next page with GET and without the body; a 307
|
|
22
|
+
* or 308 sends it again only if the protocol, host and port stay the same, and otherwise stops
|
|
23
|
+
* and names the new address. */
|
|
24
|
+
body?: string;
|
|
13
25
|
/** How many extra tries to give a page the site blocks, each from a different address. 0 (the
|
|
14
26
|
* default) takes the site's first no as the answer. A blocked page costs nothing, so a try
|
|
15
|
-
* that is blocked again is free
|
|
27
|
+
* that is blocked again is free, but it takes as long as the first one did, so the call can
|
|
16
28
|
* run that many times longer. */
|
|
17
29
|
retry?: number;
|
|
18
30
|
/** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
|
|
19
31
|
* asking for both costs no more than one. "extract" is the page read into fields by a model,
|
|
20
|
-
* and needs an `extract` block saying which ones.
|
|
21
|
-
|
|
32
|
+
* and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
|
|
33
|
+
* it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
|
|
34
|
+
* fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
|
|
35
|
+
* cut short. */
|
|
36
|
+
formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
|
|
22
37
|
/** What the page should answer with. Set `"json"` when you are reading an API: anything else
|
|
23
|
-
* that comes back
|
|
24
|
-
*
|
|
25
|
-
*
|
|
38
|
+
* that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
|
|
39
|
+
* not your answer. It costs nothing, and we try again from another address instead of handing
|
|
40
|
+
* you a page you cannot parse. Without it, whatever the page returns is your answer. */
|
|
26
41
|
expect?: "json";
|
|
27
42
|
/** What to pull out of the page, for the "extract" format. A model reads the page and fills in
|
|
28
|
-
* the shape you asked for
|
|
43
|
+
* the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
|
|
29
44
|
* says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
|
|
30
45
|
* code. */
|
|
31
46
|
extract?: {
|
|
32
|
-
/** The shape you want back, as JSON Schema
|
|
47
|
+
/** The shape you want back, as JSON Schema: an object whose properties are the fields you
|
|
33
48
|
* need. A field the page does not mention comes back null; nothing is invented to fill it. */
|
|
34
49
|
schema?: Record<string, unknown>;
|
|
35
50
|
/** What to pull out, in your own words: `the price, the author and the date it was
|
|
@@ -40,7 +55,7 @@ export interface ScrapeParams {
|
|
|
40
55
|
/** Where the request leaves from. Defaults to the managed residential pool. */
|
|
41
56
|
proxy?: {
|
|
42
57
|
/** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
|
|
43
|
-
* "custom" is your own proxy
|
|
58
|
+
* "custom" is your own proxy, free on every plan, bytes never billed. */
|
|
44
59
|
pool?: "residential" | "custom";
|
|
45
60
|
/** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
|
|
46
61
|
* for the managed one. */
|
|
@@ -60,7 +75,7 @@ export interface ScrapeParams {
|
|
|
60
75
|
* charge; ask for the state or the country alone. */
|
|
61
76
|
city?: string;
|
|
62
77
|
};
|
|
63
|
-
/** How to render it. Only the browser engine reads these
|
|
78
|
+
/** How to render it. Only the browser engine reads these. With `auto` they apply if the page
|
|
64
79
|
* turns out to need one. */
|
|
65
80
|
browser?: {
|
|
66
81
|
/** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
|
|
@@ -68,9 +83,9 @@ export interface ScrapeParams {
|
|
|
68
83
|
* page stops asking for things. Every page gets a short settle after this whichever you
|
|
69
84
|
* pick. */
|
|
70
85
|
waitUntil?: "load" | "domcontentloaded" | "networkidle";
|
|
71
|
-
/** Wait until this CSS selector is on the page
|
|
72
|
-
* script draws. If it never shows up the call comes back as a timeout at no charge,
|
|
73
|
-
* than the half-built page. */
|
|
86
|
+
/** Wait until this CSS selector is on the page. It is the dependable way to read something
|
|
87
|
+
* a script draws. If it never shows up, the call comes back as a timeout at no charge,
|
|
88
|
+
* rather than the half-built page. */
|
|
74
89
|
waitForSelector?: string;
|
|
75
90
|
/** Wait this many milliseconds after the page settles, for the scripts no selector
|
|
76
91
|
* describes. It comes out of `timeoutMs`, never on top of it. */
|
|
@@ -82,11 +97,11 @@ export interface ScrapeParams {
|
|
|
82
97
|
height: number;
|
|
83
98
|
};
|
|
84
99
|
/** Screenshot the whole scrollable page instead of just the window. A long page makes a
|
|
85
|
-
* large answer
|
|
100
|
+
* large answer: the image comes back as base64 inside the response. */
|
|
86
101
|
fullPage?: boolean;
|
|
87
102
|
};
|
|
88
|
-
/** Extra request headers to send to the page
|
|
89
|
-
* agent. The connection's own headers cannot be set. */
|
|
103
|
+
/** Extra request headers to send to the page, such as an authorization token, a cookie or a
|
|
104
|
+
* custom user agent. The connection's own headers cannot be set. */
|
|
90
105
|
headers?: Record<string, string>;
|
|
91
106
|
/** How long to give the page before giving up, in milliseconds. A page that never arrives is a
|
|
92
107
|
* free outcome. */
|
|
@@ -94,36 +109,51 @@ export interface ScrapeParams {
|
|
|
94
109
|
}
|
|
95
110
|
/** Arguments for `quote()`. */
|
|
96
111
|
export interface QuoteScrapeParams {
|
|
97
|
-
/** The page to read. An http(s) URL
|
|
112
|
+
/** The page to read. An http(s) URL. Private and loopback hosts are refused. */
|
|
98
113
|
url: string;
|
|
99
114
|
/** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
|
|
100
|
-
* Chromium; "auto" fetches first and escalates to the browser when the body
|
|
101
|
-
* JavaScript shell, billing whichever engine answered.
|
|
115
|
+
* Chromium; "auto" (the default) fetches first and escalates to the browser when the body
|
|
116
|
+
* turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
|
|
117
|
+
* always use "fetch", so leave it out for them. */
|
|
102
118
|
engine?: "auto" | "fetch" | "browser";
|
|
103
|
-
/** Whether to try again a different way when the first attempt is blocked.
|
|
104
|
-
* one try
|
|
105
|
-
* call: with `auto`, trying again is what charges the browser price instead of
|
|
119
|
+
/** Whether to try again a different way when the first attempt is blocked. On by default.
|
|
120
|
+
* `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
|
|
121
|
+
* how you bound a call: with `auto`, trying again is what charges the browser price instead of
|
|
122
|
+
* the fetch one. A POST is sent once, so it is off for a POST. */
|
|
106
123
|
escalate?: boolean;
|
|
124
|
+
/** The HTTP method. "POST" sends `body` to the URL, for a search form or a JSON query. It is
|
|
125
|
+
* sent once: never retried and never sent again another way, because sending it twice could
|
|
126
|
+
* repeat what it did. */
|
|
127
|
+
method?: "GET" | "POST";
|
|
128
|
+
/** What a POST sends, at most 64 KB. Set its type in `headers`: `{"content-type":
|
|
129
|
+
* "application/json"}` for JSON, or `application/x-www-form-urlencoded` for a form. If the
|
|
130
|
+
* site redirects, a 301, 302 or 303 reads the next page with GET and without the body; a 307
|
|
131
|
+
* or 308 sends it again only if the protocol, host and port stay the same, and otherwise stops
|
|
132
|
+
* and names the new address. */
|
|
133
|
+
body?: string;
|
|
107
134
|
/** How many extra tries to give a page the site blocks, each from a different address. 0 (the
|
|
108
135
|
* default) takes the site's first no as the answer. A blocked page costs nothing, so a try
|
|
109
|
-
* that is blocked again is free
|
|
136
|
+
* that is blocked again is free, but it takes as long as the first one did, so the call can
|
|
110
137
|
* run that many times longer. */
|
|
111
138
|
retry?: number;
|
|
112
139
|
/** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
|
|
113
140
|
* asking for both costs no more than one. "extract" is the page read into fields by a model,
|
|
114
|
-
* and needs an `extract` block saying which ones.
|
|
115
|
-
|
|
141
|
+
* and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
|
|
142
|
+
* it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
|
|
143
|
+
* fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
|
|
144
|
+
* cut short. */
|
|
145
|
+
formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
|
|
116
146
|
/** What the page should answer with. Set `"json"` when you are reading an API: anything else
|
|
117
|
-
* that comes back
|
|
118
|
-
*
|
|
119
|
-
*
|
|
147
|
+
* that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
|
|
148
|
+
* not your answer. It costs nothing, and we try again from another address instead of handing
|
|
149
|
+
* you a page you cannot parse. Without it, whatever the page returns is your answer. */
|
|
120
150
|
expect?: "json";
|
|
121
151
|
/** What to pull out of the page, for the "extract" format. A model reads the page and fills in
|
|
122
|
-
* the shape you asked for
|
|
152
|
+
* the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
|
|
123
153
|
* says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
|
|
124
154
|
* code. */
|
|
125
155
|
extract?: {
|
|
126
|
-
/** The shape you want back, as JSON Schema
|
|
156
|
+
/** The shape you want back, as JSON Schema: an object whose properties are the fields you
|
|
127
157
|
* need. A field the page does not mention comes back null; nothing is invented to fill it. */
|
|
128
158
|
schema?: Record<string, unknown>;
|
|
129
159
|
/** What to pull out, in your own words: `the price, the author and the date it was
|
|
@@ -134,7 +164,7 @@ export interface QuoteScrapeParams {
|
|
|
134
164
|
/** Where the request leaves from. Defaults to the managed residential pool. */
|
|
135
165
|
proxy?: {
|
|
136
166
|
/** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
|
|
137
|
-
* "custom" is your own proxy
|
|
167
|
+
* "custom" is your own proxy, free on every plan, bytes never billed. */
|
|
138
168
|
pool?: "residential" | "custom";
|
|
139
169
|
/** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
|
|
140
170
|
* for the managed one. */
|
|
@@ -154,7 +184,7 @@ export interface QuoteScrapeParams {
|
|
|
154
184
|
* charge; ask for the state or the country alone. */
|
|
155
185
|
city?: string;
|
|
156
186
|
};
|
|
157
|
-
/** How to render it. Only the browser engine reads these
|
|
187
|
+
/** How to render it. Only the browser engine reads these. With `auto` they apply if the page
|
|
158
188
|
* turns out to need one. */
|
|
159
189
|
browser?: {
|
|
160
190
|
/** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
|
|
@@ -162,9 +192,9 @@ export interface QuoteScrapeParams {
|
|
|
162
192
|
* page stops asking for things. Every page gets a short settle after this whichever you
|
|
163
193
|
* pick. */
|
|
164
194
|
waitUntil?: "load" | "domcontentloaded" | "networkidle";
|
|
165
|
-
/** Wait until this CSS selector is on the page
|
|
166
|
-
* script draws. If it never shows up the call comes back as a timeout at no charge,
|
|
167
|
-
* than the half-built page. */
|
|
195
|
+
/** Wait until this CSS selector is on the page. It is the dependable way to read something
|
|
196
|
+
* a script draws. If it never shows up, the call comes back as a timeout at no charge,
|
|
197
|
+
* rather than the half-built page. */
|
|
168
198
|
waitForSelector?: string;
|
|
169
199
|
/** Wait this many milliseconds after the page settles, for the scripts no selector
|
|
170
200
|
* describes. It comes out of `timeoutMs`, never on top of it. */
|
|
@@ -176,11 +206,11 @@ export interface QuoteScrapeParams {
|
|
|
176
206
|
height: number;
|
|
177
207
|
};
|
|
178
208
|
/** Screenshot the whole scrollable page instead of just the window. A long page makes a
|
|
179
|
-
* large answer
|
|
209
|
+
* large answer: the image comes back as base64 inside the response. */
|
|
180
210
|
fullPage?: boolean;
|
|
181
211
|
};
|
|
182
|
-
/** Extra request headers to send to the page
|
|
183
|
-
* agent. The connection's own headers cannot be set. */
|
|
212
|
+
/** Extra request headers to send to the page, such as an authorization token, a cookie or a
|
|
213
|
+
* custom user agent. The connection's own headers cannot be set. */
|
|
184
214
|
headers?: Record<string, string>;
|
|
185
215
|
/** How long to give the page before giving up, in milliseconds. A page that never arrives is a
|
|
186
216
|
* free outcome. */
|
|
@@ -188,10 +218,10 @@ export interface QuoteScrapeParams {
|
|
|
188
218
|
}
|
|
189
219
|
/** Arguments for `search()`. */
|
|
190
220
|
export interface SearchParams {
|
|
191
|
-
/** What to search for, in plain words
|
|
221
|
+
/** What to search for, in plain words: the same thing you would type. */
|
|
192
222
|
query: string;
|
|
193
223
|
/** How many results to hand back at most, 1–20. Billed per 10, so asking for more than 10 costs
|
|
194
|
-
* a second row. Fewer can come back than you asked for
|
|
224
|
+
* a second row. Fewer can come back than you asked for: an index has what it has, and
|
|
195
225
|
* `includeDomains` filters what it returns. */
|
|
196
226
|
limit?: number;
|
|
197
227
|
/** What kind of results. "general" is the web; "news" is dated journalism, and the only topic
|
|
@@ -199,18 +229,18 @@ export interface SearchParams {
|
|
|
199
229
|
topic?: "general" | "news";
|
|
200
230
|
/** Only pages from the last day, week, month or year. Absent is any time. */
|
|
201
231
|
timeRange?: "day" | "week" | "month" | "year";
|
|
202
|
-
/** Only results on these hosts, bare (`linkedin.com`)
|
|
232
|
+
/** Only results on these hosts, bare (`linkedin.com`). Subdomains are included. Asked of the
|
|
203
233
|
* search front AND enforced on the answer, so what comes back is on your list whatever the
|
|
204
234
|
* front did with it. */
|
|
205
235
|
includeDomains?: string[];
|
|
206
236
|
/** Also write a short paragraph answering the query from the results, in the query's language.
|
|
207
237
|
* Costs one extra credit; leave it off when your own model reads the snippets. */
|
|
208
238
|
answer?: boolean;
|
|
209
|
-
/** Where the search leaves from. `country` picks the exit AND the market the front answers for
|
|
210
|
-
*
|
|
239
|
+
/** Where the search leaves from. `country` picks the exit AND the market the front answers for.
|
|
240
|
+
* `br` gets Brazilian results in Portuguese. */
|
|
211
241
|
proxy?: {
|
|
212
242
|
/** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
|
|
213
|
-
* "custom" is your own proxy
|
|
243
|
+
* "custom" is your own proxy, free on every plan, bytes never billed. */
|
|
214
244
|
pool?: "residential" | "custom";
|
|
215
245
|
/** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
|
|
216
246
|
* for the managed one. */
|
|
@@ -225,33 +255,38 @@ export interface SearchParams {
|
|
|
225
255
|
/** Arguments for `batches.create()`. */
|
|
226
256
|
export interface CreateBatchParams {
|
|
227
257
|
/** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
|
|
228
|
-
* Chromium; "auto" fetches first and escalates to the browser when the body
|
|
229
|
-
* JavaScript shell, billing whichever engine answered.
|
|
258
|
+
* Chromium; "auto" (the default) fetches first and escalates to the browser when the body
|
|
259
|
+
* turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
|
|
260
|
+
* always use "fetch", so leave it out for them. */
|
|
230
261
|
engine?: "auto" | "fetch" | "browser";
|
|
231
|
-
/** Whether to try again a different way when the first attempt is blocked.
|
|
232
|
-
* one try
|
|
233
|
-
* call: with `auto`, trying again is what charges the browser price instead of
|
|
262
|
+
/** Whether to try again a different way when the first attempt is blocked. On by default.
|
|
263
|
+
* `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
|
|
264
|
+
* how you bound a call: with `auto`, trying again is what charges the browser price instead of
|
|
265
|
+
* the fetch one. A POST is sent once, so it is off for a POST. */
|
|
234
266
|
escalate?: boolean;
|
|
235
267
|
/** How many extra tries to give a page the site blocks, each from a different address. 0 (the
|
|
236
268
|
* default) takes the site's first no as the answer. A blocked page costs nothing, so a try
|
|
237
|
-
* that is blocked again is free
|
|
269
|
+
* that is blocked again is free, but it takes as long as the first one did, so the call can
|
|
238
270
|
* run that many times longer. */
|
|
239
271
|
retry?: number;
|
|
240
272
|
/** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
|
|
241
273
|
* asking for both costs no more than one. "extract" is the page read into fields by a model,
|
|
242
|
-
* and needs an `extract` block saying which ones.
|
|
243
|
-
|
|
274
|
+
* and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
|
|
275
|
+
* it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
|
|
276
|
+
* fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
|
|
277
|
+
* cut short. */
|
|
278
|
+
formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
|
|
244
279
|
/** What the page should answer with. Set `"json"` when you are reading an API: anything else
|
|
245
|
-
* that comes back
|
|
246
|
-
*
|
|
247
|
-
*
|
|
280
|
+
* that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
|
|
281
|
+
* not your answer. It costs nothing, and we try again from another address instead of handing
|
|
282
|
+
* you a page you cannot parse. Without it, whatever the page returns is your answer. */
|
|
248
283
|
expect?: "json";
|
|
249
284
|
/** What to pull out of the page, for the "extract" format. A model reads the page and fills in
|
|
250
|
-
* the shape you asked for
|
|
285
|
+
* the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
|
|
251
286
|
* says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
|
|
252
287
|
* code. */
|
|
253
288
|
extract?: {
|
|
254
|
-
/** The shape you want back, as JSON Schema
|
|
289
|
+
/** The shape you want back, as JSON Schema: an object whose properties are the fields you
|
|
255
290
|
* need. A field the page does not mention comes back null; nothing is invented to fill it. */
|
|
256
291
|
schema?: Record<string, unknown>;
|
|
257
292
|
/** What to pull out, in your own words: `the price, the author and the date it was
|
|
@@ -262,7 +297,7 @@ export interface CreateBatchParams {
|
|
|
262
297
|
/** Where the request leaves from. Defaults to the managed residential pool. */
|
|
263
298
|
proxy?: {
|
|
264
299
|
/** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
|
|
265
|
-
* "custom" is your own proxy
|
|
300
|
+
* "custom" is your own proxy, free on every plan, bytes never billed. */
|
|
266
301
|
pool?: "residential" | "custom";
|
|
267
302
|
/** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
|
|
268
303
|
* for the managed one. */
|
|
@@ -282,7 +317,7 @@ export interface CreateBatchParams {
|
|
|
282
317
|
* charge; ask for the state or the country alone. */
|
|
283
318
|
city?: string;
|
|
284
319
|
};
|
|
285
|
-
/** How to render it. Only the browser engine reads these
|
|
320
|
+
/** How to render it. Only the browser engine reads these. With `auto` they apply if the page
|
|
286
321
|
* turns out to need one. */
|
|
287
322
|
browser?: {
|
|
288
323
|
/** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
|
|
@@ -290,9 +325,9 @@ export interface CreateBatchParams {
|
|
|
290
325
|
* page stops asking for things. Every page gets a short settle after this whichever you
|
|
291
326
|
* pick. */
|
|
292
327
|
waitUntil?: "load" | "domcontentloaded" | "networkidle";
|
|
293
|
-
/** Wait until this CSS selector is on the page
|
|
294
|
-
* script draws. If it never shows up the call comes back as a timeout at no charge,
|
|
295
|
-
* than the half-built page. */
|
|
328
|
+
/** Wait until this CSS selector is on the page. It is the dependable way to read something
|
|
329
|
+
* a script draws. If it never shows up, the call comes back as a timeout at no charge,
|
|
330
|
+
* rather than the half-built page. */
|
|
296
331
|
waitForSelector?: string;
|
|
297
332
|
/** Wait this many milliseconds after the page settles, for the scripts no selector
|
|
298
333
|
* describes. It comes out of `timeoutMs`, never on top of it. */
|
|
@@ -304,11 +339,11 @@ export interface CreateBatchParams {
|
|
|
304
339
|
height: number;
|
|
305
340
|
};
|
|
306
341
|
/** Screenshot the whole scrollable page instead of just the window. A long page makes a
|
|
307
|
-
* large answer
|
|
342
|
+
* large answer: the image comes back as base64 inside the response. */
|
|
308
343
|
fullPage?: boolean;
|
|
309
344
|
};
|
|
310
|
-
/** Extra request headers to send to the page
|
|
311
|
-
* agent. The connection's own headers cannot be set. */
|
|
345
|
+
/** Extra request headers to send to the page, such as an authorization token, a cookie or a
|
|
346
|
+
* custom user agent. The connection's own headers cannot be set. */
|
|
312
347
|
headers?: Record<string, string>;
|
|
313
348
|
/** How long to give the page before giving up, in milliseconds. A page that never arrives is a
|
|
314
349
|
* free outcome. */
|
|
@@ -323,42 +358,48 @@ export interface CreateBatchParams {
|
|
|
323
358
|
/** Where to POST when something happens. An https URL you control; private and loopback
|
|
324
359
|
* hosts are refused, the same as any other URL we fetch. */
|
|
325
360
|
url: string;
|
|
326
|
-
/** What to call about. "batch.completed" fires once
|
|
327
|
-
*
|
|
328
|
-
*
|
|
361
|
+
/** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
|
|
362
|
+
* we could not finish the batch. `batch.status` says which: `completed` or `failed`.
|
|
363
|
+
* "batch.item.completed" fires per URL, with one request from us per item. That makes a
|
|
364
|
+
* long batch usable before it ends, so ask for it deliberately. */
|
|
329
365
|
events?: ("batch.completed" | "batch.item.completed")[];
|
|
330
366
|
};
|
|
331
367
|
}
|
|
332
368
|
/** Arguments for `batches.quote()`. */
|
|
333
369
|
export interface QuoteBatchParams {
|
|
334
370
|
/** How to read it. "fetch" is a plain HTTP request; "browser" renders the page in a real
|
|
335
|
-
* Chromium; "auto" fetches first and escalates to the browser when the body
|
|
336
|
-
* JavaScript shell, billing whichever engine answered.
|
|
371
|
+
* Chromium; "auto" (the default) fetches first and escalates to the browser when the body
|
|
372
|
+
* turns out to be a JavaScript shell, billing whichever engine answered. A POST and "raw"
|
|
373
|
+
* always use "fetch", so leave it out for them. */
|
|
337
374
|
engine?: "auto" | "fetch" | "browser";
|
|
338
|
-
/** Whether to try again a different way when the first attempt is blocked.
|
|
339
|
-
* one try
|
|
340
|
-
* call: with `auto`, trying again is what charges the browser price instead of
|
|
375
|
+
/** Whether to try again a different way when the first attempt is blocked. On by default.
|
|
376
|
+
* `false` stops after one try. The site's refusal is your answer, and it costs nothing. It is
|
|
377
|
+
* how you bound a call: with `auto`, trying again is what charges the browser price instead of
|
|
378
|
+
* the fetch one. A POST is sent once, so it is off for a POST. */
|
|
341
379
|
escalate?: boolean;
|
|
342
380
|
/** How many extra tries to give a page the site blocks, each from a different address. 0 (the
|
|
343
381
|
* default) takes the site's first no as the answer. A blocked page costs nothing, so a try
|
|
344
|
-
* that is blocked again is free
|
|
382
|
+
* that is blocked again is free, but it takes as long as the first one did, so the call can
|
|
345
383
|
* run that many times longer. */
|
|
346
384
|
retry?: number;
|
|
347
385
|
/** What to hand back. "screenshot" and "pdf" come back as base64 and need the browser engine;
|
|
348
386
|
* asking for both costs no more than one. "extract" is the page read into fields by a model,
|
|
349
|
-
* and needs an `extract` block saying which ones.
|
|
350
|
-
|
|
387
|
+
* and needs an `extract` block saying which ones. "raw" is the file exactly as the site sent
|
|
388
|
+
* it, as base64 with its content type: a PDF report, an image, a spreadsheet. It uses the
|
|
389
|
+
* fetch engine and holds up to 5 MB; a larger file answers FILE_TOO_LARGE at no charge, never
|
|
390
|
+
* cut short. */
|
|
391
|
+
formats?: ("markdown" | "html" | "text" | "links" | "screenshot" | "pdf" | "extract" | "raw")[];
|
|
351
392
|
/** What the page should answer with. Set `"json"` when you are reading an API: anything else
|
|
352
|
-
* that comes back
|
|
353
|
-
*
|
|
354
|
-
*
|
|
393
|
+
* that comes back (a maintenance notice, a "checking your browser" page, a login form) is then
|
|
394
|
+
* not your answer. It costs nothing, and we try again from another address instead of handing
|
|
395
|
+
* you a page you cannot parse. Without it, whatever the page returns is your answer. */
|
|
355
396
|
expect?: "json";
|
|
356
397
|
/** What to pull out of the page, for the "extract" format. A model reads the page and fills in
|
|
357
|
-
* the shape you asked for
|
|
398
|
+
* the shape you asked for. Give it a JSON Schema, a sentence, or both. It reads what the page
|
|
358
399
|
* says; it does not do arithmetic over it, so ask for the numbers and add them up in your own
|
|
359
400
|
* code. */
|
|
360
401
|
extract?: {
|
|
361
|
-
/** The shape you want back, as JSON Schema
|
|
402
|
+
/** The shape you want back, as JSON Schema: an object whose properties are the fields you
|
|
362
403
|
* need. A field the page does not mention comes back null; nothing is invented to fill it. */
|
|
363
404
|
schema?: Record<string, unknown>;
|
|
364
405
|
/** What to pull out, in your own words: `the price, the author and the date it was
|
|
@@ -369,7 +410,7 @@ export interface QuoteBatchParams {
|
|
|
369
410
|
/** Where the request leaves from. Defaults to the managed residential pool. */
|
|
370
411
|
proxy?: {
|
|
371
412
|
/** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
|
|
372
|
-
* "custom" is your own proxy
|
|
413
|
+
* "custom" is your own proxy, free on every plan, bytes never billed. */
|
|
373
414
|
pool?: "residential" | "custom";
|
|
374
415
|
/** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
|
|
375
416
|
* for the managed one. */
|
|
@@ -389,7 +430,7 @@ export interface QuoteBatchParams {
|
|
|
389
430
|
* charge; ask for the state or the country alone. */
|
|
390
431
|
city?: string;
|
|
391
432
|
};
|
|
392
|
-
/** How to render it. Only the browser engine reads these
|
|
433
|
+
/** How to render it. Only the browser engine reads these. With `auto` they apply if the page
|
|
393
434
|
* turns out to need one. */
|
|
394
435
|
browser?: {
|
|
395
436
|
/** When the page counts as loaded. "load" (the default) waits for images and stylesheets;
|
|
@@ -397,9 +438,9 @@ export interface QuoteBatchParams {
|
|
|
397
438
|
* page stops asking for things. Every page gets a short settle after this whichever you
|
|
398
439
|
* pick. */
|
|
399
440
|
waitUntil?: "load" | "domcontentloaded" | "networkidle";
|
|
400
|
-
/** Wait until this CSS selector is on the page
|
|
401
|
-
* script draws. If it never shows up the call comes back as a timeout at no charge,
|
|
402
|
-
* than the half-built page. */
|
|
441
|
+
/** Wait until this CSS selector is on the page. It is the dependable way to read something
|
|
442
|
+
* a script draws. If it never shows up, the call comes back as a timeout at no charge,
|
|
443
|
+
* rather than the half-built page. */
|
|
403
444
|
waitForSelector?: string;
|
|
404
445
|
/** Wait this many milliseconds after the page settles, for the scripts no selector
|
|
405
446
|
* describes. It comes out of `timeoutMs`, never on top of it. */
|
|
@@ -411,11 +452,11 @@ export interface QuoteBatchParams {
|
|
|
411
452
|
height: number;
|
|
412
453
|
};
|
|
413
454
|
/** Screenshot the whole scrollable page instead of just the window. A long page makes a
|
|
414
|
-
* large answer
|
|
455
|
+
* large answer: the image comes back as base64 inside the response. */
|
|
415
456
|
fullPage?: boolean;
|
|
416
457
|
};
|
|
417
|
-
/** Extra request headers to send to the page
|
|
418
|
-
* agent. The connection's own headers cannot be set. */
|
|
458
|
+
/** Extra request headers to send to the page, such as an authorization token, a cookie or a
|
|
459
|
+
* custom user agent. The connection's own headers cannot be set. */
|
|
419
460
|
headers?: Record<string, string>;
|
|
420
461
|
/** How long to give the page before giving up, in milliseconds. A page that never arrives is a
|
|
421
462
|
* free outcome. */
|
|
@@ -430,9 +471,10 @@ export interface QuoteBatchParams {
|
|
|
430
471
|
/** Where to POST when something happens. An https URL you control; private and loopback
|
|
431
472
|
* hosts are refused, the same as any other URL we fetch. */
|
|
432
473
|
url: string;
|
|
433
|
-
/** What to call about. "batch.completed" fires once
|
|
434
|
-
*
|
|
435
|
-
*
|
|
474
|
+
/** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
|
|
475
|
+
* we could not finish the batch. `batch.status` says which: `completed` or `failed`.
|
|
476
|
+
* "batch.item.completed" fires per URL, with one request from us per item. That makes a
|
|
477
|
+
* long batch usable before it ends, so ask for it deliberately. */
|
|
436
478
|
events?: ("batch.completed" | "batch.item.completed")[];
|
|
437
479
|
};
|
|
438
480
|
}
|
|
@@ -467,7 +509,7 @@ export interface CreateSessionParams {
|
|
|
467
509
|
* pool. */
|
|
468
510
|
proxy?: {
|
|
469
511
|
/** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
|
|
470
|
-
* "custom" is your own proxy
|
|
512
|
+
* "custom" is your own proxy, free on every plan, bytes never billed. */
|
|
471
513
|
pool?: "residential" | "custom";
|
|
472
514
|
/** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
|
|
473
515
|
* for the managed one. */
|
|
@@ -498,7 +540,7 @@ export interface QuoteSessionParams {
|
|
|
498
540
|
* pool. */
|
|
499
541
|
proxy?: {
|
|
500
542
|
/** Which exit to leave through. "residential" is the managed pool, metered by the megabyte;
|
|
501
|
-
* "custom" is your own proxy
|
|
543
|
+
* "custom" is your own proxy, free on every plan, bytes never billed. */
|
|
502
544
|
pool?: "residential" | "custom";
|
|
503
545
|
/** Your own proxy, `http(s)://[user:pass@]host:port`. Required for pool "custom", refused
|
|
504
546
|
* for the managed one. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,
|
|
1
|
+
{"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;8BAE0B;IAC1B,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB;;;;qCAIiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+BAA+B;AAC/B,MAAM,WAAW,iBAAiB;IAC9B,gFAAgF;IAChF,GAAG,EAAE,MAAM,CAAC;IACZ;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;8BAE0B;IAC1B,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB;;;;qCAIiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,yEAAyE;IACzE,KAAK,EAAE,MAAM,CAAC;IACd;;oDAEgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;+CAC2C;IAC3C,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;6BAEyB;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;uFACmF;IACnF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;qDACiD;IACjD,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;4EAGoE;QACpE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC7B;;;wDAGoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;;uEAGmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;qBAKiB;IACjB,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,EAAE,CAAC;IAChG;;;6FAGyF;IACzF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;+CAEuC;QACvC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;gFACwE;QACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;yEACqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;4EAGoE;QACpE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC3B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX;kEAC8D;IAC9D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;uFACmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAChC;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;kFAC0E;QAC1E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IAC/B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC7B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd"}
|
|
@@ -36,9 +36,9 @@ export interface RetryOptions {
|
|
|
36
36
|
/**
|
|
37
37
|
* Should this failure be retried?
|
|
38
38
|
*
|
|
39
|
-
* if (attempt < 3 &&
|
|
39
|
+
* if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
|
|
40
40
|
*/
|
|
41
|
-
export declare function
|
|
41
|
+
export declare function isRetryable(error: unknown, options: RetryOptions): boolean;
|
|
42
42
|
/**
|
|
43
43
|
* How long before retry number `attemptIndex` (0 for the first): the server's stated wait when
|
|
44
44
|
* there is one, an exponential backoff otherwise, and never less than the backoff. A
|
package/dist/generated/retry.js
CHANGED
|
@@ -9,9 +9,9 @@ const DEFAULT_MAX_WAIT_SECS = 10;
|
|
|
9
9
|
/**
|
|
10
10
|
* Should this failure be retried?
|
|
11
11
|
*
|
|
12
|
-
* if (attempt < 3 &&
|
|
12
|
+
* if (attempt < 3 && isRetryable(error, { repeatable: method === "GET" || !!key }))
|
|
13
13
|
*/
|
|
14
|
-
export function
|
|
14
|
+
export function isRetryable(error, options) {
|
|
15
15
|
const { repeatable, durableCodes = [], maxWaitSecs = DEFAULT_MAX_WAIT_SECS } = options;
|
|
16
16
|
const status = statusOf(error);
|
|
17
17
|
// No answer at all: nothing about it is a refusal, but a write may have landed before the
|
|
@@ -104,7 +104,7 @@ export function parseRetryAfter(value) {
|
|
|
104
104
|
* frees when another job ends. Present-and-`null` is the whole test; an absent key said nothing.
|
|
105
105
|
*
|
|
106
106
|
* Not exported. It is not "is this refusal durable?": it knows nothing of `durableCodes` or of a
|
|
107
|
-
* wait too long to hold, so a screen branching on it would disagree with `
|
|
107
|
+
* wait too long to hold, so a screen branching on it would disagree with `isRetryable`.
|
|
108
108
|
*/
|
|
109
109
|
function waitingNeverHelps(error) {
|
|
110
110
|
const details = field(error, "details");
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* - **The header is read first, in both its forms.** Two copies read the body's wait before the
|
|
24
24
|
* `Retry-After` header, and read the header only as seconds, so an HTTP date was no wait at all.
|
|
25
25
|
*/
|
|
26
|
-
import { parseRetryAfter, retryDelayMs,
|
|
26
|
+
import { parseRetryAfter, retryDelayMs, isRetryable } from "./retry.js";
|
|
27
27
|
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
28
28
|
const DEFAULT_MAX_RETRIES = 2;
|
|
29
29
|
/**
|
|
@@ -75,7 +75,7 @@ export async function send(transport, spec, params = {}, opts = {}) {
|
|
|
75
75
|
retryAfterSecs: failure.retryAfterSecs,
|
|
76
76
|
};
|
|
77
77
|
const again = attempt < maxRetries &&
|
|
78
|
-
|
|
78
|
+
isRetryable(answer, { repeatable, durableCodes: transport.durableCodes });
|
|
79
79
|
if (!again)
|
|
80
80
|
throw transport.error(failure);
|
|
81
81
|
await sleep(retryDelayMs(attempt, answer), signal);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@scrapa/sdk",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Official TypeScript client for scrapa.app
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "Official TypeScript client for scrapa.app. Scrape any web page as markdown, HTML, text, links, a screenshot or a PDF, search the web, and drive a real browser over CDP.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://scrapa.app",
|
|
7
7
|
"keywords": [
|