@scrapa/sdk 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/dist/client.js +2 -2
- package/dist/client.js.map +1 -1
- package/dist/errors.js +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/generated/contract.d.ts +21 -8
- package/dist/generated/contract.d.ts.map +1 -1
- package/dist/generated/contract.js +14 -6
- package/dist/generated/contract.js.map +1 -1
- package/dist/generated/operations.d.ts +45 -43
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +45 -43
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/params.d.ts +142 -100
- package/dist/generated/params.d.ts.map +1 -1
- package/dist/generated/retry.d.ts +2 -2
- package/dist/generated/retry.js +3 -3
- package/dist/generated/transport.js +2 -2
- package/package.json +2 -2
- package/src/client.ts +2 -2
- package/src/errors.ts +1 -1
- package/src/generated/contract.ts +25 -6
- package/src/generated/operations.ts +45 -43
- package/src/generated/params.ts +142 -100
- package/src/generated/retry.ts +3 -3
- package/src/generated/transport.ts +2 -2
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// One method per capability, grouped the way the docs group them, returning the
|
|
5
5
|
// DTO the operation declares. `Scrapa` in ../client.ts supplies request().
|
|
6
6
|
/** The version this client reports in `x-scrapa-client`. */
|
|
7
|
-
export const SDK_VERSION = "0.
|
|
7
|
+
export const SDK_VERSION = "0.3.0";
|
|
8
8
|
/** The hosted API. */
|
|
9
9
|
export const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
10
10
|
export class GeneratedOperations {
|
|
@@ -12,12 +12,13 @@ export class GeneratedOperations {
|
|
|
12
12
|
* Scrape a page
|
|
13
13
|
*
|
|
14
14
|
* Read one web page and hand it back as markdown, HTML, plain text, its links, a screenshot, a
|
|
15
|
-
* PDF
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
15
|
+
* PDF, or the fields you ask for, read off the page by a model. `engine: "auto"` (the default)
|
|
16
|
+
* makes a plain HTTP request first and re-reads the page in a real Chromium when the body
|
|
17
|
+
* turns out to be a JavaScript shell, so you pay for a browser only when the page needed one.
|
|
18
|
+
* To send a search form or a JSON query, use `method: "POST"` with a `body`; to download a
|
|
19
|
+
* file exactly as the site sent it, ask for `formats: ["raw"]`. Every request leaves through a
|
|
20
|
+
* residential proxy, or through your own with `proxy: { pool: "custom", server }`. The answer
|
|
21
|
+
* says which engine replied, where the page finally was and what it cost.
|
|
21
22
|
*
|
|
22
23
|
* Credits: 1 credit with the fetch engine, 5 with the browser (a screenshot or a PDF is
|
|
23
24
|
* included, never a surcharge), 5 more when you ask for `extract`, plus managed-proxy
|
|
@@ -38,11 +39,11 @@ export class GeneratedOperations {
|
|
|
38
39
|
* Price a scrape
|
|
39
40
|
*
|
|
40
41
|
* What the same request would cost, without making it. Free, and refused by the same guards a
|
|
41
|
-
* real call meets
|
|
42
|
+
* real call meets, so a quote is never a price for a call that would be denied. `maxCost` is
|
|
42
43
|
* the ceiling: for `auto` it is the browser's price, which is only charged if the page turns
|
|
43
44
|
* out to need one.
|
|
44
45
|
*
|
|
45
|
-
* Credits: 0 credits
|
|
46
|
+
* Credits: 0 credits. Pricing a call is free.
|
|
46
47
|
*
|
|
47
48
|
* `POST /v1/scrape/quote`
|
|
48
49
|
*/
|
|
@@ -57,12 +58,13 @@ export class GeneratedOperations {
|
|
|
57
58
|
/**
|
|
58
59
|
* Search the web
|
|
59
60
|
*
|
|
60
|
-
* Ask the web a question and get back ranked results
|
|
61
|
-
* holds
|
|
62
|
-
* cache. `country` picks both the exit and the market, so `br` returns what
|
|
63
|
-
* would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
64
|
-
* when you want dated journalism, and add `answer: true` when you want a
|
|
65
|
-
* over the results instead of writing it yourself. To read a result, pass
|
|
61
|
+
* Ask the web a question and get back ranked results: title, URL and the snippet the index
|
|
62
|
+
* holds. They are read live off a real search front through a residential exit, never a
|
|
63
|
+
* reseller's cache. `country` picks both the exit and the market, so `br` returns what
|
|
64
|
+
* somebody in Brazil would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
65
|
+
* for `topic: "news"` when you want dated journalism, and add `answer: true` when you want a
|
|
66
|
+
* paragraph written over the results instead of writing it yourself. To read a result, pass
|
|
67
|
+
* its URL to `scrape`.
|
|
66
68
|
*
|
|
67
69
|
* Credits: 4 credits per 10 results, so a `limit` above 10 costs two rows, plus 1 for `answer:
|
|
68
70
|
* true`. A search every front refused, and anything we broke, cost nothing; a search that ran
|
|
@@ -84,8 +86,8 @@ export class GeneratedOperations {
|
|
|
84
86
|
* Start a batch
|
|
85
87
|
*
|
|
86
88
|
* Hand us a list of URLs and stop holding a connection open. Every one is read with the
|
|
87
|
-
* options you pass here
|
|
88
|
-
* `scrape
|
|
89
|
+
* options you pass here. It uses the same engines, the same exit and the same meter as a
|
|
90
|
+
* single `scrape`, and a page the site blocked costs nothing here too. Nothing is charged
|
|
89
91
|
* when you submit: a batch spends as it goes, so use `quote` for the ceiling. Poll `get`
|
|
90
92
|
* for progress, or pass a `webhook` and we will call you instead. Results are readable for
|
|
91
93
|
* 24 hours.
|
|
@@ -109,9 +111,9 @@ export class GeneratedOperations {
|
|
|
109
111
|
* What that submission would cost at most, without submitting it. `maxCost` prices every
|
|
110
112
|
* URL at the ceiling one page could reach, so the real bill comes in lower by whatever the
|
|
111
113
|
* web refuses for free and by every page that never needed a browser. Bandwidth is on top
|
|
112
|
-
* and cannot be quoted
|
|
114
|
+
* and cannot be quoted, because it depends on what the pages weigh.
|
|
113
115
|
*
|
|
114
|
-
* Credits: 0 credits
|
|
116
|
+
* Credits: 0 credits. Pricing a batch is free.
|
|
115
117
|
*
|
|
116
118
|
* `POST /v1/batches/quote`
|
|
117
119
|
*/
|
|
@@ -124,10 +126,10 @@ export class GeneratedOperations {
|
|
|
124
126
|
/**
|
|
125
127
|
* List your batches
|
|
126
128
|
*
|
|
127
|
-
* Your most recent batches, newest first
|
|
128
|
-
* what it has cost. Results are not included; ask `get` for one batch's pages.
|
|
129
|
+
* Your most recent batches, newest first, with what each one is doing, how far it has got
|
|
130
|
+
* and what it has cost. Results are not included; ask `get` for one batch's pages.
|
|
129
131
|
*
|
|
130
|
-
* Credits: 0 credits
|
|
132
|
+
* Credits: 0 credits. Listing batches is free.
|
|
131
133
|
*
|
|
132
134
|
* `GET /v1/batches`
|
|
133
135
|
*/
|
|
@@ -137,13 +139,13 @@ export class GeneratedOperations {
|
|
|
137
139
|
*
|
|
138
140
|
* Where a batch got to: whether it is still running, how many of its URLs now have a
|
|
139
141
|
* verdict, how many of those the web or our policy refused, how many were never attempted,
|
|
140
|
-
* and what it has spent so far. Ask for `items` to get the results themselves
|
|
142
|
+
* and what it has spent so far. Ask for `items` to get the results themselves: the pages
|
|
141
143
|
* and the refusals, in the order you submitted them, each one the same answer a single
|
|
142
|
-
* `scrape` would have given
|
|
143
|
-
* whose results have expired still answers with everything else, because somebody
|
|
144
|
-
* for those pages must never be told the batch does not exist.
|
|
144
|
+
* `scrape` would have given. You get up to 200 at a time, from wherever `from` says. A
|
|
145
|
+
* batch whose results have expired still answers with everything else, because somebody
|
|
146
|
+
* who paid for those pages must never be told the batch does not exist.
|
|
145
147
|
*
|
|
146
|
-
* Credits: 0 credits
|
|
148
|
+
* Credits: 0 credits. Reading a batch is free.
|
|
147
149
|
*
|
|
148
150
|
* `GET /v1/batches/{id}`
|
|
149
151
|
*/
|
|
@@ -151,12 +153,12 @@ export class GeneratedOperations {
|
|
|
151
153
|
/**
|
|
152
154
|
* Cancel a batch
|
|
153
155
|
*
|
|
154
|
-
* Stop a batch. A URL already being read finishes and is charged
|
|
155
|
-
* already paying for
|
|
156
|
-
*
|
|
157
|
-
*
|
|
156
|
+
* Stop a batch. A URL already being read finishes and is charged, because it is a page we
|
|
157
|
+
* are already paying for. Every URL not yet started becomes `skipped`, which costs nothing
|
|
158
|
+
* and is not a verdict about the URL: submit those positions again whenever you like. Safe
|
|
159
|
+
* to repeat; a batch that has already ended answers with how it ended.
|
|
158
160
|
*
|
|
159
|
-
* Credits: 0 credits
|
|
161
|
+
* Credits: 0 credits. Cancelling is free, and it stops the spending.
|
|
160
162
|
*
|
|
161
163
|
* `DELETE /v1/batches/{id}`
|
|
162
164
|
*/
|
|
@@ -171,10 +173,10 @@ export class GeneratedOperations {
|
|
|
171
173
|
* `puppeteer.connect({ browserWSEndpoint: connectUrl })` in Puppeteer. The browser is
|
|
172
174
|
* yours alone for the whole window, every byte it pulls leaves through the exit you
|
|
173
175
|
* picked, and the socket survives our deploys. The URL carries the session's token and is
|
|
174
|
-
* shown once
|
|
176
|
+
* shown once. It cannot be rebuilt from the id.
|
|
175
177
|
*
|
|
176
178
|
* Credits: 2 credits per minute, reserved for the whole window when the session starts and
|
|
177
|
-
* settled by the minutes actually used when it closes
|
|
179
|
+
* settled by the minutes actually used when it closes, so closing early gives the rest
|
|
178
180
|
* back, and a browser that never came up costs nothing at all. Managed-proxy bandwidth
|
|
179
181
|
* past the first megabyte is added at close; your own proxy is never metered.
|
|
180
182
|
*
|
|
@@ -190,10 +192,10 @@ export class GeneratedOperations {
|
|
|
190
192
|
* Price a session
|
|
191
193
|
*
|
|
192
194
|
* What holding a browser for that long would cost, without holding one. `maxCost` is the
|
|
193
|
-
* whole window at the per-minute rate
|
|
194
|
-
* not use come back when the session closes.
|
|
195
|
+
* whole window at the per-minute rate. It is a ceiling, not a bill, because the minutes
|
|
196
|
+
* you do not use come back when the session closes.
|
|
195
197
|
*
|
|
196
|
-
* Credits: 0 credits
|
|
198
|
+
* Credits: 0 credits. Pricing a session is free.
|
|
197
199
|
*
|
|
198
200
|
* `POST /v1/sessions/quote`
|
|
199
201
|
*/
|
|
@@ -206,10 +208,10 @@ export class GeneratedOperations {
|
|
|
206
208
|
/**
|
|
207
209
|
* List your sessions
|
|
208
210
|
*
|
|
209
|
-
* Your most recent sessions, newest first
|
|
211
|
+
* Your most recent sessions, newest first: the live ones and the ones that have ended,
|
|
210
212
|
* with what each of them cost.
|
|
211
213
|
*
|
|
212
|
-
* Credits: 0 credits
|
|
214
|
+
* Credits: 0 credits. Listing sessions is free.
|
|
213
215
|
*
|
|
214
216
|
* `GET /v1/sessions`
|
|
215
217
|
*/
|
|
@@ -218,10 +220,10 @@ export class GeneratedOperations {
|
|
|
218
220
|
* Read one session
|
|
219
221
|
*
|
|
220
222
|
* Where a session got to: whether it is still live, when it expires, what it has pulled
|
|
221
|
-
* and
|
|
223
|
+
* and, once it has ended, what it finally cost. The connect URL is not repeated here; it
|
|
222
224
|
* is shown once, when the session starts.
|
|
223
225
|
*
|
|
224
|
-
* Credits: 0 credits
|
|
226
|
+
* Credits: 0 credits. Reading a session is free.
|
|
225
227
|
*
|
|
226
228
|
* `GET /v1/sessions/{id}`
|
|
227
229
|
*/
|
|
@@ -230,8 +232,8 @@ export class GeneratedOperations {
|
|
|
230
232
|
* Close a session
|
|
231
233
|
*
|
|
232
234
|
* Stop the browser and settle the bill: the minutes you held it are kept and the rest of
|
|
233
|
-
* the reservation goes back. Safe to repeat
|
|
234
|
-
*
|
|
235
|
+
* the reservation goes back. Safe to repeat: a session that has already ended answers with
|
|
236
|
+
* how it ended.
|
|
235
237
|
*
|
|
236
238
|
* Credits: Closing is free. It is where the session's own bill lands: the unused minutes
|
|
237
239
|
* are refunded and the managed pool's bandwidth is charged.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"operations.js","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;AAChF,2EAA2E;AA+B3E,4DAA4D;AAC5D,MAAM,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC;AAEnC,sBAAsB;AACtB,MAAM,CAAC,MAAM,gBAAgB,GAAG,2BAA2B,CAAC;AAE5D,MAAM,OAAgB,mBAAmB;IAOrC
|
|
1
|
+
{"version":3,"file":"operations.js","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;AAChF,2EAA2E;AA+B3E,4DAA4D;AAC5D,MAAM,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC;AAEnC,sBAAsB;AACtB,MAAM,CAAC,MAAM,gBAAgB,GAAG,2BAA2B,CAAC;AAE5D,MAAM,OAAgB,mBAAmB;IAOrC;;;;;;;;;;;;;;;;;;OAkBG;IACH,MAAM,CACF,MAAoB,EACpB,IAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CACf;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;YACf,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CACD,MAAyB,EACzB,IAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CACf;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,eAAe;YACrB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CACF,MAAoB,EACpB,IAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CACf;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;YACf,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;SACvC,EACD,MAAM,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAEQ,OAAO,GAAG;QACf;;;;;;;;;;;;;;;;WAgBG;QACH,MAAM,EAAE,CACJ,MAAyB,EACzB,IAA6C,EACrB,EAAE,CAC1B,IAAI,CAAC,OAAO,CACR;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,UAAU;YAChB,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP;QACL;;;;;;;;;;;WAWG;QACH,KAAK,EAAE,CACH,MAAwB,EACxB,IAA6C,EACvB,EAAE,CACxB,IAAI,CAAC,OAAO,CACR;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,gBAAgB;YACtB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP;QACL;;;;;;;;;WASG;QACH,IAAI,EAAE,CACF,MAA0B,EAC1B,IAA6C,EAC1B,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;QACzF;;;;;;;;;;;;;;WAcG;QACH,GAAG,EAAE,CACD,MAAsB,EACtB,IAA6C,EAC5B,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;QAC3F;;;;;;;;;;;WAWG;QACH,MAAM,EAAE,CACJ,MAAyB,EACzB,IAA6C,EAC5B,EAAE,CACnB,IAAI,CAAC,OAAO,CACR,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,UAAU,EAAE,IAAI,EAAE,EAC5D,MAAM,EACN,IAAI,CACP;KACR,CAAC;IAEO,QAAQ,GAAG;QAChB;;;;;;;;;;;;;;;;WAgBG;QACH,MAAM,EAAE,CACJ,MAA4B,EAC5B,IAA6C,EACtB,EAAE,CACzB,IAAI,CAAC,OAAO,CACR;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,WAAW;YACjB,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP;QACL;;;;;;;;;;WAUG;QACH,KAAK,EAAE,CACH,MAA2B,EAC3B,IAA6C,EACrB,EAAE,CAC1B,IAAI,CAAC,OAAO,CACR;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,iBAAiB;YACvB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP;QACL;;;;;;;;;WASG;QACH,IAAI,EAAE,CACF,MAA2B,EAC3B,IAA6C,EACxB,EAAE,CACvB,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;QACpE;;;;;;;;;;WAUG;QACH,GAAG,EAAE,CACD,MAAwB,EACxB,IAA6C,EAC1B,EAAE,CACrB,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;QACxE;;;;;;;;;;;WAWG;QACH,KAAK,EAAE,CACH,MAA0B,EAC1B,IAA6C,EAC1B,EAAE,CACrB,IAAI,CAAC,OAAO,CACR,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,EAAE,EAC7D,MAAM,EACN,IAAI,CACP;KACR,CAAC;IAEO,KAAK,GAAG;QACb;;;;;;;;;;WAUG;QACH,GAAG,EAAE,CAAC,IAA6C,EAAqB,EAAE,CACtE,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC;KACvE,CAAC;CACL"}
|