@scrapa/sdk 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -2
- package/dist/errors.d.ts +7 -4
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +8 -5
- package/dist/errors.js.map +1 -1
- package/dist/generated/contract.d.ts +114 -17
- package/dist/generated/contract.d.ts.map +1 -1
- package/dist/generated/contract.js +33 -13
- package/dist/generated/contract.js.map +1 -1
- package/dist/generated/operations.d.ts +61 -21
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +73 -19
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/params.d.ts +493 -61
- package/dist/generated/params.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/errors.ts +11 -7
- package/src/generated/contract.ts +138 -20
- package/src/generated/operations.ts +93 -19
- package/src/generated/params.ts +531 -61
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { RequestOptions, RequestSpec } from "../client.js";
|
|
2
|
-
import type { BatchDto, BatchQuoteDto, CreatedBatchDto, LiveSessionDto, ScrapeQuoteDto, ScrapeResultDto, SearchResultDto, SessionDto, SessionQuoteDto, UsageDto } from "./contract.js";
|
|
3
|
-
import type { CancelBatchParams, CloseSessionParams, CreateBatchParams, CreateSessionParams, GetBatchParams, GetSessionParams, ListBatchesParams, ListSessionsParams, QuoteBatchParams, QuoteScrapeParams, QuoteSessionParams, ScrapeParams, SearchParams } from "./params.js";
|
|
2
|
+
import type { BatchDto, BatchQuoteDto, CreatedBatchDto, LiveSessionDto, MapQuoteDto, MapResultDto, ScrapeQuoteDto, ScrapeResultDto, SearchResultDto, SessionDto, SessionQuoteDto, UsageDto } from "./contract.js";
|
|
3
|
+
import type { CancelBatchParams, CloseSessionParams, CreateBatchParams, CreateSessionParams, GetBatchParams, GetSessionParams, ListBatchesParams, ListSessionsParams, MapSiteParams, QuoteBatchParams, QuoteMapParams, QuoteScrapeParams, QuoteSessionParams, ScrapeParams, SearchParams } from "./params.js";
|
|
4
4
|
/** The version this client reports in `x-scrapa-client`. */
|
|
5
|
-
export declare const SDK_VERSION = "0.
|
|
5
|
+
export declare const SDK_VERSION = "0.5.0";
|
|
6
6
|
/** The hosted API. */
|
|
7
7
|
export declare const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
8
8
|
export declare abstract class GeneratedOperations {
|
|
@@ -15,14 +15,19 @@ export declare abstract class GeneratedOperations {
|
|
|
15
15
|
* makes a plain HTTP request first and re-reads the page in a real Chromium when the body
|
|
16
16
|
* turns out to be a JavaScript shell, so you pay for a browser only when the page needed one.
|
|
17
17
|
* To send a search form or a JSON query, use `method: "POST"` with a `body`; to download a
|
|
18
|
-
* file exactly as the site sent it, ask for `formats: ["raw"]`.
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* file exactly as the site sent it, ask for `formats: ["raw"]`. A link to a PDF or a Word file
|
|
19
|
+
* is read into text too, headings and paragraphs included. To click a cookie banner, type into
|
|
20
|
+
* a search box or run a script before the page is read, send `browser.actions`. Every request
|
|
21
|
+
* leaves through a residential proxy, or through your own with `proxy: { pool: "custom",
|
|
22
|
+
* server }`. The answer says which engine replied, where the page finally was and what it
|
|
23
|
+
* cost.
|
|
21
24
|
*
|
|
22
25
|
* Credits: 1 credit with the fetch engine, 5 with the browser (a screenshot or a PDF is
|
|
23
26
|
* included, never a surcharge), 5 more when you ask for `extract`, plus managed-proxy
|
|
24
|
-
* bandwidth past the first megabyte. A page the site blocked
|
|
25
|
-
*
|
|
27
|
+
* bandwidth past the first megabyte. A page the site blocked and a URL our policy refused cost
|
|
28
|
+
* nothing, unless your own `browser.actions` script ran in the page first. Anything we broke
|
|
29
|
+
* costs nothing, and neither does a page served from a copy you read earlier (`maxAge`),
|
|
30
|
+
* though `extract` on it is still charged.
|
|
26
31
|
*
|
|
27
32
|
* `POST /v1/scrape`
|
|
28
33
|
*/
|
|
@@ -33,13 +38,44 @@ export declare abstract class GeneratedOperations {
|
|
|
33
38
|
* What the same request would cost, without making it. Free, and refused by the same guards a
|
|
34
39
|
* real call meets, so a quote is never a price for a call that would be denied. `maxCost` is
|
|
35
40
|
* the ceiling: for `auto` it is the browser's price, which is only charged if the page turns
|
|
36
|
-
* out to need one.
|
|
41
|
+
* out to need one. With `maxAge`, the quote is the price of reading the page now; if a copy is
|
|
42
|
+
* there, the page costs nothing and only `extract` is charged.
|
|
37
43
|
*
|
|
38
44
|
* Credits: 0 credits. Pricing a call is free.
|
|
39
45
|
*
|
|
40
46
|
* `POST /v1/scrape/quote`
|
|
41
47
|
*/
|
|
42
48
|
quote(params: QuoteScrapeParams, opts?: Omit<RequestOptions, "idempotencyKey">): Promise<ScrapeQuoteDto>;
|
|
49
|
+
/**
|
|
50
|
+
* List a site's URLs
|
|
51
|
+
*
|
|
52
|
+
* Hand back every URL a site lists, without reading the pages: what its robots.txt and
|
|
53
|
+
* sitemaps name (gzipped ones too, and indexes of sitemaps), plus the links on the page you
|
|
54
|
+
* named. Each URL comes back once, without tracking parameters or a trailing slash, with
|
|
55
|
+
* `lastmod` when the sitemap gives one. Only the host you named is kept unless you set
|
|
56
|
+
* `includeSubdomains`, and `search` keeps the URLs that contain a word. To read a page, pass
|
|
57
|
+
* its URL to `scrape`.
|
|
58
|
+
*
|
|
59
|
+
* Credits: 2 credits, plus managed-proxy bandwidth past the first megabyte, every file the map
|
|
60
|
+
* read counted together. A site that lists nothing is charged, because it answered. A site
|
|
61
|
+
* that refused every file, and anything we broke, cost nothing. When our side fails partway,
|
|
62
|
+
* you still get the URLs found so far, with `truncated: true`.
|
|
63
|
+
*
|
|
64
|
+
* `POST /v1/map`
|
|
65
|
+
*/
|
|
66
|
+
map(params: MapSiteParams, opts?: Omit<RequestOptions, "idempotencyKey">): Promise<MapResultDto>;
|
|
67
|
+
/**
|
|
68
|
+
* Price a map
|
|
69
|
+
*
|
|
70
|
+
* What the same map would cost, without making it. Free, and refused by the same guards a real
|
|
71
|
+
* call meets. Bandwidth is priced per megabyte because nobody knows a site's sitemap size
|
|
72
|
+
* before reading it.
|
|
73
|
+
*
|
|
74
|
+
* Credits: 0 credits. Pricing a call is free.
|
|
75
|
+
*
|
|
76
|
+
* `POST /v1/map/quote`
|
|
77
|
+
*/
|
|
78
|
+
quoteMap(params: QuoteMapParams, opts?: Omit<RequestOptions, "idempotencyKey">): Promise<MapQuoteDto>;
|
|
43
79
|
/**
|
|
44
80
|
* Search the web
|
|
45
81
|
*
|
|
@@ -47,9 +83,9 @@ export declare abstract class GeneratedOperations {
|
|
|
47
83
|
* holds. They are read live off a real search front through a residential exit, never a
|
|
48
84
|
* reseller's cache. `country` picks both the exit and the market, so `br` returns what
|
|
49
85
|
* somebody in Brazil would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
50
|
-
* for `topic: "news"` when you want dated journalism
|
|
51
|
-
*
|
|
52
|
-
* its URL to `scrape`.
|
|
86
|
+
* for `topic: "news"` when you want dated journalism or `topic: "images"` when you want
|
|
87
|
+
* pictures, and add `answer: true` when you want a paragraph written over the results instead
|
|
88
|
+
* of writing it yourself. To read a result, pass its URL to `scrape`.
|
|
53
89
|
*
|
|
54
90
|
* Credits: 4 credits per 10 results, so a `limit` above 10 costs two rows, plus 1 for `answer:
|
|
55
91
|
* true`. A search every front refused, and anything we broke, cost nothing; a search that ran
|
|
@@ -65,15 +101,17 @@ export declare abstract class GeneratedOperations {
|
|
|
65
101
|
*
|
|
66
102
|
* Hand us a list of URLs and stop holding a connection open. Every one is read with the
|
|
67
103
|
* options you pass here. It uses the same engines, the same exit and the same meter as a
|
|
68
|
-
* single `scrape`, and a page the site blocked costs nothing here too
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* 24 hours
|
|
104
|
+
* single `scrape`, and a page the site blocked costs nothing here too, unless your own
|
|
105
|
+
* `browser.actions` script ran in the page first. Nothing is charged when you submit: a
|
|
106
|
+
* batch spends as it goes, so use `quote` for the ceiling. Poll `get` for progress, or
|
|
107
|
+
* pass a `webhook` and we will call you instead. Results are readable for 24 hours from
|
|
108
|
+
* when you submit. A batch still running then stops, and the URLs it has not read are
|
|
109
|
+
* `skipped`, free.
|
|
72
110
|
*
|
|
73
111
|
* Credits: Creating a batch is free, and nothing is held: each URL then costs exactly what
|
|
74
112
|
* the same call would have cost on its own, and the pages the web refused cost nothing
|
|
75
|
-
* here too
|
|
76
|
-
* runs.
|
|
113
|
+
* here too, unless your own `browser.actions` script ran in them first. What the batch has
|
|
114
|
+
* spent so far is on the batch itself, and it grows while it runs.
|
|
77
115
|
*
|
|
78
116
|
* `POST /v1/batches`
|
|
79
117
|
*/
|
|
@@ -82,9 +120,11 @@ export declare abstract class GeneratedOperations {
|
|
|
82
120
|
* Price a batch
|
|
83
121
|
*
|
|
84
122
|
* What that submission would cost at most, without submitting it. `maxCost` prices every
|
|
85
|
-
* URL at the ceiling one page could reach
|
|
86
|
-
*
|
|
87
|
-
*
|
|
123
|
+
* URL at the ceiling one page could reach. The real bill comes in lower by every page that
|
|
124
|
+
* never needed a browser and by every outcome that costs nothing: a page the site blocked
|
|
125
|
+
* or our policy refused (unless your own `browser.actions` script ran in it first) and
|
|
126
|
+
* anything we broke. Bandwidth is on top and cannot be quoted, because it depends on what
|
|
127
|
+
* the pages weigh.
|
|
88
128
|
*
|
|
89
129
|
* Credits: 0 credits. Pricing a batch is free.
|
|
90
130
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EACR,QAAQ,EACR,aAAa,EACb,eAAe,EACf,cAAc,EACd,cAAc,EACd,eAAe,EACf,eAAe,EACf,UAAU,EACV,eAAe,EACf,QAAQ,EACX,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACR,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACf,MAAM,aAAa,CAAC;AAErB,4DAA4D;AAC5D,eAAO,MAAM,WAAW,UAAU,CAAC;AAEnC,sBAAsB;AACtB,eAAO,MAAM,gBAAgB,8BAA8B,CAAC;AAE5D,8BAAsB,mBAAmB;IACrC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACxB,IAAI,EAAE,WAAW,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,CAAC,CAAC;IAEb
|
|
1
|
+
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EACR,QAAQ,EACR,aAAa,EACb,eAAe,EACf,cAAc,EACd,WAAW,EACX,YAAY,EACZ,cAAc,EACd,eAAe,EACf,eAAe,EACf,UAAU,EACV,eAAe,EACf,QAAQ,EACX,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACR,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACf,MAAM,aAAa,CAAC;AAErB,4DAA4D;AAC5D,eAAO,MAAM,WAAW,UAAU,CAAC;AAEnC,sBAAsB;AACtB,eAAO,MAAM,gBAAgB,8BAA8B,CAAC;AAE5D,8BAAsB,mBAAmB;IACrC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACxB,IAAI,EAAE,WAAW,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,CAAC,CAAC;IAEb;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CACF,MAAM,EAAE,YAAY,EACpB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,eAAe,CAAC;IAa3B;;;;;;;;;;;;OAYG;IACH,KAAK,CACD,MAAM,EAAE,iBAAiB,EACzB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,cAAc,CAAC;IAa1B;;;;;;;;;;;;;;;;OAgBG;IACH,GAAG,CACC,MAAM,EAAE,aAAa,EACrB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,YAAY,CAAC;IAaxB;;;;;;;;;;OAUG;IACH,QAAQ,CACJ,MAAM,EAAE,cAAc,EACtB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,WAAW,CAAC;IAavB;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CACF,MAAM,EAAE,YAAY,EACpB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,eAAe,CAAC;IAa3B,QAAQ,CAAC,OAAO;QACZ;;;;;;;;;;;;;;;;;;WAkBG;yBAES,iBAAiB,SAClB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,eAAe,CAAC;QAW3B;;;;;;;;;;;;;WAaG;wBAES,gBAAgB,SACjB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,aAAa,CAAC;QAWzB;;;;;;;;;WASG;wBAEU,iBAAiB,SACnB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB;;;;;;;;;;;;;;WAcG;sBAES,cAAc,SACf,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,CAAC;QACpB;;;;;;;;;;;WAWG;yBAES,iBAAiB,SAClB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,CAAC;MAMtB;IAEF,QAAQ,CAAC,QAAQ;QACb;;;;;;;;;;;;;;;;WAgBG;0BAEU,mBAAmB,SACrB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,cAAc,CAAC;QAW1B;;;;;;;;;;WAUG;yBAEU,kBAAkB,SACpB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,eAAe,CAAC;QAW3B;;;;;;;;;WASG;wBAEU,kBAAkB,SACpB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,EAAE,CAAC;QAExB;;;;;;;;;;WAUG;sBAES,gBAAgB,SACjB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,CAAC;QAEtB;;;;;;;;;;;WAWG;wBAES,kBAAkB,SACnB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,CAAC;MAMxB;IAEF,QAAQ,CAAC,KAAK;QACV;;;;;;;;;;WAUG;qBACU,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAAG,OAAO,CAAC,QAAQ,CAAC;MAEzE;CACL"}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// One method per capability, grouped the way the docs group them, returning the
|
|
5
5
|
// DTO the operation declares. `Scrapa` in ../client.ts supplies request().
|
|
6
6
|
/** The version this client reports in `x-scrapa-client`. */
|
|
7
|
-
export const SDK_VERSION = "0.
|
|
7
|
+
export const SDK_VERSION = "0.5.0";
|
|
8
8
|
/** The hosted API. */
|
|
9
9
|
export const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
10
10
|
export class GeneratedOperations {
|
|
@@ -16,14 +16,19 @@ export class GeneratedOperations {
|
|
|
16
16
|
* makes a plain HTTP request first and re-reads the page in a real Chromium when the body
|
|
17
17
|
* turns out to be a JavaScript shell, so you pay for a browser only when the page needed one.
|
|
18
18
|
* To send a search form or a JSON query, use `method: "POST"` with a `body`; to download a
|
|
19
|
-
* file exactly as the site sent it, ask for `formats: ["raw"]`.
|
|
20
|
-
*
|
|
21
|
-
*
|
|
19
|
+
* file exactly as the site sent it, ask for `formats: ["raw"]`. A link to a PDF or a Word file
|
|
20
|
+
* is read into text too, headings and paragraphs included. To click a cookie banner, type into
|
|
21
|
+
* a search box or run a script before the page is read, send `browser.actions`. Every request
|
|
22
|
+
* leaves through a residential proxy, or through your own with `proxy: { pool: "custom",
|
|
23
|
+
* server }`. The answer says which engine replied, where the page finally was and what it
|
|
24
|
+
* cost.
|
|
22
25
|
*
|
|
23
26
|
* Credits: 1 credit with the fetch engine, 5 with the browser (a screenshot or a PDF is
|
|
24
27
|
* included, never a surcharge), 5 more when you ask for `extract`, plus managed-proxy
|
|
25
|
-
* bandwidth past the first megabyte. A page the site blocked
|
|
26
|
-
*
|
|
28
|
+
* bandwidth past the first megabyte. A page the site blocked and a URL our policy refused cost
|
|
29
|
+
* nothing, unless your own `browser.actions` script ran in the page first. Anything we broke
|
|
30
|
+
* costs nothing, and neither does a page served from a copy you read earlier (`maxAge`),
|
|
31
|
+
* though `extract` on it is still charged.
|
|
27
32
|
*
|
|
28
33
|
* `POST /v1/scrape`
|
|
29
34
|
*/
|
|
@@ -41,7 +46,8 @@ export class GeneratedOperations {
|
|
|
41
46
|
* What the same request would cost, without making it. Free, and refused by the same guards a
|
|
42
47
|
* real call meets, so a quote is never a price for a call that would be denied. `maxCost` is
|
|
43
48
|
* the ceiling: for `auto` it is the browser's price, which is only charged if the page turns
|
|
44
|
-
* out to need one.
|
|
49
|
+
* out to need one. With `maxAge`, the quote is the price of reading the page now; if a copy is
|
|
50
|
+
* there, the page costs nothing and only `extract` is charged.
|
|
45
51
|
*
|
|
46
52
|
* Credits: 0 credits. Pricing a call is free.
|
|
47
53
|
*
|
|
@@ -55,6 +61,50 @@ export class GeneratedOperations {
|
|
|
55
61
|
proxy: ["pool", "server", "country", "state", "city"],
|
|
56
62
|
}, params, opts);
|
|
57
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* List a site's URLs
|
|
66
|
+
*
|
|
67
|
+
* Hand back every URL a site lists, without reading the pages: what its robots.txt and
|
|
68
|
+
* sitemaps name (gzipped ones too, and indexes of sitemaps), plus the links on the page you
|
|
69
|
+
* named. Each URL comes back once, without tracking parameters or a trailing slash, with
|
|
70
|
+
* `lastmod` when the sitemap gives one. Only the host you named is kept unless you set
|
|
71
|
+
* `includeSubdomains`, and `search` keeps the URLs that contain a word. To read a page, pass
|
|
72
|
+
* its URL to `scrape`.
|
|
73
|
+
*
|
|
74
|
+
* Credits: 2 credits, plus managed-proxy bandwidth past the first megabyte, every file the map
|
|
75
|
+
* read counted together. A site that lists nothing is charged, because it answered. A site
|
|
76
|
+
* that refused every file, and anything we broke, cost nothing. When our side fails partway,
|
|
77
|
+
* you still get the URLs found so far, with `truncated: true`.
|
|
78
|
+
*
|
|
79
|
+
* `POST /v1/map`
|
|
80
|
+
*/
|
|
81
|
+
map(params, opts) {
|
|
82
|
+
return this.request({
|
|
83
|
+
method: "POST",
|
|
84
|
+
path: "/map",
|
|
85
|
+
repeatable: false,
|
|
86
|
+
proxy: ["pool", "server", "country"],
|
|
87
|
+
}, params, opts);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Price a map
|
|
91
|
+
*
|
|
92
|
+
* What the same map would cost, without making it. Free, and refused by the same guards a real
|
|
93
|
+
* call meets. Bandwidth is priced per megabyte because nobody knows a site's sitemap size
|
|
94
|
+
* before reading it.
|
|
95
|
+
*
|
|
96
|
+
* Credits: 0 credits. Pricing a call is free.
|
|
97
|
+
*
|
|
98
|
+
* `POST /v1/map/quote`
|
|
99
|
+
*/
|
|
100
|
+
quoteMap(params, opts) {
|
|
101
|
+
return this.request({
|
|
102
|
+
method: "POST",
|
|
103
|
+
path: "/map/quote",
|
|
104
|
+
repeatable: true,
|
|
105
|
+
proxy: ["pool", "server", "country"],
|
|
106
|
+
}, params, opts);
|
|
107
|
+
}
|
|
58
108
|
/**
|
|
59
109
|
* Search the web
|
|
60
110
|
*
|
|
@@ -62,9 +112,9 @@ export class GeneratedOperations {
|
|
|
62
112
|
* holds. They are read live off a real search front through a residential exit, never a
|
|
63
113
|
* reseller's cache. `country` picks both the exit and the market, so `br` returns what
|
|
64
114
|
* somebody in Brazil would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
65
|
-
* for `topic: "news"` when you want dated journalism
|
|
66
|
-
*
|
|
67
|
-
* its URL to `scrape`.
|
|
115
|
+
* for `topic: "news"` when you want dated journalism or `topic: "images"` when you want
|
|
116
|
+
* pictures, and add `answer: true` when you want a paragraph written over the results instead
|
|
117
|
+
* of writing it yourself. To read a result, pass its URL to `scrape`.
|
|
68
118
|
*
|
|
69
119
|
* Credits: 4 credits per 10 results, so a `limit` above 10 costs two rows, plus 1 for `answer:
|
|
70
120
|
* true`. A search every front refused, and anything we broke, cost nothing; a search that ran
|
|
@@ -87,15 +137,17 @@ export class GeneratedOperations {
|
|
|
87
137
|
*
|
|
88
138
|
* Hand us a list of URLs and stop holding a connection open. Every one is read with the
|
|
89
139
|
* options you pass here. It uses the same engines, the same exit and the same meter as a
|
|
90
|
-
* single `scrape`, and a page the site blocked costs nothing here too
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
* 24 hours
|
|
140
|
+
* single `scrape`, and a page the site blocked costs nothing here too, unless your own
|
|
141
|
+
* `browser.actions` script ran in the page first. Nothing is charged when you submit: a
|
|
142
|
+
* batch spends as it goes, so use `quote` for the ceiling. Poll `get` for progress, or
|
|
143
|
+
* pass a `webhook` and we will call you instead. Results are readable for 24 hours from
|
|
144
|
+
* when you submit. A batch still running then stops, and the URLs it has not read are
|
|
145
|
+
* `skipped`, free.
|
|
94
146
|
*
|
|
95
147
|
* Credits: Creating a batch is free, and nothing is held: each URL then costs exactly what
|
|
96
148
|
* the same call would have cost on its own, and the pages the web refused cost nothing
|
|
97
|
-
* here too
|
|
98
|
-
* runs.
|
|
149
|
+
* here too, unless your own `browser.actions` script ran in them first. What the batch has
|
|
150
|
+
* spent so far is on the batch itself, and it grows while it runs.
|
|
99
151
|
*
|
|
100
152
|
* `POST /v1/batches`
|
|
101
153
|
*/
|
|
@@ -109,9 +161,11 @@ export class GeneratedOperations {
|
|
|
109
161
|
* Price a batch
|
|
110
162
|
*
|
|
111
163
|
* What that submission would cost at most, without submitting it. `maxCost` prices every
|
|
112
|
-
* URL at the ceiling one page could reach
|
|
113
|
-
*
|
|
114
|
-
*
|
|
164
|
+
* URL at the ceiling one page could reach. The real bill comes in lower by every page that
|
|
165
|
+
* never needed a browser and by every outcome that costs nothing: a page the site blocked
|
|
166
|
+
* or our policy refused (unless your own `browser.actions` script ran in it first) and
|
|
167
|
+
* anything we broke. Bandwidth is on top and cannot be quoted, because it depends on what
|
|
168
|
+
* the pages weigh.
|
|
115
169
|
*
|
|
116
170
|
* Credits: 0 credits. Pricing a batch is free.
|
|
117
171
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"operations.js","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;AAChF,2EAA2E;
|
|
1
|
+
{"version":3,"file":"operations.js","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,0DAA0D;AAC1D,EAAE;AACF,gFAAgF;AAChF,2EAA2E;AAmC3E,4DAA4D;AAC5D,MAAM,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC;AAEnC,sBAAsB;AACtB,MAAM,CAAC,MAAM,gBAAgB,GAAG,2BAA2B,CAAC;AAE5D,MAAM,OAAgB,mBAAmB;IAOrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CACF,MAAoB,EACpB,IAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CACf;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;YACf,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CACD,MAAyB,EACzB,IAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CACf;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,eAAe;YACrB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,GAAG,CACC,MAAqB,EACrB,IAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CACf;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;SACvC,EACD,MAAM,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAED;;;;;;;;;;OAUG;IACH,QAAQ,CACJ,MAAsB,EACtB,IAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CACf;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,YAAY;YAClB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;SACvC,EACD,MAAM,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CACF,MAAoB,EACpB,IAA6C;QAE7C,OAAO,IAAI,CAAC,OAAO,CACf;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,SAAS;YACf,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC;SACvC,EACD,MAAM,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAEQ,OAAO,GAAG;QACf;;;;;;;;;;;;;;;;;;WAkBG;QACH,MAAM,EAAE,CACJ,MAAyB,EACzB,IAA6C,EACrB,EAAE,CAC1B,IAAI,CAAC,OAAO,CACR;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,UAAU;YAChB,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP;QACL;;;;;;;;;;;;;WAaG;QACH,KAAK,EAAE,CACH,MAAwB,EACxB,IAA6C,EACvB,EAAE,CACxB,IAAI,CAAC,OAAO,CACR;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,gBAAgB;YACtB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP;QACL;;;;;;;;;WASG;QACH,IAAI,EAAE,CACF,MAA0B,EAC1B,IAA6C,EAC1B,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;QACzF;;;;;;;;;;;;;;WAcG;QACH,GAAG,EAAE,CACD,MAAsB,EACtB,IAA6C,EAC5B,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;QAC3F;;;;;;;;;;;WAWG;QACH,MAAM,EAAE,CACJ,MAAyB,EACzB,IAA6C,EAC5B,EAAE,CACnB,IAAI,CAAC,OAAO,CACR,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,UAAU,EAAE,IAAI,EAAE,EAC5D,MAAM,EACN,IAAI,CACP;KACR,CAAC;IAEO,QAAQ,GAAG;QAChB;;;;;;;;;;;;;;;;WAgBG;QACH,MAAM,EAAE,CACJ,MAA4B,EAC5B,IAA6C,EACtB,EAAE,CACzB,IAAI,CAAC,OAAO,CACR;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,WAAW;YACjB,UAAU,EAAE,KAAK;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP;QACL;;;;;;;;;;WAUG;QACH,KAAK,EAAE,CACH,MAA2B,EAC3B,IAA6C,EACrB,EAAE,CAC1B,IAAI,CAAC,OAAO,CACR;YACI,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,iBAAiB;YACvB,UAAU,EAAE,IAAI;YAChB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC;SACxD,EACD,MAAM,EACN,IAAI,CACP;QACL;;;;;;;;;WASG;QACH,IAAI,EAAE,CACF,MAA2B,EAC3B,IAA6C,EACxB,EAAE,CACvB,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;QACpE;;;;;;;;;;WAUG;QACH,GAAG,EAAE,CACD,MAAwB,EACxB,IAA6C,EAC1B,EAAE,CACrB,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC;QACxE;;;;;;;;;;;WAWG;QACH,KAAK,EAAE,CACH,MAA0B,EAC1B,IAA6C,EAC1B,EAAE,CACrB,IAAI,CAAC,OAAO,CACR,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI,EAAE,EAC7D,MAAM,EACN,IAAI,CACP;KACR,CAAC;IAEO,KAAK,GAAG;QACb;;;;;;;;;;WAUG;QACH,GAAG,EAAE,CAAC,IAA6C,EAAqB,EAAE,CACtE,IAAI,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC;KACvE,CAAC;CACL"}
|