@scrapa/sdk 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -2
- package/dist/errors.d.ts +7 -4
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +8 -5
- package/dist/errors.js.map +1 -1
- package/dist/generated/contract.d.ts +72 -18
- package/dist/generated/contract.d.ts.map +1 -1
- package/dist/generated/contract.js +27 -14
- package/dist/generated/contract.js.map +1 -1
- package/dist/generated/operations.d.ts +30 -20
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +30 -20
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/params.d.ts +385 -65
- package/dist/generated/params.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/errors.ts +11 -7
- package/src/generated/contract.ts +89 -24
- package/src/generated/operations.ts +30 -20
- package/src/generated/params.ts +421 -65
package/README.md
CHANGED
|
@@ -58,6 +58,18 @@ const shot = await scrapa.scrape({ url: "https://stripe.com", formats: ["screens
|
|
|
58
58
|
await writeFile("stripe.png", Buffer.from(shot.screenshot!, "base64"));
|
|
59
59
|
```
|
|
60
60
|
|
|
61
|
+
Reading the same page again within the hour? Send `maxAge`, the oldest copy you accept in
|
|
62
|
+
milliseconds (up to `3600000`). If we still hold a copy your account read of that URL, from
|
|
63
|
+
the same country with the same engine, in that time, you get it back for 0 credits (`extract` is
|
|
64
|
+
still charged), with `cached: true` and `cachedAt`, the time it was read. If not, the page is read now at the usual
|
|
65
|
+
price. We may drop a copy sooner, to make room or on a deploy, and a batch keeps its own
|
|
66
|
+
copies, apart from single calls. A copy is never shared with another account.
|
|
67
|
+
|
|
68
|
+
```ts
|
|
69
|
+
const page = await scrapa.scrape({ url: "https://example.com/pricing", maxAge: 600_000 }); // 10 minutes
|
|
70
|
+
page.cached; // true when it came from a copy, so the page cost nothing
|
|
71
|
+
```
|
|
72
|
+
|
|
61
73
|
## Where calls come from
|
|
62
74
|
|
|
63
75
|
Every call leaves through a residential address in the United States unless it says
|
|
@@ -101,6 +113,9 @@ found.results[0].url; // array order IS the ranking: there is no score to sort b
|
|
|
101
113
|
found.cached; // the same question inside the window costs nothing and says so
|
|
102
114
|
```
|
|
103
115
|
|
|
116
|
+
`topic: "images"` hands back pictures: each hit's `url` is the page that shows one, and
|
|
117
|
+
`hit.image` holds the picture's own address, a thumbnail that always loads and its size.
|
|
118
|
+
Credit the page, not the file: no hit says who owns a picture.
|
|
104
119
|
`topic: "news"` is the one topic that reliably carries `publishedAt`. `includeDomains` is
|
|
105
120
|
asked of the front *and* enforced on the answer, so what comes back is on your list whatever
|
|
106
121
|
the front did with the operator. `answer: true` adds a paragraph written over the hits. To
|
|
@@ -230,7 +245,10 @@ try {
|
|
|
230
245
|
} catch (error) {
|
|
231
246
|
if (!(error instanceof ScrapaError)) throw error;
|
|
232
247
|
switch (error.outcome) {
|
|
233
|
-
case "blocked":
|
|
248
|
+
case "blocked":
|
|
249
|
+
// time ran out, nobody said no: give the page more time
|
|
250
|
+
if (error.code === "NAV_TIMEOUT") return scrapa.scrape({ url, timeoutMs: 60_000 });
|
|
251
|
+
// the site said no: try another pool, country or engine
|
|
234
252
|
return scrapa.scrape({ url, proxy: { pool: "residential", country: "de" } });
|
|
235
253
|
case "denied": // our policy said no: retrying cannot help, change the URL
|
|
236
254
|
return null;
|
|
@@ -242,7 +260,9 @@ try {
|
|
|
242
260
|
```
|
|
243
261
|
|
|
244
262
|
`error.cost` is `0` on every one of those, because a page you did not get is a page you do not
|
|
245
|
-
pay for.
|
|
263
|
+
pay for. The one exception is a call whose own `browser.actions` script ran in the page first:
|
|
264
|
+
a `blocked` or `denied` answer then keeps the browser price and its bandwidth, because the
|
|
265
|
+
script may already have read the page. A `broken` call is always refunded. `error.code` carries the exact reason, `error.retryAfter` says when a limit clears,
|
|
246
266
|
and `error.requestId` is what support needs to find the call.
|
|
247
267
|
|
|
248
268
|
One code has no `outcome` and is not your mistake: `INVALID_RESPONSE`. The call got a success
|
package/dist/errors.d.ts
CHANGED
|
@@ -18,8 +18,9 @@ export interface ScrapaErrorInit {
|
|
|
18
18
|
requestId?: string;
|
|
19
19
|
/** The call may have run, and been charged, though no answer from the API says so. */
|
|
20
20
|
mayHaveRun?: boolean;
|
|
21
|
-
/** What this call cost. Zero on
|
|
22
|
-
*
|
|
21
|
+
/** What this call cost. Zero on a refusal, except one your own `browser.actions` script
|
|
22
|
+
* could have caused once it started: that one keeps the browser price and its bandwidth
|
|
23
|
+
* (https://docs.scrapa.app/en/guides/credits lists which). A 5xx is always refunded. */
|
|
23
24
|
cost?: number;
|
|
24
25
|
creditsRemaining?: number;
|
|
25
26
|
cause?: unknown;
|
|
@@ -45,8 +46,10 @@ export declare class ScrapaError extends Error {
|
|
|
45
46
|
/**
|
|
46
47
|
* Who said no — the one thing that decides what to do next.
|
|
47
48
|
*
|
|
48
|
-
* "blocked" the site refused the page. It cost you nothing
|
|
49
|
-
*
|
|
49
|
+
* "blocked" the site refused the page. It cost you nothing, unless your own
|
|
50
|
+
* `browser.actions` script ran in the page first. Try another engine, pool
|
|
51
|
+
* or country. The same call again will be refused again. One code here is
|
|
52
|
+
* not a refusal: on NAV_TIMEOUT time ran out, so send a larger `timeoutMs`.
|
|
50
53
|
* "denied" our policy refused this URL (a private address, the acceptable-use list).
|
|
51
54
|
* Retrying cannot help; change the URL.
|
|
52
55
|
* "broken" we broke it, and already refunded it. Worth one retry as-is.
|
package/dist/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAQA,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAE9E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAExD;;4BAE4B;AAC5B,MAAM,MAAM,eAAe,GAAG,eAAe,GAAG,SAAS,GAAG,kBAAkB,CAAC;AAE/E,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,SAAS,GAAG,eAAe,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;6FAEyF;IACzF,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,qBAAa,WAAY,SAAQ,KAAK;IAClC,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,eAAe,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAClD,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;gBAEvB,IAAI,EAAE,eAAe;IAejC;;;;;;;;;;;;;;OAcG;IACH,IAAI,OAAO,IAAI,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAEzD;IAED;;uEAEmE;IACnE,IAAI,SAAS,IAAI,OAAO,CAQvB;CACJ;AAiBD;;4CAE4C;AAC5C,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAuDvE"}
|
package/dist/errors.js
CHANGED
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
//
|
|
3
3
|
// The API answers a refusal with a machine-readable code, and acting on the code is the
|
|
4
4
|
// whole point: it says WHO said no, which is what decides the caller's next move. The site
|
|
5
|
-
// blocked the page (try another engine, pool or country),
|
|
6
|
-
// (retrying cannot help), a limit needs waiting out,
|
|
7
|
-
// refunded). Collapse that into `Error("Request failed")`
|
|
5
|
+
// blocked the page (try another engine, pool or country), time ran out (send a larger
|
|
6
|
+
// `timeoutMs`), our policy refused the URL (retrying cannot help), a limit needs waiting out,
|
|
7
|
+
// or we broke it (retry as-is, it was refunded). Collapse that into `Error("Request failed")`
|
|
8
|
+
// and the caller is left guessing.
|
|
8
9
|
import { HEADERS, OUTCOME_OF } from "./generated/contract.js";
|
|
9
10
|
import { retryAfterSecs } from "./generated/retry.js";
|
|
10
11
|
export class ScrapaError extends Error {
|
|
@@ -41,8 +42,10 @@ export class ScrapaError extends Error {
|
|
|
41
42
|
/**
|
|
42
43
|
* Who said no — the one thing that decides what to do next.
|
|
43
44
|
*
|
|
44
|
-
* "blocked" the site refused the page. It cost you nothing
|
|
45
|
-
*
|
|
45
|
+
* "blocked" the site refused the page. It cost you nothing, unless your own
|
|
46
|
+
* `browser.actions` script ran in the page first. Try another engine, pool
|
|
47
|
+
* or country. The same call again will be refused again. One code here is
|
|
48
|
+
* not a refusal: on NAV_TIMEOUT time ran out, so send a larger `timeoutMs`.
|
|
46
49
|
* "denied" our policy refused this URL (a private address, the acceptable-use list).
|
|
47
50
|
* Retrying cannot help; change the URL.
|
|
48
51
|
* "broken" we broke it, and already refunded it. Worth one retry as-is.
|
package/dist/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,EAAE;AACF,wFAAwF;AACxF,2FAA2F;AAC3F,
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,EAAE;AACF,wFAAwF;AACxF,2FAA2F;AAC3F,sFAAsF;AACtF,8FAA8F;AAC9F,8FAA8F;AAC9F,mCAAmC;AACnC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAkB,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA8BtD,MAAM,OAAO,WAAY,SAAQ,KAAK;IACzB,IAAI,CAA8B;IAClC,MAAM,CAAS;IACf,UAAU,CAAU;IACpB,MAAM,CAAmC;IACzC,OAAO,CAAW;IAClB,UAAU,CAAU;IACpB,SAAS,CAAU;IAC5B;;;;;OAKG;IACM,UAAU,CAAU;IACpB,IAAI,CAAU;IACd,gBAAgB,CAAU;IAEnC,YAAY,IAAqB;QAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClF,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;IAClD,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,IAAI,OAAO;QACP,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAiB,CAAC,CAAC;IAC5E,CAAC;IAED;;uEAEmE;IACnE,IAAI,SAAS;QACT,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,KAAK,CAAC;QAClC,OAAO,CACH,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,IAAI,CAAC,IAAI,KAAK,SAAS;YACvB,IAAI,CAAC,IAAI,KAAK,qBAAqB;YACnC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAC5B,CAAC;IACN,CAAC;CACJ;AAED,SAAS,YAAY,CAAC,OAA4B,EAAE,IAAY;IAC5D,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACxD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,oEAAoE;AACpE,SAAS,UAAU,CAAC,MAAc;IAC9B,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,qBAAqB,CAAC;IACjD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,WAAW,CAAC;IACvC,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,cAAc,CAAC;IAC1C,OAAO,gBAAgB,CAAC;AAC5B,CAAC;AAED;;4CAE4C;AAC5C,MAAM,UAAU,WAAW,CAAC,OAAgB,EAAE,IAAY;IACtD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC1D,IAAI,MAAM,KAAK,CAAC;QACZ,OAAO,IAAI,WAAW,CAAC;YACnB,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAA2B;YAChF,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,MAAM,KAAK,KAAK;YAC5B,OAAO,EAAE,OAAO,CAAC,QAAQ;gBACrB,CAAC,CAAC,4BAA4B,OAAO,CAAC,SAAS,oDAAoD;gBACnG,CAAC,CAAC,2BAA2B,IAAI,GAAG;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK;SACvB,CAAC,CAAC;IACP,sFAAsF;IACtF,0FAA0F;IAC1F,wFAAwF;IACxF,uEAAuE;IACvE,IAAI,MAAM,GAAG,GAAG;QACZ,OAAO,IAAI,WAAW,CAAC;YACnB,IAAI,EAAE,kBAA4C;YAClD,MAAM;YACN,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,WAAW,MAAM,IAAI,IAAI,aAAa,MAAM,kHAAkH;YACvK,OAAO,EAAE,OAAO,CAAC,IAAI;YACrB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;YACjD,gBAAgB,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC;SAC5E,CAAC,CAAC;IACP,yFAAyF;IACzF,sFAAsF;IACtF,mFAAmF;IACnF,MAAM,UAAU,GAAG,CAAC,QAAQ,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,KAAK,KAAK,CAAC;IAClE,OAAO,IAAI,WAAW,CAAC;QACnB,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAE,QAAQ,CAAC,IAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAClE,MAAM;QACN,UAAU;QACV,mFAAmF;QACnF,gCAAgC;QAChC,OAAO,EACH,QAAQ,EAAE,OAAO;YACjB,WAAW,MAAM,IAAI,IAAI,gBAAgB,MAAM,IAAI,UAAU,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9J,UAAU,EAAE,QAAQ,EAAE,UAAU;QAChC,MAAM,EAAE,QAAQ,EAAE,MAAM;QACxB,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI;QACnD,sEAAsE;QACtE,UAAU,EACN,cAAc,CAAC;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,OAAO,EAAE,QAAQ,EAAE,OAAO;SAC7B,CAAC,IAAI,SAAS;QACnB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,uFAAuF;QACvF,mFAAmF;QACnF,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;QACjD,gBAAgB,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC;KAC5E,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* `AUP_DENIED` are "our policy said no" (retrying cannot help, change the URL); a 5xx is
|
|
11
11
|
* "we broke it" (retry as-is, already refunded).
|
|
12
12
|
*/
|
|
13
|
-
export declare const ERROR_CODES: readonly ["VALIDATION_ERROR", "INVALID_URL", "SSRF_DENIED", "UNAUTHORIZED", "INVALID_TOKEN", "PLAN_GATE", "FORBIDDEN", "AUP_DENIED", "NOT_FOUND", "SESSION_NOT_FOUND", "CONFLICT", "PAYLOAD_TOO_LARGE", "UPSTREAM_BLOCKED", "NAV_TIMEOUT", "PROXY_REFUSED", "FILE_TOO_LARGE", "DOCUMENT_TIMEOUT", "RATE_LIMIT_EXCEEDED", "QUOTA_EXCEEDED", "INTERNAL_ERROR", "SERVICE_UNAVAILABLE", "BILLING_UNAVAILABLE"];
|
|
13
|
+
export declare const ERROR_CODES: readonly ["VALIDATION_ERROR", "INVALID_URL", "SSRF_DENIED", "UNAUTHORIZED", "INVALID_TOKEN", "PLAN_GATE", "FORBIDDEN", "AUP_DENIED", "NOT_FOUND", "SESSION_NOT_FOUND", "CONFLICT", "PAYLOAD_TOO_LARGE", "UPSTREAM_BLOCKED", "NAV_TIMEOUT", "PROXY_REFUSED", "FILE_TOO_LARGE", "DOCUMENT_TIMEOUT", "ACTION_FAILED", "RATE_LIMIT_EXCEEDED", "QUOTA_EXCEEDED", "INTERNAL_ERROR", "SERVICE_UNAVAILABLE", "BILLING_UNAVAILABLE"];
|
|
14
14
|
export type ErrorCode = (typeof ERROR_CODES)[number];
|
|
15
15
|
/** The status each code answers with. The server's error factory, the SDK's fallback and
|
|
16
16
|
* the docs' errors table all read this one map — three copies could drift, one cannot. */
|
|
@@ -32,6 +32,7 @@ export declare const ERROR_STATUS: {
|
|
|
32
32
|
readonly PROXY_REFUSED: 422;
|
|
33
33
|
readonly FILE_TOO_LARGE: 422;
|
|
34
34
|
readonly DOCUMENT_TIMEOUT: 422;
|
|
35
|
+
readonly ACTION_FAILED: 422;
|
|
35
36
|
readonly RATE_LIMIT_EXCEEDED: 429;
|
|
36
37
|
readonly QUOTA_EXCEEDED: 429;
|
|
37
38
|
readonly INTERNAL_ERROR: 500;
|
|
@@ -39,11 +40,15 @@ export declare const ERROR_STATUS: {
|
|
|
39
40
|
readonly BILLING_UNAVAILABLE: 503;
|
|
40
41
|
};
|
|
41
42
|
/**
|
|
42
|
-
* The three
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
43
|
+
* The three outcomes that get a colour, as DESIGN.md names them. Each code here maps to
|
|
44
|
+
* exactly one, and the SDK's `outcome` reads this map to tell a caller what to do next. A code
|
|
45
|
+
* absent from here gets no colour: the caller's own mistake (a bad body, a missing key), or a
|
|
46
|
+
* refusal whose fix is in the request rather than the site (PROXY_REFUSED, FILE_TOO_LARGE,
|
|
47
|
+
* DOCUMENT_TIMEOUT, ACTION_FAILED).
|
|
48
|
+
*
|
|
49
|
+
* Every one of these refusals is free, unless the caller's own `browser.actions` script ran
|
|
50
|
+
* in the page first: then a refusal the script could have caused keeps the browser price and
|
|
51
|
+
* its bandwidth (https://docs.scrapa.app/en/guides/credits lists which). `broken` is always refunded.
|
|
47
52
|
*
|
|
48
53
|
* blocked → the site said no → try another engine, pool or country
|
|
49
54
|
* denied → our policy said no → change the URL; retrying cannot help
|
|
@@ -131,8 +136,9 @@ export type ScrapeFormat = (typeof SCRAPE_FORMATS)[number];
|
|
|
131
136
|
export declare const SESSION_STATUSES: readonly ["starting", "live", "closed", "expired", "failed"];
|
|
132
137
|
export type SessionStatus = (typeof SESSION_STATUSES)[number];
|
|
133
138
|
/** What a search is looking for. `general` is the web; `news` is dated journalism — a
|
|
134
|
-
* different door at every engine, and the only one that reliably carries a date
|
|
135
|
-
|
|
139
|
+
* different door at every engine, and the only one that reliably carries a date; `images`
|
|
140
|
+
* is pictures, each tied to the page that holds it. */
|
|
141
|
+
export declare const SEARCH_TOPICS: readonly ["general", "news", "images"];
|
|
136
142
|
export type SearchTopic = (typeof SEARCH_TOPICS)[number];
|
|
137
143
|
/** Which front answered a search. Not a caller's choice — the ladder picks, and says which
|
|
138
144
|
* one it was, because a result set's flavour follows its index. */
|
|
@@ -193,8 +199,8 @@ export interface ScrapeMetadataDto {
|
|
|
193
199
|
/** How many pages the PDF has, when a PDF was read into text. Absent for anything else,
|
|
194
200
|
* and when only "raw" was asked for. */
|
|
195
201
|
pages?: number;
|
|
196
|
-
/** True when
|
|
197
|
-
*
|
|
202
|
+
/** True when we read only part of the PDF: its first 200 pages, or its first 5 million
|
|
203
|
+
* characters of text, are in the answer and the rest is not. Absent otherwise. */
|
|
198
204
|
truncated?: boolean;
|
|
199
205
|
}
|
|
200
206
|
/** A file, exactly as the site sent it. */
|
|
@@ -204,6 +210,25 @@ export interface ScrapeRawDto {
|
|
|
204
210
|
/** The response's `content-type`, media type only: `application/pdf`. */
|
|
205
211
|
contentType: string | null;
|
|
206
212
|
}
|
|
213
|
+
/** The page's address and HTML at the moment a `scrape` step ran. */
|
|
214
|
+
export interface ScrapeActionPageDto {
|
|
215
|
+
url: string;
|
|
216
|
+
html: string;
|
|
217
|
+
}
|
|
218
|
+
/** What a script returned, as JSON. */
|
|
219
|
+
export interface ScrapeActionReturnDto {
|
|
220
|
+
/** null when the script returned nothing, or something JSON cannot hold. */
|
|
221
|
+
value: unknown;
|
|
222
|
+
}
|
|
223
|
+
/** What the `browser.actions` steps kept along the way, each list in the order its steps ran. */
|
|
224
|
+
export interface ScrapeActionsDto {
|
|
225
|
+
/** One base64 PNG per `screenshot` step. */
|
|
226
|
+
screenshots: string[];
|
|
227
|
+
/** One per `scrape` step. */
|
|
228
|
+
scrapes: ScrapeActionPageDto[];
|
|
229
|
+
/** One per `executeJavascript` step. */
|
|
230
|
+
javascriptReturns: ScrapeActionReturnDto[];
|
|
231
|
+
}
|
|
207
232
|
/** `POST /v1/scrape`'s answer. Only the requested formats are present; everything else is
|
|
208
233
|
* always there. Artifacts (`screenshot`, `pdf`) are base64 in v1. */
|
|
209
234
|
export interface ScrapeResultDto {
|
|
@@ -228,6 +253,12 @@ export interface ScrapeResultDto {
|
|
|
228
253
|
* `fetch`: the body, decompressed — what the caller received. `browser`: every response
|
|
229
254
|
* the page pulled, as carried on the wire. An `auto` call that escalated counts both. */
|
|
230
255
|
bytes: number;
|
|
256
|
+
/** True when this answer is a copy of the page we read for your account earlier, because
|
|
257
|
+
* you sent `maxAge`. The page then costs 0 credits and carries no bandwidth; `extract` on
|
|
258
|
+
* it is still charged. */
|
|
259
|
+
cached: boolean;
|
|
260
|
+
/** When the copy was read, ISO 8601. Present only when `cached` is true. */
|
|
261
|
+
cachedAt?: string;
|
|
231
262
|
markdown?: string;
|
|
232
263
|
html?: string;
|
|
233
264
|
text?: string;
|
|
@@ -241,6 +272,9 @@ export interface ScrapeResultDto {
|
|
|
241
272
|
* could not answer — that credit is handed back and the page still stands. Present only
|
|
242
273
|
* when `extract` was in `formats`. */
|
|
243
274
|
extract?: Record<string, unknown> | null;
|
|
275
|
+
/** What the `browser.actions` steps kept. Present only when the request sent steps; the
|
|
276
|
+
* formats above read the page as the last step left it. */
|
|
277
|
+
actions?: ScrapeActionsDto;
|
|
244
278
|
metadata: ScrapeMetadataDto;
|
|
245
279
|
}
|
|
246
280
|
/** `dryRun: true`'s answer — the price of the call, without making it. */
|
|
@@ -264,8 +298,9 @@ export interface ScrapeQuoteDto {
|
|
|
264
298
|
*
|
|
265
299
|
* The rule that shapes everything here is that **a batch is N scrapes and nothing more**.
|
|
266
300
|
* Each item is charged exactly what the same call would have cost synchronously, on the
|
|
267
|
-
* same terms
|
|
268
|
-
*
|
|
301
|
+
* same terms (a blocked page is free inside a batch too, unless the caller's own
|
|
302
|
+
* `browser.actions` script ran in it first), so there is no batch price, no discount and no
|
|
303
|
+
* surcharge to reason about. What the caller buys is not having to hold a
|
|
269
304
|
* connection open for an hour.
|
|
270
305
|
*
|
|
271
306
|
* `cost` and `bytes` therefore GROW while the batch runs. They are what it has spent so
|
|
@@ -283,7 +318,8 @@ export interface BatchDto {
|
|
|
283
318
|
completed: number;
|
|
284
319
|
/** Of those, how many the web or our policy refused. A `failed` count equal to `total`
|
|
285
320
|
* is still a `completed` batch: every URL got a verdict, and the verdicts were
|
|
286
|
-
* refusals. Free ones
|
|
321
|
+
* refusals. Free ones: a blocked page costs nothing here too, unless the caller's own
|
|
322
|
+
* `browser.actions` script ran in it first. */
|
|
287
323
|
failed: number;
|
|
288
324
|
/** Of those, how many were never attempted — the batch was cancelled, the month's
|
|
289
325
|
* credits ran out under it, or we could not finish it. Counted APART from `failed`
|
|
@@ -346,13 +382,15 @@ export interface BatchWebhookDto {
|
|
|
346
382
|
* events there are many deliveries against one batch, and this is not their sum. */
|
|
347
383
|
attempts: number;
|
|
348
384
|
}
|
|
349
|
-
/** `POST /v1/batches/quote
|
|
350
|
-
* synchronous call would be, so this is `total` × that
|
|
351
|
-
*
|
|
385
|
+
/** `POST /v1/batches/quote`: the ceiling, not a bill. Every item is priced as the same
|
|
386
|
+
* synchronous call would be, so this is `total` × that. The real cost is lower by every page
|
|
387
|
+
* that never needed a browser, and by the outcomes that cost nothing: a page the site blocked
|
|
388
|
+
* or our policy refused (unless the item's own `browser.actions` script ran in it first) and
|
|
389
|
+
* anything we broke. */
|
|
352
390
|
export interface BatchQuoteDto {
|
|
353
391
|
total: number;
|
|
354
392
|
pool: ProxyPool;
|
|
355
|
-
/** What one item would cost at most
|
|
393
|
+
/** What one item would cost at most: the browser row for `auto`, since that is the
|
|
356
394
|
* ceiling a page can escalate to. */
|
|
357
395
|
perItemMaxCost: number;
|
|
358
396
|
/** `total` × `perItemMaxCost`. Bandwidth is on top and cannot be quoted: it depends on
|
|
@@ -391,7 +429,8 @@ export interface MapResultDto {
|
|
|
391
429
|
source: MapSource;
|
|
392
430
|
/** True when the site may list more than this answer holds: `limit` cut the list, or the
|
|
393
431
|
* robots.txt or a sitemap was not read to its end (the site refused it, its size, the
|
|
394
|
-
* 25-document cap,
|
|
432
|
+
* 25-document cap, the time limit, or our own proxy failing, in which case the map is
|
|
433
|
+
* free and these are the URLs found before it). */
|
|
395
434
|
truncated: boolean;
|
|
396
435
|
pool: ProxyPool;
|
|
397
436
|
/** What the managed pool carried for every document the map read, decompressed. The
|
|
@@ -408,12 +447,25 @@ export interface MapQuoteDto {
|
|
|
408
447
|
/** Credits per additional megabyte through the managed pool. 0 on BYO. */
|
|
409
448
|
perMb: number;
|
|
410
449
|
}
|
|
450
|
+
/** The picture behind a hit on the `images` topic. */
|
|
451
|
+
export interface SearchImageDto {
|
|
452
|
+
/** The picture itself, on the site that hosts it. It can be gone, or refuse a download
|
|
453
|
+
* that does not come from a browser: the thumbnail is the one that always loads. */
|
|
454
|
+
url: string;
|
|
455
|
+
/** A small copy the engine keeps, safe to show in a page. */
|
|
456
|
+
thumbnailUrl: string;
|
|
457
|
+
/** Pixels of the original, as the engine reported them. Null when it did not say. */
|
|
458
|
+
width: number | null;
|
|
459
|
+
height: number | null;
|
|
460
|
+
}
|
|
411
461
|
/** One hit, as the index ranked it. The array's ORDER is the ranking — there is no score,
|
|
412
462
|
* because no engine we read publishes one and a rank-derived float would be a made-up
|
|
413
463
|
* number in a public contract. */
|
|
414
464
|
export interface SearchHitDto {
|
|
415
465
|
/** 1-based rank within this answer. */
|
|
416
466
|
position: number;
|
|
467
|
+
/** The page. On the `images` topic it is the page that shows the picture, never the
|
|
468
|
+
* picture's own address: that is `image.url`. */
|
|
417
469
|
url: string;
|
|
418
470
|
title: string;
|
|
419
471
|
/** The engine's own summary of the page, a couple of lines. Never the page itself —
|
|
@@ -422,6 +474,8 @@ export interface SearchHitDto {
|
|
|
422
474
|
/** When the engine dated it, ISO 8601. Null whenever it did not, which on the general
|
|
423
475
|
* topic is most of the time — never guessed from the snippet. */
|
|
424
476
|
publishedAt: string | null;
|
|
477
|
+
/** The picture, on the `images` topic. Null on every other topic. */
|
|
478
|
+
image: SearchImageDto | null;
|
|
425
479
|
}
|
|
426
480
|
/** `POST /v1/search`'s answer. */
|
|
427
481
|
export interface SearchResultDto {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../src/generated/contract.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../src/generated/contract.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,WAAW,6ZAiDd,CAAC;AAEX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD;2FAC2F;AAC3F,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;CAwBqB,CAAC;AAE/C;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAUlF,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,OAAO;IAChB,uEAAuE;;IAEvE,+EAA+E;;;IAG/E,sEAAsE;;IAEtE,yCAAyC;;;CAGnC,CAAC;AAEX,MAAM,WAAW,cAAc;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CACpB;AAED;;kEAEkE;AAClE,MAAM,WAAW,SAAS;IACtB,+EAA+E;IAC/E,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,IAAI,CAAC;CACjB;AAED,oEAAoE;AACpE,MAAM,WAAW,UAAU,CAAC,CAAC;IACzB,IAAI,EAAE,CAAC,CAAC;IACR,IAAI,CAAC,EAAE,cAAc,GAAG,SAAS,CAAC;CACrC;AAED,uFAAuF;AACvF,MAAM,WAAW,QAAQ;IACrB,KAAK,EAAE;QACH,IAAI,EAAE,SAAS,CAAC;QAChB,yFAAyF;QACzF,OAAO,EAAE,MAAM,CAAC;QAChB,gFAAgF;QAChF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,6CAA6C;QAC7C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;QACzC,OAAO,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;CACL;AAGD;;;wDAGwD;AACxD,eAAO,MAAM,WAAW,oCAAqC,CAAC;AAE9D,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD;iFACiF;AACjF,eAAO,MAAM,OAAO,uCAAwC,CAAC;AAE7D,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9C,4FAA4F;AAC5F,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAErD;;;;2DAI2D;AAC3D,eAAO,MAAM,cAAc,uFASjB,CAAC;AAEX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D;;;;4EAI4E;AAC5E,eAAO,MAAM,gBAAgB,8DAA+D,CAAC;AAE7F,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9D;;wDAEwD;AACxD,eAAO,MAAM,aAAa,wCAAyC,CAAC;AAEpE,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzD;oEACoE;AACpE,eAAO,MAAM,cAAc,0CAA2C,CAAC;AAEvE,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D,iFAAiF;AACjF,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;AAErD;;;;mEAImE;AACnE,eAAO,MAAM,cAAc,mEAAoE,CAAC;AAEhG,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE1D;;;;;eAKe;AACf,eAAO,MAAM,mBAAmB,6DAA8D,CAAC;AAE/F,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnE;;;;;;8BAM8B;AAC9B,eAAO,MAAM,cAAc,sDAAuD,CAAC;AAEnF,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D;;;kBAGkB;AAClB,eAAO,MAAM,yBAAyB,6CAA8C,CAAC;AAErF,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,yBAAyB,CAAC,CAAC,MAAM,CAAC,CAAC;AAG/E,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC;AAGvE;qEACqE;AACrE,MAAM,WAAW,iBAAiB;IAC9B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,sDAAsD;IACtD,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;uEAGmE;IACnE,OAAO,EAAE,OAAO,CAAC;IACjB;6CACyC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;uFACmF;IACnF,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,2CAA2C;AAC3C,MAAM,WAAW,YAAY;IACzB,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,qEAAqE;AACrE,MAAM,WAAW,mBAAmB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,uCAAuC;AACvC,MAAM,WAAW,qBAAqB;IAClC,4EAA4E;IAC5E,KAAK,EAAE,OAAO,CAAC;CAClB;AAED,iGAAiG;AACjG,MAAM,WAAW,gBAAgB;IAC7B,4CAA4C;IAC5C,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,6BAA6B;IAC7B,OAAO,EAAE,mBAAmB,EAAE,CAAC;IAC/B,wCAAwC;IACxC,iBAAiB,EAAE,qBAAqB,EAAE,CAAC;CAC9C;AAED;sEACsE;AACtE,MAAM,WAAW,eAAe;IAC5B,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,EAAE,SAAS,CAAC;IAChB;;;mDAG+C;IAC/C,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;gDAE4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;8FAE0F;IAC1F,KAAK,EAAE,MAAM,CAAC;IACd;;+BAE2B;IAC3B,MAAM,EAAE,OAAO,CAAC;IAChB,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;6FACyF;IACzF,GAAG,CAAC,EAAE,YAAY,CAAC;IACnB;;2CAEuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACzC;gEAC4D;IAC5D,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B,QAAQ,EAAE,iBAAiB,CAAC;CAC/B;AAED,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC3B,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,EAAE,SAAS,CAAC;IAChB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,mFAAmF;IACnF,IAAI,EAAE,MAAM,CAAC;IACb;sFACkF;IAClF,OAAO,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;CACjB;AAGD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,WAAW,CAAC;IACpB,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd;gDAC4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB;;;oDAGgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf;;;mBAGe;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,8EAA8E;IAC9E,IAAI,EAAE,SAAS,CAAC;IAChB,wDAAwD;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;8DAG0D;IAC1D,SAAS,EAAE,MAAM,CAAC;IAClB;+DAC2D;IAC3D,KAAK,CAAC,EAAE,YAAY,EAAE,CAAC;IACvB,oFAAoF;IACpF,OAAO,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;uDACuD;AACvD,MAAM,WAAW,YAAY;IACzB;0EACsE;IACtE,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,eAAe,CAAC;IACxB,oEAAoE;IACpE,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB;0DACsD;IACtD,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,8EAA8E;IAC9E,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,6CAA6C;AAC7C,MAAM,WAAW,eAAe;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB;uEACmE;IACnE,MAAM,EAAE,qBAAqB,CAAC;IAC9B;8EAC0E;IAC1E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;yFAEqF;IACrF,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;;yBAIyB;AACzB,MAAM,WAAW,aAAa;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB;0CACsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB;gCAC4B;IAC5B,OAAO,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,eAAgB,SAAQ,QAAQ;IAC7C;;;6DAGyD;IACzD,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAGD;yBACyB;AACzB,MAAM,WAAW,SAAS;IACtB,GAAG,EAAE,MAAM,CAAC;IACZ;iGAC6F;IAC7F,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,+BAA+B;AAC/B,MAAM,WAAW,YAAY;IACzB,wFAAwF;IACxF,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,yEAAyE;IACzE,MAAM,EAAE,SAAS,CAAC;IAClB;;;wDAGoD;IACpD,SAAS,EAAE,OAAO,CAAC;IACnB,IAAI,EAAE,SAAS,CAAC;IAChB;oFACgF;IAChF,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,gDAAgD;AAChD,MAAM,WAAW,WAAW;IACxB,IAAI,EAAE,SAAS,CAAC;IAChB,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;CACjB;AAGD,sDAAsD;AACtD,MAAM,WAAW,cAAc;IAC3B;yFACqF;IACrF,GAAG,EAAE,MAAM,CAAC;IACZ,6DAA6D;IAC7D,YAAY,EAAE,MAAM,CAAC;IACrB,qFAAqF;IACrF,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED;;mCAEmC;AACnC,MAAM,WAAW,YAAY;IACzB,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB;sDACkD;IAClD,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd;uDACmD;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB;sEACkE;IAClE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,qEAAqE;IACrE,KAAK,EAAE,cAAc,GAAG,IAAI,CAAC;CAChC;AAED,kCAAkC;AAClC,MAAM,WAAW,eAAe;IAC5B,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,WAAW,CAAC;IACnB,4BAA4B;IAC5B,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,SAAS,CAAC;IAChB;yEACqE;IACrE,KAAK,EAAE,MAAM,CAAC;IACd;sCACkC;IAClC,MAAM,EAAE,OAAO,CAAC;IAChB;4FACwF;IACxF,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,YAAY,EAAE,CAAC;CAC3B;AAGD;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,aAAa,CAAC;IACtB;;;kEAG8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;4FACwF;IACxF,IAAI,EAAE,SAAS,CAAC;IAChB,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAC;IAClB,oFAAoF;IACpF,eAAe,EAAE,MAAM,CAAC;IACxB;uFACmF;IACnF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAe,SAAQ,UAAU;IAC9C,UAAU,EAAE,MAAM,CAAC;CACtB;AAED;;8CAE8C;AAC9C,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,SAAS,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,yFAAyF;IACzF,OAAO,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;CACjB;AAGD;;kCAEkC;AAClC,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB;mEAC+D;IAC/D,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,sFAAsF;AACtF,MAAM,WAAW,QAAQ;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB;+FAC2F;IAC3F,aAAa,EAAE,MAAM,CAAC;IACtB,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,WAAW,EAAE,CAAC;CACvB"}
|
|
@@ -34,24 +34,31 @@ export const ERROR_CODES = [
|
|
|
34
34
|
"CONFLICT", // 409 — e.g. the api-key name cap race
|
|
35
35
|
"PAYLOAD_TOO_LARGE", // 413 — the request body is over the surface's limit
|
|
36
36
|
// ── 422: the request was fine, the WEB did not cooperate. Deliberately 4xx, not 5xx:
|
|
37
|
-
// "5xx is reserved for what we broke" (AGENTS.md), and these cost the caller nothing
|
|
38
|
-
//
|
|
37
|
+
// "5xx is reserved for what we broke" (AGENTS.md), and these cost the caller nothing
|
|
38
|
+
// unless their own `browser.actions` script ran in the page first (OUTCOME_OF). 422
|
|
39
|
+
// also tells an HTTP client the truth: replaying the identical request gets the
|
|
39
40
|
// identical refusal, so the fix is a different engine/pool/country, not a retry.
|
|
40
41
|
"UPSTREAM_BLOCKED", // 422 — an anti-bot wall, a captcha, a hard 403 from the target
|
|
41
42
|
"NAV_TIMEOUT", // 422 — the page never reached a usable state inside the budget
|
|
42
43
|
// The exit proxy would not place the call, so the site never saw it: YOUR proxy refused
|
|
43
44
|
// or did not answer, or the managed pool has no exit in the place you asked for. Free,
|
|
44
|
-
//
|
|
45
|
+
// except your own proxy's refusal once your own `browser.actions` script ran. No colour:
|
|
46
|
+
// the fix is in the request (the proxy, the credentials, a wider place).
|
|
45
47
|
"PROXY_REFUSED", // 422 — details {reason: "no_exit" | "refused" | "unreachable", place?, status?}
|
|
46
48
|
// The site sent a file bigger than `raw` hands back whole. Not 413, which tells an HTTP
|
|
47
49
|
// client its OWN request was too big. Free, and no colour, for PROXY_REFUSED's reason:
|
|
48
50
|
// the fix is in the request (another format, or download it yourself), not another road.
|
|
49
51
|
"FILE_TOO_LARGE", // 422 — details {limit}, in bytes
|
|
50
52
|
// A PDF or a Word file arrived whole, and reading it into text took longer than the time
|
|
51
|
-
// the call had left. Free, and no colour: the site
|
|
52
|
-
// pool or country reads the same file just as
|
|
53
|
-
// "raw" and reading it yourself.
|
|
54
|
-
"DOCUMENT_TIMEOUT", // 422: details {budgetMs
|
|
53
|
+
// the call had left, or than the most one file may read. Free, and no colour: the site
|
|
54
|
+
// answered in full, so another engine, pool or country reads the same file just as
|
|
55
|
+
// slowly. The fix is "raw" and reading it yourself, or a smaller file.
|
|
56
|
+
"DOCUMENT_TIMEOUT", // 422: details {budgetMs, readMaxMs}: the call's timeout, and the most one file may read
|
|
57
|
+
// One of the caller's `browser.actions` steps did not work on the page: a selector that
|
|
58
|
+
// matched nothing, a key Chrome does not know, a script that threw. Free unless one of the
|
|
59
|
+
// caller's scripts ran (their code read the page), and no colour, for PROXY_REFUSED's
|
|
60
|
+
// reason: the page answered, and the fix is in the step, not the road.
|
|
61
|
+
"ACTION_FAILED", // 422: details {index, type, reason}, index counting from 0
|
|
55
62
|
"RATE_LIMIT_EXCEEDED", // 429 — the per-minute burst bucket; Retry-After + details {limit}
|
|
56
63
|
"QUOTA_EXCEEDED", // 429 — a durable bound that WAITING clears; details {scope, limit, resetAt}
|
|
57
64
|
"INTERNAL_ERROR", // 500 — we broke it; the body is always generic, the cause stays server-side
|
|
@@ -78,6 +85,7 @@ export const ERROR_STATUS = {
|
|
|
78
85
|
PROXY_REFUSED: 422,
|
|
79
86
|
FILE_TOO_LARGE: 422,
|
|
80
87
|
DOCUMENT_TIMEOUT: 422,
|
|
88
|
+
ACTION_FAILED: 422,
|
|
81
89
|
RATE_LIMIT_EXCEEDED: 429,
|
|
82
90
|
QUOTA_EXCEEDED: 429,
|
|
83
91
|
INTERNAL_ERROR: 500,
|
|
@@ -85,11 +93,15 @@ export const ERROR_STATUS = {
|
|
|
85
93
|
BILLING_UNAVAILABLE: 503,
|
|
86
94
|
};
|
|
87
95
|
/**
|
|
88
|
-
* The three
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
96
|
+
* The three outcomes that get a colour, as DESIGN.md names them. Each code here maps to
|
|
97
|
+
* exactly one, and the SDK's `outcome` reads this map to tell a caller what to do next. A code
|
|
98
|
+
* absent from here gets no colour: the caller's own mistake (a bad body, a missing key), or a
|
|
99
|
+
* refusal whose fix is in the request rather than the site (PROXY_REFUSED, FILE_TOO_LARGE,
|
|
100
|
+
* DOCUMENT_TIMEOUT, ACTION_FAILED).
|
|
101
|
+
*
|
|
102
|
+
* Every one of these refusals is free, unless the caller's own `browser.actions` script ran
|
|
103
|
+
* in the page first: then a refusal the script could have caused keeps the browser price and
|
|
104
|
+
* its bandwidth (https://docs.scrapa.app/en/guides/credits lists which). `broken` is always refunded.
|
|
93
105
|
*
|
|
94
106
|
* blocked → the site said no → try another engine, pool or country
|
|
95
107
|
* denied → our policy said no → change the URL; retrying cannot help
|
|
@@ -157,8 +169,9 @@ export const SCRAPE_FORMATS = [
|
|
|
157
169
|
* is ours). A session that never reached `live` never charged a minute. */
|
|
158
170
|
export const SESSION_STATUSES = ["starting", "live", "closed", "expired", "failed"];
|
|
159
171
|
/** What a search is looking for. `general` is the web; `news` is dated journalism — a
|
|
160
|
-
* different door at every engine, and the only one that reliably carries a date
|
|
161
|
-
|
|
172
|
+
* different door at every engine, and the only one that reliably carries a date; `images`
|
|
173
|
+
* is pictures, each tied to the page that holds it. */
|
|
174
|
+
export const SEARCH_TOPICS = ["general", "news", "images"];
|
|
162
175
|
/** Which front answered a search. Not a caller's choice — the ladder picks, and says which
|
|
163
176
|
* one it was, because a result set's flavour follows its index. */
|
|
164
177
|
export const SEARCH_ENGINES = ["bing", "brave", "duckduckgo"];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contract.js","sourceRoot":"","sources":["../../src/generated/contract.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,gZAAgZ;AAChZ,EAAE;AACF,oEAAoE;AAEpE,wEAAwE;AAExE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACvB,kBAAkB,EAAE,iFAAiF;IACrG,aAAa,EAAE,+DAA+D;IAC9E,aAAa,EAAE,sFAAsF;IACrG,cAAc,EAAE,gCAAgC;IAChD,eAAe,EAAE,6EAA6E;IAC9F,oFAAoF;IACpF,mFAAmF;IACnF,qFAAqF;IACrF,uFAAuF;IACvF,qFAAqF;IACrF,WAAW,EAAE,2EAA2E;IACxF,WAAW,EAAE,uCAAuC;IACpD,YAAY,EAAE,yEAAyE;IACvF,WAAW,EAAE,uEAAuE;IACpF,mBAAmB,EAAE,0DAA0D;IAC/E,UAAU,EAAE,uCAAuC;IACnD,mBAAmB,EAAE,qDAAqD;IAC1E,sFAAsF;IACtF,
|
|
1
|
+
{"version":3,"file":"contract.js","sourceRoot":"","sources":["../../src/generated/contract.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,gZAAgZ;AAChZ,EAAE;AACF,oEAAoE;AAEpE,wEAAwE;AAExE;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACvB,kBAAkB,EAAE,iFAAiF;IACrG,aAAa,EAAE,+DAA+D;IAC9E,aAAa,EAAE,sFAAsF;IACrG,cAAc,EAAE,gCAAgC;IAChD,eAAe,EAAE,6EAA6E;IAC9F,oFAAoF;IACpF,mFAAmF;IACnF,qFAAqF;IACrF,uFAAuF;IACvF,qFAAqF;IACrF,WAAW,EAAE,2EAA2E;IACxF,WAAW,EAAE,uCAAuC;IACpD,YAAY,EAAE,yEAAyE;IACvF,WAAW,EAAE,uEAAuE;IACpF,mBAAmB,EAAE,0DAA0D;IAC/E,UAAU,EAAE,uCAAuC;IACnD,mBAAmB,EAAE,qDAAqD;IAC1E,sFAAsF;IACtF,wFAAwF;IACxF,uFAAuF;IACvF,mFAAmF;IACnF,oFAAoF;IACpF,kBAAkB,EAAE,gEAAgE;IACpF,aAAa,EAAE,gEAAgE;IAC/E,wFAAwF;IACxF,uFAAuF;IACvF,yFAAyF;IACzF,yEAAyE;IACzE,eAAe,EAAE,iFAAiF;IAClG,wFAAwF;IACxF,uFAAuF;IACvF,yFAAyF;IACzF,gBAAgB,EAAE,kCAAkC;IACpD,yFAAyF;IACzF,uFAAuF;IACvF,mFAAmF;IACnF,uEAAuE;IACvE,kBAAkB,EAAE,yFAAyF;IAC7G,wFAAwF;IACxF,2FAA2F;IAC3F,sFAAsF;IACtF,uEAAuE;IACvE,eAAe,EAAE,4DAA4D;IAC7E,qBAAqB,EAAE,mEAAmE;IAC1F,gBAAgB,EAAE,6EAA6E;IAC/F,gBAAgB,EAAE,6EAA6E;IAC/F,qBAAqB,EAAE,wFAAwF;IAC/G,qBAAqB,EAAE,uFAAuF;CACxG,CAAC;AAIX;2FAC2F;AAC3F,MAAM,CAAC,MAAM,YAAY,GAAG;IACxB,gBAAgB,EAAE,GAAG;IACrB,WAAW,EAAE,GAAG;IAChB,WAAW,EAAE,GAAG;IAChB,YAAY,EAAE,GAAG;IACjB,aAAa,EAAE,GAAG;IAClB,SAAS,EAAE,GAAG;IACd,SAAS,EAAE,GAAG;IACd,UAAU,EAAE,GAAG;IACf,SAAS,EAAE,GAAG;IACd,iBAAiB,EAAE,GAAG;IACtB,QAAQ,EAAE,GAAG;IACb,iBAAiB,EAAE,GAAG;IACtB,gBAAgB,EAAE,GAAG;IACrB,WAAW,EAAE,GAAG;IAChB,aAAa,EAAE,GAAG;IAClB,cAAc,EAAE,GAAG;IACnB,gBAAgB,EAAE,GAAG;IACrB,aAAa,EAAE,GAAG;IAClB,mBAAmB,EAAE,GAAG;IACxB,cAAc,EAAE,GAAG;IACnB,cAAc,EAAE,GAAG;IACnB,mBAAmB,EAAE,GAAG;IACxB,mBAAmB,EAAE,GAAG;CACkB,CAAC;AAE/C;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,UAAU,GAAgE;IACnF,gBAAgB,EAAE,SAAS;IAC3B,WAAW,EAAE,SAAS;IACtB,WAAW,EAAE,QAAQ;IACrB,UAAU,EAAE,QAAQ;IACpB,cAAc,EAAE,QAAQ;IACxB,mBAAmB,EAAE,QAAQ;IAC7B,qFAAqF;IACrF,uFAAuF;IACvF,mBAAmB,EAAE,QAAQ;CAChC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG;IACnB,uEAAuE;IACvE,IAAI,EAAE,gBAAgB;IACtB,+EAA+E;IAC/E,gBAAgB,EAAE,qBAAqB;IACvC,SAAS,EAAE,cAAc;IACzB,sEAAsE;IACtE,MAAM,EAAE,iBAAiB;IACzB,yCAAyC;IACzC,MAAM,EAAE,iBAAiB;IACzB,UAAU,EAAE,aAAa;CACnB,CAAC;AAuCX,wEAAwE;AAExE;;;wDAGwD;AACxD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,QAAQ,CAAU,CAAC;AAI9D;iFACiF;AACjF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAU,CAAC;AAO7D;;;;2DAI2D;AAC3D,MAAM,CAAC,MAAM,cAAc,GAAG;IAC1B,UAAU;IACV,MAAM;IACN,MAAM;IACN,OAAO;IACP,YAAY;IACZ,KAAK;IACL,SAAS;IACT,KAAK;CACC,CAAC;AAIX;;;;4EAI4E;AAC5E,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAU,CAAC;AAI7F;;wDAEwD;AACxD,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAU,CAAC;AAIpE;oEACoE;AACpE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,YAAY,CAAU,CAAC;AAOvE;;;;mEAImE;AACnE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAU,CAAC;AAIhG;;;;;eAKe;AACf,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAU,CAAC;AAI/F;;;;;;8BAM8B;AAC9B,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,iBAAiB,EAAE,sBAAsB,CAAU,CAAC;AAInF;;;kBAGkB;AAClB,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAU,CAAC"}
|
|
@@ -2,7 +2,7 @@ import type { RequestOptions, RequestSpec } from "../client.js";
|
|
|
2
2
|
import type { BatchDto, BatchQuoteDto, CreatedBatchDto, LiveSessionDto, MapQuoteDto, MapResultDto, ScrapeQuoteDto, ScrapeResultDto, SearchResultDto, SessionDto, SessionQuoteDto, UsageDto } from "./contract.js";
|
|
3
3
|
import type { CancelBatchParams, CloseSessionParams, CreateBatchParams, CreateSessionParams, GetBatchParams, GetSessionParams, ListBatchesParams, ListSessionsParams, MapSiteParams, QuoteBatchParams, QuoteMapParams, QuoteScrapeParams, QuoteSessionParams, ScrapeParams, SearchParams } from "./params.js";
|
|
4
4
|
/** The version this client reports in `x-scrapa-client`. */
|
|
5
|
-
export declare const SDK_VERSION = "0.
|
|
5
|
+
export declare const SDK_VERSION = "0.5.0";
|
|
6
6
|
/** The hosted API. */
|
|
7
7
|
export declare const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
8
8
|
export declare abstract class GeneratedOperations {
|
|
@@ -16,14 +16,18 @@ export declare abstract class GeneratedOperations {
|
|
|
16
16
|
* turns out to be a JavaScript shell, so you pay for a browser only when the page needed one.
|
|
17
17
|
* To send a search form or a JSON query, use `method: "POST"` with a `body`; to download a
|
|
18
18
|
* file exactly as the site sent it, ask for `formats: ["raw"]`. A link to a PDF or a Word file
|
|
19
|
-
* is read into text too, headings and paragraphs included.
|
|
20
|
-
*
|
|
21
|
-
*
|
|
19
|
+
* is read into text too, headings and paragraphs included. To click a cookie banner, type into
|
|
20
|
+
* a search box or run a script before the page is read, send `browser.actions`. Every request
|
|
21
|
+
* leaves through a residential proxy, or through your own with `proxy: { pool: "custom",
|
|
22
|
+
* server }`. The answer says which engine replied, where the page finally was and what it
|
|
23
|
+
* cost.
|
|
22
24
|
*
|
|
23
25
|
* Credits: 1 credit with the fetch engine, 5 with the browser (a screenshot or a PDF is
|
|
24
26
|
* included, never a surcharge), 5 more when you ask for `extract`, plus managed-proxy
|
|
25
|
-
* bandwidth past the first megabyte. A page the site blocked
|
|
26
|
-
*
|
|
27
|
+
* bandwidth past the first megabyte. A page the site blocked and a URL our policy refused cost
|
|
28
|
+
* nothing, unless your own `browser.actions` script ran in the page first. Anything we broke
|
|
29
|
+
* costs nothing, and neither does a page served from a copy you read earlier (`maxAge`),
|
|
30
|
+
* though `extract` on it is still charged.
|
|
27
31
|
*
|
|
28
32
|
* `POST /v1/scrape`
|
|
29
33
|
*/
|
|
@@ -34,7 +38,8 @@ export declare abstract class GeneratedOperations {
|
|
|
34
38
|
* What the same request would cost, without making it. Free, and refused by the same guards a
|
|
35
39
|
* real call meets, so a quote is never a price for a call that would be denied. `maxCost` is
|
|
36
40
|
* the ceiling: for `auto` it is the browser's price, which is only charged if the page turns
|
|
37
|
-
* out to need one.
|
|
41
|
+
* out to need one. With `maxAge`, the quote is the price of reading the page now; if a copy is
|
|
42
|
+
* there, the page costs nothing and only `extract` is charged.
|
|
38
43
|
*
|
|
39
44
|
* Credits: 0 credits. Pricing a call is free.
|
|
40
45
|
*
|
|
@@ -53,7 +58,8 @@ export declare abstract class GeneratedOperations {
|
|
|
53
58
|
*
|
|
54
59
|
* Credits: 2 credits, plus managed-proxy bandwidth past the first megabyte, every file the map
|
|
55
60
|
* read counted together. A site that lists nothing is charged, because it answered. A site
|
|
56
|
-
* that refused every file, and anything we broke, cost nothing.
|
|
61
|
+
* that refused every file, and anything we broke, cost nothing. When our side fails partway,
|
|
62
|
+
* you still get the URLs found so far, with `truncated: true`.
|
|
57
63
|
*
|
|
58
64
|
* `POST /v1/map`
|
|
59
65
|
*/
|
|
@@ -77,9 +83,9 @@ export declare abstract class GeneratedOperations {
|
|
|
77
83
|
* holds. They are read live off a real search front through a residential exit, never a
|
|
78
84
|
* reseller's cache. `country` picks both the exit and the market, so `br` returns what
|
|
79
85
|
* somebody in Brazil would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
80
|
-
* for `topic: "news"` when you want dated journalism
|
|
81
|
-
*
|
|
82
|
-
* its URL to `scrape`.
|
|
86
|
+
* for `topic: "news"` when you want dated journalism or `topic: "images"` when you want
|
|
87
|
+
* pictures, and add `answer: true` when you want a paragraph written over the results instead
|
|
88
|
+
* of writing it yourself. To read a result, pass its URL to `scrape`.
|
|
83
89
|
*
|
|
84
90
|
* Credits: 4 credits per 10 results, so a `limit` above 10 costs two rows, plus 1 for `answer:
|
|
85
91
|
* true`. A search every front refused, and anything we broke, cost nothing; a search that ran
|
|
@@ -95,15 +101,17 @@ export declare abstract class GeneratedOperations {
|
|
|
95
101
|
*
|
|
96
102
|
* Hand us a list of URLs and stop holding a connection open. Every one is read with the
|
|
97
103
|
* options you pass here. It uses the same engines, the same exit and the same meter as a
|
|
98
|
-
* single `scrape`, and a page the site blocked costs nothing here too
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* 24 hours
|
|
104
|
+
* single `scrape`, and a page the site blocked costs nothing here too, unless your own
|
|
105
|
+
* `browser.actions` script ran in the page first. Nothing is charged when you submit: a
|
|
106
|
+
* batch spends as it goes, so use `quote` for the ceiling. Poll `get` for progress, or
|
|
107
|
+
* pass a `webhook` and we will call you instead. Results are readable for 24 hours from
|
|
108
|
+
* when you submit. A batch still running then stops, and the URLs it has not read are
|
|
109
|
+
* `skipped`, free.
|
|
102
110
|
*
|
|
103
111
|
* Credits: Creating a batch is free, and nothing is held: each URL then costs exactly what
|
|
104
112
|
* the same call would have cost on its own, and the pages the web refused cost nothing
|
|
105
|
-
* here too
|
|
106
|
-
* runs.
|
|
113
|
+
* here too, unless your own `browser.actions` script ran in them first. What the batch has
|
|
114
|
+
* spent so far is on the batch itself, and it grows while it runs.
|
|
107
115
|
*
|
|
108
116
|
* `POST /v1/batches`
|
|
109
117
|
*/
|
|
@@ -112,9 +120,11 @@ export declare abstract class GeneratedOperations {
|
|
|
112
120
|
* Price a batch
|
|
113
121
|
*
|
|
114
122
|
* What that submission would cost at most, without submitting it. `maxCost` prices every
|
|
115
|
-
* URL at the ceiling one page could reach
|
|
116
|
-
*
|
|
117
|
-
*
|
|
123
|
+
* URL at the ceiling one page could reach. The real bill comes in lower by every page that
|
|
124
|
+
* never needed a browser and by every outcome that costs nothing: a page the site blocked
|
|
125
|
+
* or our policy refused (unless your own `browser.actions` script ran in it first) and
|
|
126
|
+
* anything we broke. Bandwidth is on top and cannot be quoted, because it depends on what
|
|
127
|
+
* the pages weigh.
|
|
118
128
|
*
|
|
119
129
|
* Credits: 0 credits. Pricing a batch is free.
|
|
120
130
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EACR,QAAQ,EACR,aAAa,EACb,eAAe,EACf,cAAc,EACd,WAAW,EACX,YAAY,EACZ,cAAc,EACd,eAAe,EACf,eAAe,EACf,UAAU,EACV,eAAe,EACf,QAAQ,EACX,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACR,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACf,MAAM,aAAa,CAAC;AAErB,4DAA4D;AAC5D,eAAO,MAAM,WAAW,UAAU,CAAC;AAEnC,sBAAsB;AACtB,eAAO,MAAM,gBAAgB,8BAA8B,CAAC;AAE5D,8BAAsB,mBAAmB;IACrC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACxB,IAAI,EAAE,WAAW,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,CAAC,CAAC;IAEb
|
|
1
|
+
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EACR,QAAQ,EACR,aAAa,EACb,eAAe,EACf,cAAc,EACd,WAAW,EACX,YAAY,EACZ,cAAc,EACd,eAAe,EACf,eAAe,EACf,UAAU,EACV,eAAe,EACf,QAAQ,EACX,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACR,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACf,MAAM,aAAa,CAAC;AAErB,4DAA4D;AAC5D,eAAO,MAAM,WAAW,UAAU,CAAC;AAEnC,sBAAsB;AACtB,eAAO,MAAM,gBAAgB,8BAA8B,CAAC;AAE5D,8BAAsB,mBAAmB;IACrC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACxB,IAAI,EAAE,WAAW,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,CAAC,CAAC;IAEb;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CACF,MAAM,EAAE,YAAY,EACpB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,eAAe,CAAC;IAa3B;;;;;;;;;;;;OAYG;IACH,KAAK,CACD,MAAM,EAAE,iBAAiB,EACzB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,cAAc,CAAC;IAa1B;;;;;;;;;;;;;;;;OAgBG;IACH,GAAG,CACC,MAAM,EAAE,aAAa,EACrB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,YAAY,CAAC;IAaxB;;;;;;;;;;OAUG;IACH,QAAQ,CACJ,MAAM,EAAE,cAAc,EACtB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,WAAW,CAAC;IAavB;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CACF,MAAM,EAAE,YAAY,EACpB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,eAAe,CAAC;IAa3B,QAAQ,CAAC,OAAO;QACZ;;;;;;;;;;;;;;;;;;WAkBG;yBAES,iBAAiB,SAClB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,eAAe,CAAC;QAW3B;;;;;;;;;;;;;WAaG;wBAES,gBAAgB,SACjB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,aAAa,CAAC;QAWzB;;;;;;;;;WASG;wBAEU,iBAAiB,SACnB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB;;;;;;;;;;;;;;WAcG;sBAES,cAAc,SACf,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,CAAC;QACpB;;;;;;;;;;;WAWG;yBAES,iBAAiB,SAClB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,CAAC;MAMtB;IAEF,QAAQ,CAAC,QAAQ;QACb;;;;;;;;;;;;;;;;WAgBG;0BAEU,mBAAmB,SACrB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,cAAc,CAAC;QAW1B;;;;;;;;;;WAUG;yBAEU,kBAAkB,SACpB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,eAAe,CAAC;QAW3B;;;;;;;;;WASG;wBAEU,kBAAkB,SACpB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,EAAE,CAAC;QAExB;;;;;;;;;;WAUG;sBAES,gBAAgB,SACjB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,CAAC;QAEtB;;;;;;;;;;;WAWG;wBAES,kBAAkB,SACnB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,CAAC;MAMxB;IAEF,QAAQ,CAAC,KAAK;QACV;;;;;;;;;;WAUG;qBACU,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAAG,OAAO,CAAC,QAAQ,CAAC;MAEzE;CACL"}
|