@scrapa/sdk 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -17
- package/dist/client.js +2 -2
- package/dist/client.js.map +1 -1
- package/dist/errors.js +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/generated/contract.d.ts +21 -8
- package/dist/generated/contract.d.ts.map +1 -1
- package/dist/generated/contract.js +14 -6
- package/dist/generated/contract.js.map +1 -1
- package/dist/generated/operations.d.ts +45 -43
- package/dist/generated/operations.d.ts.map +1 -1
- package/dist/generated/operations.js +45 -43
- package/dist/generated/operations.js.map +1 -1
- package/dist/generated/params.d.ts +142 -100
- package/dist/generated/params.d.ts.map +1 -1
- package/dist/generated/retry.d.ts +2 -2
- package/dist/generated/retry.js +3 -3
- package/dist/generated/transport.js +2 -2
- package/package.json +2 -2
- package/src/client.ts +2 -2
- package/src/errors.ts +1 -1
- package/src/generated/contract.ts +25 -6
- package/src/generated/operations.ts +45 -43
- package/src/generated/params.ts +142 -100
- package/src/generated/retry.ts +3 -3
- package/src/generated/transport.ts +2 -2
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @scrapa/sdk
|
|
2
2
|
|
|
3
|
-
The official TypeScript client for [scrapa.app](https://scrapa.app)
|
|
3
|
+
The official TypeScript client for [scrapa.app](https://scrapa.app): read any web page as
|
|
4
4
|
markdown, HTML, text, links, a screenshot or a PDF, through a plain HTTP request or a real
|
|
5
5
|
browser, whichever the page turns out to need. Search the web through the same key, and hold
|
|
6
6
|
a real Chromium your own Playwright or Puppeteer drives.
|
|
@@ -20,13 +20,13 @@ console.log(page.markdown);
|
|
|
20
20
|
|
|
21
21
|
Every method, every argument and every field is generated from the API's own definitions, so
|
|
22
22
|
what your editor autocompletes is what the server accepts. Node 20+, Bun, Deno, workers and
|
|
23
|
-
the browser
|
|
23
|
+
the browser. It runs anywhere `fetch` exists.
|
|
24
24
|
|
|
25
25
|
## Reading a page
|
|
26
26
|
|
|
27
27
|
`scrape()` takes a URL and gives you the page. `engine` defaults to `auto`: a plain HTTP
|
|
28
28
|
request first, re-read in a real Chromium only when the body turns out to be a JavaScript
|
|
29
|
-
shell
|
|
29
|
+
shell. That is the difference between 1 credit and 5.
|
|
30
30
|
|
|
31
31
|
```ts
|
|
32
32
|
const page = await scrapa.scrape({
|
|
@@ -34,7 +34,7 @@ const page = await scrapa.scrape({
|
|
|
34
34
|
formats: ["markdown", "links"],
|
|
35
35
|
});
|
|
36
36
|
|
|
37
|
-
page.engine; // "fetch"
|
|
37
|
+
page.engine; // "fetch": this one did not need a browser
|
|
38
38
|
page.links; // every absolute link, in document order
|
|
39
39
|
page.metadata.title;
|
|
40
40
|
```
|
|
@@ -76,7 +76,7 @@ takes the default country and leaves a default city out.
|
|
|
76
76
|
## Searching the web
|
|
77
77
|
|
|
78
78
|
`search()` runs a real search and hands back results we read ourselves. That is not a
|
|
79
|
-
detail
|
|
79
|
+
detail: every search vendor forbids redistributing their results through another API, so
|
|
80
80
|
our own fetch is the only kind you are allowed to build on.
|
|
81
81
|
|
|
82
82
|
```ts
|
|
@@ -86,7 +86,7 @@ const found = await scrapa.search({
|
|
|
86
86
|
proxy: { country: "br" }, // picks the exit AND the market: Brazilian results, in Portuguese
|
|
87
87
|
});
|
|
88
88
|
|
|
89
|
-
found.results[0].url; // array order IS the ranking
|
|
89
|
+
found.results[0].url; // array order IS the ranking: there is no score to sort by, because none is real
|
|
90
90
|
found.cached; // the same question inside the window costs nothing and says so
|
|
91
91
|
```
|
|
92
92
|
|
|
@@ -95,8 +95,8 @@ asked of the front *and* enforced on the answer, so what comes back is on your l
|
|
|
95
95
|
the front did with the operator. `answer: true` adds a paragraph written over the hits. To
|
|
96
96
|
read any result, hand its URL straight to `scrape()`.
|
|
97
97
|
|
|
98
|
-
A search every front refused costs nothing. A search that ran and found nothing is charged
|
|
99
|
-
|
|
98
|
+
A search every front refused costs nothing. A search that ran and found nothing is charged.
|
|
99
|
+
The index answered, and "nothing published that" is an answer.
|
|
100
100
|
|
|
101
101
|
## Driving a real browser
|
|
102
102
|
|
|
@@ -121,8 +121,8 @@ try {
|
|
|
121
121
|
```
|
|
122
122
|
|
|
123
123
|
`create()` returns only once Chromium is actually up, so `connectUrl` is a required field and
|
|
124
|
-
needs no `!`. It carries the session's token, is shown once and cannot be rebuilt from the id
|
|
125
|
-
|
|
124
|
+
needs no `!`. It carries the session's token, is shown once and cannot be rebuilt from the id.
|
|
125
|
+
That is why `get()` does not repeat it.
|
|
126
126
|
|
|
127
127
|
Minutes are reserved for the whole window and settled by what you used, so closing early
|
|
128
128
|
hands the rest back. A browser that never came up costs nothing at all, and if you forget to
|
|
@@ -131,7 +131,7 @@ close one, it is reaped at `expiresAt` and settled the same way.
|
|
|
131
131
|
## Reading a list of pages
|
|
132
132
|
|
|
133
133
|
A batch is N scrapes on our schedule instead of your connection. Same options, same engines,
|
|
134
|
-
same meter
|
|
134
|
+
same meter. What you buy is not holding a socket open for an hour.
|
|
135
135
|
|
|
136
136
|
```ts
|
|
137
137
|
const batch = await scrapa.batches.create({
|
|
@@ -156,7 +156,7 @@ for (let from = 0; from < status.total; ) {
|
|
|
156
156
|
|
|
157
157
|
Nothing is charged when you submit and nothing is reserved: each URL then costs what the same
|
|
158
158
|
call would have cost alone, and `cost` on the batch is what it has spent so far. Because
|
|
159
|
-
nothing is held up front, a batch can outlive the credits that started it
|
|
159
|
+
nothing is held up front, a batch can outlive the credits that started it. Those URLs come
|
|
160
160
|
back `skipped` rather than `failed`, which is not a verdict about the URL, and you can submit
|
|
161
161
|
exactly those positions again.
|
|
162
162
|
|
|
@@ -165,14 +165,14 @@ exactly those positions again.
|
|
|
165
165
|
hundred items per read.
|
|
166
166
|
|
|
167
167
|
Rather not poll? Pass a `webhook` and we will POST when it is done. The signing secret comes
|
|
168
|
-
back once, on the create
|
|
168
|
+
back once, on the create. Verify a delivery with HMAC-SHA256 over `<timestamp>.<raw body>`
|
|
169
169
|
against the `v1=` value in the `scrapa-signature` header, and refuse a `t=` more than five
|
|
170
170
|
minutes old, or a delivery somebody copied can be sent to you again.
|
|
171
171
|
|
|
172
172
|
## What it costs, per call
|
|
173
173
|
|
|
174
174
|
Every metered answer carries what it cost and what is left. Methods hand you the page itself,
|
|
175
|
-
so the meter arrives through a callback
|
|
175
|
+
so the meter arrives through a callback, per call, rather than a mutable field two concurrent
|
|
176
176
|
scrapes would fight over.
|
|
177
177
|
|
|
178
178
|
```ts
|
|
@@ -200,11 +200,11 @@ try {
|
|
|
200
200
|
} catch (error) {
|
|
201
201
|
if (!(error instanceof ScrapaError)) throw error;
|
|
202
202
|
switch (error.outcome) {
|
|
203
|
-
case "blocked": // the site said no
|
|
203
|
+
case "blocked": // the site said no: try another pool, country or engine
|
|
204
204
|
return scrapa.scrape({ url, proxy: { pool: "residential", country: "de" } });
|
|
205
|
-
case "denied": // our policy said no
|
|
205
|
+
case "denied": // our policy said no: retrying cannot help, change the URL
|
|
206
206
|
return null;
|
|
207
|
-
case "broken": // the API said it broke, and refunded the call
|
|
207
|
+
case "broken": // the API said it broke, and refunded the call. Worth one retry
|
|
208
208
|
return scrapa.scrape({ url });
|
|
209
209
|
}
|
|
210
210
|
throw error; // your own mistake (a bad field, a missing key, a spent quota), or error.mayHaveRun
|
package/dist/client.js
CHANGED
|
@@ -53,9 +53,9 @@ export class Scrapa extends GeneratedOperations {
|
|
|
53
53
|
super();
|
|
54
54
|
const opts = typeof options === "string" ? { apiKey: options } : options;
|
|
55
55
|
if (!opts.apiKey)
|
|
56
|
-
throw new TypeError("scrapa: an API key is required
|
|
56
|
+
throw new TypeError("scrapa: an API key is required. Create one at https://console.scrapa.app/keys");
|
|
57
57
|
if (typeof (opts.fetch ?? globalThis.fetch) !== "function")
|
|
58
|
-
throw new TypeError("scrapa: no global fetch
|
|
58
|
+
throw new TypeError("scrapa: no global fetch. Use Node 20+, or pass `fetch` in the options.");
|
|
59
59
|
const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
|
|
60
60
|
const host = new URL(baseUrl).host;
|
|
61
61
|
const timeout = opts.timeout ?? 65_000;
|
package/dist/client.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,6FAA6F;AAC7F,EAAE;AACF,0FAA0F;AAC1F,2FAA2F;AAC3F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4FAA4F;AAC5F,2FAA2F;AAC3F,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAkC,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC/F,OAAO,EACH,IAAI,GAIP,MAAM,0BAA0B,CAAC;AAMlC;;8BAE8B;AAC9B,MAAM,cAAc,GAAG,aAAa,CAAC;AAgDrC;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC7B,MAAc,EACd,QAAmC,EACnC,OAAsC;IAEtC,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC;IACzC,MAAM,GAAG,GACL,OAAO,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;QAC1E,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE;QACrB,CAAC,CAAC,EAAE,CAAC;IACb,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/C,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAC1E,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC;QACnB,OAAO,MAAM,CAAC,MAAM,CAAC;IACzB,CAAC;IACD,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC,KAAK,CAAC;QACpB,OAAO,MAAM,CAAC,IAAI,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;IACpC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACrE,CAAC;AAED,MAAM,OAAO,MAAO,SAAQ,mBAAmB;IAC1B,SAAS,CAAY;IACrB,aAAa,CAA4B;IAE1D,YAAY,OAA+B;QACvC,KAAK,EAAE,CAAC;QACR,MAAM,IAAI,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QACzE,IAAI,CAAC,IAAI,CAAC,MAAM;YACZ,MAAM,IAAI,SAAS,CACf,
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,8FAA8F;AAC9F,6FAA6F;AAC7F,EAAE;AACF,0FAA0F;AAC1F,2FAA2F;AAC3F,4FAA4F;AAC5F,8FAA8F;AAC9F,8FAA8F;AAC9F,4FAA4F;AAC5F,2FAA2F;AAC3F,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAkC,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC/F,OAAO,EACH,IAAI,GAIP,MAAM,0BAA0B,CAAC;AAMlC;;8BAE8B;AAC9B,MAAM,cAAc,GAAG,aAAa,CAAC;AAgDrC;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC7B,MAAc,EACd,QAAmC,EACnC,OAAsC;IAEtC,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC;IACzC,MAAM,GAAG,GACL,OAAO,IAAI,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI;QAC1E,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE;QACrB,CAAC,CAAC,EAAE,CAAC;IACb,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/C,IAAI,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAC1E,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACrD,OAAO,MAAM,CAAC,IAAI,CAAC;QACnB,OAAO,MAAM,CAAC,MAAM,CAAC;IACzB,CAAC;IACD,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC,KAAK,CAAC;QACpB,OAAO,MAAM,CAAC,IAAI,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;IACpC,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACrE,CAAC;AAED,MAAM,OAAO,MAAO,SAAQ,mBAAmB;IAC1B,SAAS,CAAY;IACrB,aAAa,CAA4B;IAE1D,YAAY,OAA+B;QACvC,KAAK,EAAE,CAAC;QACR,MAAM,IAAI,GAAG,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QACzE,IAAI,CAAC,IAAI,CAAC,MAAM;YACZ,MAAM,IAAI,SAAS,CACf,+EAA+E,CAClF,CAAC;QACN,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,KAAK,UAAU;YACtD,MAAM,IAAI,SAAS,CACf,wEAAwE,CAC3E,CAAC;QACN,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC;QACjD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,cAAc;YACrE,MAAM,IAAI,UAAU,CAChB,yEAAyE,cAAc,GAAG,CAC7F,CAAC;QACN,IAAI,CAAC,SAAS,GAAG;YACb,OAAO;YACP,OAAO,EAAE;gBACL,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;gBACtC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,aAAa,WAAW,EAAE;aAC/C;YACD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,SAAS,EAAE,GAAG,EAAE,CAAC,OAAO;YACxB,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC;YAChC,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC;SACjD,CAAC;QACF,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC;IAC9C,CAAC;IAES,KAAK,CAAC,OAAO,CACnB,IAAiB,EACjB,SAAiB,EAAE,EACnB,OAAuB,EAAE;QAEzB,MAAM,YAAY,GAAG,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/E,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAe,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;QAC1F,IAAI,IAAI,KAAK,SAAS;YAAE,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}
|
package/dist/errors.js
CHANGED
|
@@ -94,7 +94,7 @@ export function scrapaError(failure, host) {
|
|
|
94
94
|
status: 0,
|
|
95
95
|
mayHaveRun: method !== "GET",
|
|
96
96
|
message: failure.timedOut
|
|
97
|
-
? `scrapa: no answer within ${failure.timeoutMs}ms
|
|
97
|
+
? `scrapa: no answer within ${failure.timeoutMs}ms. The page may still have been read and charged.`
|
|
98
98
|
: `scrapa: could not reach ${host}.`,
|
|
99
99
|
cause: failure.cause,
|
|
100
100
|
});
|
package/dist/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,EAAE;AACF,wFAAwF;AACxF,2FAA2F;AAC3F,qFAAqF;AACrF,yFAAyF;AACzF,2FAA2F;AAC3F,OAAO,EAAE,OAAO,EAAE,UAAU,EAAkB,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA6BtD,MAAM,OAAO,WAAY,SAAQ,KAAK;IACzB,IAAI,CAA8B;IAClC,MAAM,CAAS;IACf,UAAU,CAAU;IACpB,MAAM,CAAmC;IACzC,OAAO,CAAW;IAClB,UAAU,CAAU;IACpB,SAAS,CAAU;IAC5B;;;;;OAKG;IACM,UAAU,CAAU;IACpB,IAAI,CAAU;IACd,gBAAgB,CAAU;IAEnC,YAAY,IAAqB;QAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClF,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;IAClD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,IAAI,OAAO;QACP,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAiB,CAAC,CAAC;IAC5E,CAAC;IAED;;uEAEmE;IACnE,IAAI,SAAS;QACT,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,KAAK,CAAC;QAClC,OAAO,CACH,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,IAAI,CAAC,IAAI,KAAK,SAAS;YACvB,IAAI,CAAC,IAAI,KAAK,qBAAqB;YACnC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAC5B,CAAC;IACN,CAAC;CACJ;AAED,SAAS,YAAY,CAAC,OAA4B,EAAE,IAAY;IAC5D,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACxD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,oEAAoE;AACpE,SAAS,UAAU,CAAC,MAAc;IAC9B,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,qBAAqB,CAAC;IACjD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,WAAW,CAAC;IACvC,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,cAAc,CAAC;IAC1C,OAAO,gBAAgB,CAAC;AAC5B,CAAC;AAED;;4CAE4C;AAC5C,MAAM,UAAU,WAAW,CAAC,OAAgB,EAAE,IAAY;IACtD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC1D,IAAI,MAAM,KAAK,CAAC;QACZ,OAAO,IAAI,WAAW,CAAC;YACnB,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAA2B;YAChF,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,MAAM,KAAK,KAAK;YAC5B,OAAO,EAAE,OAAO,CAAC,QAAQ;gBACrB,CAAC,CAAC,4BAA4B,OAAO,CAAC,SAAS,
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,EAAE;AACF,wFAAwF;AACxF,2FAA2F;AAC3F,qFAAqF;AACrF,yFAAyF;AACzF,2FAA2F;AAC3F,OAAO,EAAE,OAAO,EAAE,UAAU,EAAkB,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA6BtD,MAAM,OAAO,WAAY,SAAQ,KAAK;IACzB,IAAI,CAA8B;IAClC,MAAM,CAAS;IACf,UAAU,CAAU;IACpB,MAAM,CAAmC;IACzC,OAAO,CAAW;IAClB,UAAU,CAAU;IACpB,SAAS,CAAU;IAC5B;;;;;OAKG;IACM,UAAU,CAAU;IACpB,IAAI,CAAU;IACd,gBAAgB,CAAU;IAEnC,YAAY,IAAqB;QAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClF,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;IAClD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,IAAI,OAAO;QACP,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAiB,CAAC,CAAC;IAC5E,CAAC;IAED;;uEAEmE;IACnE,IAAI,SAAS;QACT,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,KAAK,CAAC;QAClC,OAAO,CACH,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,IAAI,CAAC,IAAI,KAAK,SAAS;YACvB,IAAI,CAAC,IAAI,KAAK,qBAAqB;YACnC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAC5B,CAAC;IACN,CAAC;CACJ;AAED,SAAS,YAAY,CAAC,OAA4B,EAAE,IAAY;IAC5D,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACxD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,oEAAoE;AACpE,SAAS,UAAU,CAAC,MAAc;IAC9B,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,qBAAqB,CAAC;IACjD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,WAAW,CAAC;IACvC,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,cAAc,CAAC;IAC1C,OAAO,gBAAgB,CAAC;AAC5B,CAAC;AAED;;4CAE4C;AAC5C,MAAM,UAAU,WAAW,CAAC,OAAgB,EAAE,IAAY;IACtD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC1D,IAAI,MAAM,KAAK,CAAC;QACZ,OAAO,IAAI,WAAW,CAAC;YACnB,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAA2B;YAChF,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,MAAM,KAAK,KAAK;YAC5B,OAAO,EAAE,OAAO,CAAC,QAAQ;gBACrB,CAAC,CAAC,4BAA4B,OAAO,CAAC,SAAS,oDAAoD;gBACnG,CAAC,CAAC,2BAA2B,IAAI,GAAG;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK;SACvB,CAAC,CAAC;IACP,sFAAsF;IACtF,0FAA0F;IAC1F,wFAAwF;IACxF,uEAAuE;IACvE,IAAI,MAAM,GAAG,GAAG;QACZ,OAAO,IAAI,WAAW,CAAC;YACnB,IAAI,EAAE,kBAA4C;YAClD,MAAM;YACN,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,WAAW,MAAM,IAAI,IAAI,aAAa,MAAM,kHAAkH;YACvK,OAAO,EAAE,OAAO,CAAC,IAAI;YACrB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;YACjD,gBAAgB,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC;SAC5E,CAAC,CAAC;IACP,yFAAyF;IACzF,sFAAsF;IACtF,mFAAmF;IACnF,MAAM,UAAU,GAAG,CAAC,QAAQ,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,KAAK,KAAK,CAAC;IAClE,OAAO,IAAI,WAAW,CAAC;QACnB,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAE,QAAQ,CAAC,IAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAClE,MAAM;QACN,UAAU;QACV,mFAAmF;QACnF,gCAAgC;QAChC,OAAO,EACH,QAAQ,EAAE,OAAO;YACjB,WAAW,MAAM,IAAI,IAAI,gBAAgB,MAAM,IAAI,UAAU,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9J,UAAU,EAAE,QAAQ,EAAE,UAAU;QAChC,MAAM,EAAE,QAAQ,EAAE,MAAM;QACxB,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI;QACnD,sEAAsE;QACtE,UAAU,EACN,cAAc,CAAC;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,OAAO,EAAE,QAAQ,EAAE,OAAO;SAC7B,CAAC,IAAI,SAAS;QACnB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,uFAAuF;QACvF,mFAAmF;QACnF,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;QACjD,gBAAgB,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC;KAC5E,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* `AUP_DENIED` are "our policy said no" (retrying cannot help, change the URL); a 5xx is
|
|
10
10
|
* "we broke it" (retry as-is, already refunded).
|
|
11
11
|
*/
|
|
12
|
-
export declare const ERROR_CODES: readonly ["VALIDATION_ERROR", "INVALID_URL", "SSRF_DENIED", "UNAUTHORIZED", "INVALID_TOKEN", "PLAN_GATE", "FORBIDDEN", "AUP_DENIED", "NOT_FOUND", "SESSION_NOT_FOUND", "CONFLICT", "PAYLOAD_TOO_LARGE", "UPSTREAM_BLOCKED", "NAV_TIMEOUT", "PROXY_REFUSED", "RATE_LIMIT_EXCEEDED", "QUOTA_EXCEEDED", "INTERNAL_ERROR", "SERVICE_UNAVAILABLE", "BILLING_UNAVAILABLE"];
|
|
12
|
+
export declare const ERROR_CODES: readonly ["VALIDATION_ERROR", "INVALID_URL", "SSRF_DENIED", "UNAUTHORIZED", "INVALID_TOKEN", "PLAN_GATE", "FORBIDDEN", "AUP_DENIED", "NOT_FOUND", "SESSION_NOT_FOUND", "CONFLICT", "PAYLOAD_TOO_LARGE", "UPSTREAM_BLOCKED", "NAV_TIMEOUT", "PROXY_REFUSED", "FILE_TOO_LARGE", "RATE_LIMIT_EXCEEDED", "QUOTA_EXCEEDED", "INTERNAL_ERROR", "SERVICE_UNAVAILABLE", "BILLING_UNAVAILABLE"];
|
|
13
13
|
export type ErrorCode = (typeof ERROR_CODES)[number];
|
|
14
14
|
/** The status each code answers with. The server's error factory, the SDK's fallback and
|
|
15
15
|
* the docs' errors table all read this one map — three copies could drift, one cannot. */
|
|
@@ -29,6 +29,7 @@ export declare const ERROR_STATUS: {
|
|
|
29
29
|
readonly UPSTREAM_BLOCKED: 422;
|
|
30
30
|
readonly NAV_TIMEOUT: 422;
|
|
31
31
|
readonly PROXY_REFUSED: 422;
|
|
32
|
+
readonly FILE_TOO_LARGE: 422;
|
|
32
33
|
readonly RATE_LIMIT_EXCEEDED: 429;
|
|
33
34
|
readonly QUOTA_EXCEEDED: 429;
|
|
34
35
|
readonly INTERNAL_ERROR: 500;
|
|
@@ -36,10 +37,11 @@ export declare const ERROR_STATUS: {
|
|
|
36
37
|
readonly BILLING_UNAVAILABLE: 503;
|
|
37
38
|
};
|
|
38
39
|
/**
|
|
39
|
-
* The three no-charge outcomes, as DESIGN.md names them.
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
40
|
+
* The three no-charge outcomes, as DESIGN.md names them. Each code here maps to exactly one
|
|
41
|
+
* — this is the map the console reads to pick the hue and the word, and the SDK reads to
|
|
42
|
+
* tell a caller what to do next. A code absent from here gets no colour: the caller's own
|
|
43
|
+
* mistake (a bad body, a missing key), or a free refusal whose fix is in the request rather
|
|
44
|
+
* than the site (PROXY_REFUSED, FILE_TOO_LARGE).
|
|
43
45
|
*
|
|
44
46
|
* blocked → the site said no → try another engine, pool or country
|
|
45
47
|
* denied → our policy said no → change the URL; retrying cannot help
|
|
@@ -115,8 +117,9 @@ export type ResolvedEngine = Exclude<Engine, "auto">;
|
|
|
115
117
|
/** What a scrape can hand back. `markdown` is the default because it is what an LLM, a
|
|
116
118
|
* notebook and a human all read; `html` is the page as fetched; `text` is prose with the
|
|
117
119
|
* markup gone; `links` every absolute anchor in document order; `extract` is the page read
|
|
118
|
-
* into the fields the caller asked for, which is the one format a model writes
|
|
119
|
-
|
|
120
|
+
* into the fields the caller asked for, which is the one format a model writes; `raw` is the
|
|
121
|
+
* response's own bytes, the file a browser would save. */
|
|
122
|
+
export declare const SCRAPE_FORMATS: readonly ["markdown", "html", "text", "links", "screenshot", "pdf", "extract", "raw"];
|
|
120
123
|
export type ScrapeFormat = (typeof SCRAPE_FORMATS)[number];
|
|
121
124
|
/** A session's life. `starting` is the driver bringing Chromium up; `live` is the only
|
|
122
125
|
* status whose `connectUrl` accepts a CDP client. The three end states answer one
|
|
@@ -131,7 +134,7 @@ export declare const SEARCH_TOPICS: readonly ["general", "news"];
|
|
|
131
134
|
export type SearchTopic = (typeof SEARCH_TOPICS)[number];
|
|
132
135
|
/** Which front answered a search. Not a caller's choice — the ladder picks, and says which
|
|
133
136
|
* one it was, because a result set's flavour follows its index. */
|
|
134
|
-
export declare const SEARCH_ENGINES: readonly ["bing", "brave"];
|
|
137
|
+
export declare const SEARCH_ENGINES: readonly ["bing", "brave", "duckduckgo"];
|
|
135
138
|
export type SearchEngine = (typeof SEARCH_ENGINES)[number];
|
|
136
139
|
/** A batch's life. `queued` is accepted but not yet picked up; `running` has at least one
|
|
137
140
|
* item in flight. The three end states answer the same question the session statuses do —
|
|
@@ -184,6 +187,13 @@ export interface ScrapeMetadataDto {
|
|
|
184
187
|
* Always false from the browser engine, which ran the scripts. */
|
|
185
188
|
needsJs: boolean;
|
|
186
189
|
}
|
|
190
|
+
/** A file, exactly as the site sent it. */
|
|
191
|
+
export interface ScrapeRawDto {
|
|
192
|
+
/** The bytes, base64-encoded. */
|
|
193
|
+
base64: string;
|
|
194
|
+
/** The response's `content-type`, media type only: `application/pdf`. */
|
|
195
|
+
contentType: string | null;
|
|
196
|
+
}
|
|
187
197
|
/** `POST /v1/scrape`'s answer. Only the requested formats are present; everything else is
|
|
188
198
|
* always there. Artifacts (`screenshot`, `pdf`) are base64 in v1. */
|
|
189
199
|
export interface ScrapeResultDto {
|
|
@@ -214,6 +224,9 @@ export interface ScrapeResultDto {
|
|
|
214
224
|
links?: string[];
|
|
215
225
|
screenshot?: string;
|
|
216
226
|
pdf?: string;
|
|
227
|
+
/** The response's own bytes, as a browser would save them (any compression undone), when
|
|
228
|
+
* `raw` was in `formats`. Never cut short: a file over the limit is refused instead. */
|
|
229
|
+
raw?: ScrapeRawDto;
|
|
217
230
|
/** The fields asked for in `extract`, read off the page by a model. Null when the model
|
|
218
231
|
* could not answer — that credit is handed back and the page still stands. Present only
|
|
219
232
|
* when `extract` was in `formats`. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../src/generated/contract.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;GAUG;AACH,eAAO,MAAM,WAAW,
|
|
1
|
+
{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../src/generated/contract.ts"],"names":[],"mappings":"AAOA;;;;;;;;;;GAUG;AACH,eAAO,MAAM,WAAW,wXAqCd,CAAC;AAEX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD;2FAC2F;AAC3F,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;CAsBqB,CAAC;AAE/C;;;;;;;;;;GAUG;AACH,eAAO,MAAM,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAUlF,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,OAAO;IAChB,uEAAuE;;IAEvE,+EAA+E;;;IAG/E,sEAAsE;;IAEtE,yCAAyC;;;CAGnC,CAAC;AAEX,MAAM,WAAW,cAAc;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;CACpB;AAED;;kEAEkE;AAClE,MAAM,WAAW,SAAS;IACtB,+EAA+E;IAC/E,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,IAAI,CAAC;CACjB;AAED,oEAAoE;AACpE,MAAM,WAAW,UAAU,CAAC,CAAC;IACzB,IAAI,EAAE,CAAC,CAAC;IACR,IAAI,CAAC,EAAE,cAAc,GAAG,SAAS,CAAC;CACrC;AAED,uFAAuF;AACvF,MAAM,WAAW,QAAQ;IACrB,KAAK,EAAE;QACH,IAAI,EAAE,SAAS,CAAC;QAChB,yFAAyF;QACzF,OAAO,EAAE,MAAM,CAAC;QAChB,gFAAgF;QAChF,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,6CAA6C;QAC7C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;QACzC,OAAO,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;CACL;AAGD;;;wDAGwD;AACxD,eAAO,MAAM,WAAW,oCAAqC,CAAC;AAE9D,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC;AAErD;iFACiF;AACjF,eAAO,MAAM,OAAO,uCAAwC,CAAC;AAE7D,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9C,4FAA4F;AAC5F,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAErD;;;;2DAI2D;AAC3D,eAAO,MAAM,cAAc,uFASjB,CAAC;AAEX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D;;;;4EAI4E;AAC5E,eAAO,MAAM,gBAAgB,8DAA+D,CAAC;AAE7F,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9D;qFACqF;AACrF,eAAO,MAAM,aAAa,8BAA+B,CAAC;AAE1D,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAEzD;oEACoE;AACpE,eAAO,MAAM,cAAc,0CAA2C,CAAC;AAEvE,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D;;;;mEAImE;AACnE,eAAO,MAAM,cAAc,mEAAoE,CAAC;AAEhG,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE1D;;;;;eAKe;AACf,eAAO,MAAM,mBAAmB,6DAA8D,CAAC;AAE/F,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnE;;;;;;8BAM8B;AAC9B,eAAO,MAAM,cAAc,sDAAuD,CAAC;AAEnF,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D;;;kBAGkB;AAClB,eAAO,MAAM,yBAAyB,6CAA8C,CAAC;AAErF,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,yBAAyB,CAAC,CAAC,MAAM,CAAC,CAAC;AAG/E,MAAM,MAAM,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC;AAGvE;qEACqE;AACrE,MAAM,WAAW,iBAAiB;IAC9B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,sDAAsD;IACtD,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B;;;uEAGmE;IACnE,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,2CAA2C;AAC3C,MAAM,WAAW,YAAY;IACzB,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED;sEACsE;AACtE,MAAM,WAAW,eAAe;IAC5B,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,EAAE,SAAS,CAAC;IAChB;;;mDAG+C;IAC/C,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;gDAE4C;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;8FAE0F;IAC1F,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;6FACyF;IACzF,GAAG,CAAC,EAAE,YAAY,CAAC;IACnB;;2CAEuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACzC,QAAQ,EAAE,iBAAiB,CAAC;CAC/B;AAED,0EAA0E;AAC1E,MAAM,WAAW,cAAc;IAC3B,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,EAAE,SAAS,CAAC;IAChB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,mFAAmF;IACnF,IAAI,EAAE,MAAM,CAAC;IACb;sFACkF;IAClF,OAAO,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;CACjB;AAGD;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,WAAW,CAAC;IACpB,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd;gDAC4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB;;uEAEmE;IACnE,MAAM,EAAE,MAAM,CAAC;IACf;;;mBAGe;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,8EAA8E;IAC9E,IAAI,EAAE,SAAS,CAAC;IAChB,wDAAwD;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;8DAG0D;IAC1D,SAAS,EAAE,MAAM,CAAC;IAClB;+DAC2D;IAC3D,KAAK,CAAC,EAAE,YAAY,EAAE,CAAC;IACvB,oFAAoF;IACpF,OAAO,CAAC,EAAE,eAAe,CAAC;CAC7B;AAED;uDACuD;AACvD,MAAM,WAAW,YAAY;IACzB;0EACsE;IACtE,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,eAAe,CAAC;IACxB,oEAAoE;IACpE,MAAM,CAAC,EAAE,eAAe,CAAC;IACzB;0DACsD;IACtD,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,SAAS,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,8EAA8E;IAC9E,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,6CAA6C;AAC7C,MAAM,WAAW,eAAe;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB;uEACmE;IACnE,MAAM,EAAE,qBAAqB,CAAC;IAC9B;8EAC0E;IAC1E,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;yFAEqF;IACrF,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;yCAEyC;AACzC,MAAM,WAAW,aAAa;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,SAAS,CAAC;IAChB;0CACsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB;gCAC4B;IAC5B,OAAO,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,eAAgB,SAAQ,QAAQ;IAC7C;;;6DAGyD;IACzD,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAGD;;mCAEmC;AACnC,MAAM,WAAW,YAAY;IACzB,uCAAuC;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd;uDACmD;IACnD,OAAO,EAAE,MAAM,CAAC;IAChB;sEACkE;IAClE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,kCAAkC;AAClC,MAAM,WAAW,eAAe;IAC5B,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,WAAW,CAAC;IACnB,4BAA4B;IAC5B,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,SAAS,CAAC;IAChB;yEACqE;IACrE,KAAK,EAAE,MAAM,CAAC;IACd;sCACkC;IAClC,MAAM,EAAE,OAAO,CAAC;IAChB;4FACwF;IACxF,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,YAAY,EAAE,CAAC;CAC3B;AAGD;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,aAAa,CAAC;IACtB;;;kEAG8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;4FACwF;IACxF,IAAI,EAAE,SAAS,CAAC;IAChB,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC;IACnB,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAC;IAClB,oFAAoF;IACpF,eAAe,EAAE,MAAM,CAAC;IACxB;uFACmF;IACnF,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kFAAkF;IAClF,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAe,SAAQ,UAAU;IAC9C,UAAU,EAAE,MAAM,CAAC;CACtB;AAED;;8CAE8C;AAC9C,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,SAAS,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,+BAA+B;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,yFAAyF;IACzF,OAAO,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,MAAM,EAAE,MAAM,CAAC;IACf,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;CACjB;AAGD;;kCAEkC;AAClC,MAAM,WAAW,WAAW;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,2DAA2D;IAC3D,OAAO,EAAE,MAAM,CAAC;IAChB;mEAC+D;IAC/D,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,sFAAsF;AACtF,MAAM,WAAW,QAAQ;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAC;IACpB,gCAAgC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB;+FAC2F;IAC3F,aAAa,EAAE,MAAM,CAAC;IACtB,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,uFAAuF;IACvF,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,WAAW,EAAE,CAAC;CACvB"}
|
|
@@ -42,6 +42,10 @@ export const ERROR_CODES = [
|
|
|
42
42
|
// or did not answer, or the managed pool has no exit in the place you asked for. Free,
|
|
43
43
|
// and no colour — the fix is in the request (the proxy, the credentials, a wider place).
|
|
44
44
|
"PROXY_REFUSED", // 422 — details {reason: "no_exit" | "refused" | "unreachable", place?, status?}
|
|
45
|
+
// The site sent a file bigger than `raw` hands back whole. Not 413, which tells an HTTP
|
|
46
|
+
// client its OWN request was too big. Free, and no colour, for PROXY_REFUSED's reason:
|
|
47
|
+
// the fix is in the request (another format, or download it yourself), not another road.
|
|
48
|
+
"FILE_TOO_LARGE", // 422 — details {limit}, in bytes
|
|
45
49
|
"RATE_LIMIT_EXCEEDED", // 429 — the per-minute burst bucket; Retry-After + details {limit}
|
|
46
50
|
"QUOTA_EXCEEDED", // 429 — a durable bound that WAITING clears; details {scope, limit, resetAt}
|
|
47
51
|
"INTERNAL_ERROR", // 500 — we broke it; the body is always generic, the cause stays server-side
|
|
@@ -66,6 +70,7 @@ export const ERROR_STATUS = {
|
|
|
66
70
|
UPSTREAM_BLOCKED: 422,
|
|
67
71
|
NAV_TIMEOUT: 422,
|
|
68
72
|
PROXY_REFUSED: 422,
|
|
73
|
+
FILE_TOO_LARGE: 422,
|
|
69
74
|
RATE_LIMIT_EXCEEDED: 429,
|
|
70
75
|
QUOTA_EXCEEDED: 429,
|
|
71
76
|
INTERNAL_ERROR: 500,
|
|
@@ -73,10 +78,11 @@ export const ERROR_STATUS = {
|
|
|
73
78
|
BILLING_UNAVAILABLE: 503,
|
|
74
79
|
};
|
|
75
80
|
/**
|
|
76
|
-
* The three no-charge outcomes, as DESIGN.md names them.
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
81
|
+
* The three no-charge outcomes, as DESIGN.md names them. Each code here maps to exactly one
|
|
82
|
+
* — this is the map the console reads to pick the hue and the word, and the SDK reads to
|
|
83
|
+
* tell a caller what to do next. A code absent from here gets no colour: the caller's own
|
|
84
|
+
* mistake (a bad body, a missing key), or a free refusal whose fix is in the request rather
|
|
85
|
+
* than the site (PROXY_REFUSED, FILE_TOO_LARGE).
|
|
80
86
|
*
|
|
81
87
|
* blocked → the site said no → try another engine, pool or country
|
|
82
88
|
* denied → our policy said no → change the URL; retrying cannot help
|
|
@@ -125,7 +131,8 @@ export const ENGINES = ["auto", "fetch", "browser"];
|
|
|
125
131
|
/** What a scrape can hand back. `markdown` is the default because it is what an LLM, a
|
|
126
132
|
* notebook and a human all read; `html` is the page as fetched; `text` is prose with the
|
|
127
133
|
* markup gone; `links` every absolute anchor in document order; `extract` is the page read
|
|
128
|
-
* into the fields the caller asked for, which is the one format a model writes
|
|
134
|
+
* into the fields the caller asked for, which is the one format a model writes; `raw` is the
|
|
135
|
+
* response's own bytes, the file a browser would save. */
|
|
129
136
|
export const SCRAPE_FORMATS = [
|
|
130
137
|
"markdown",
|
|
131
138
|
"html",
|
|
@@ -134,6 +141,7 @@ export const SCRAPE_FORMATS = [
|
|
|
134
141
|
"screenshot",
|
|
135
142
|
"pdf",
|
|
136
143
|
"extract",
|
|
144
|
+
"raw",
|
|
137
145
|
];
|
|
138
146
|
/** A session's life. `starting` is the driver bringing Chromium up; `live` is the only
|
|
139
147
|
* status whose `connectUrl` accepts a CDP client. The three end states answer one
|
|
@@ -146,7 +154,7 @@ export const SESSION_STATUSES = ["starting", "live", "closed", "expired", "faile
|
|
|
146
154
|
export const SEARCH_TOPICS = ["general", "news"];
|
|
147
155
|
/** Which front answered a search. Not a caller's choice — the ladder picks, and says which
|
|
148
156
|
* one it was, because a result set's flavour follows its index. */
|
|
149
|
-
export const SEARCH_ENGINES = ["bing", "brave"];
|
|
157
|
+
export const SEARCH_ENGINES = ["bing", "brave", "duckduckgo"];
|
|
150
158
|
/** A batch's life. `queued` is accepted but not yet picked up; `running` has at least one
|
|
151
159
|
* item in flight. The three end states answer the same question the session statuses do —
|
|
152
160
|
* who ended it: `completed` ran every item to a verdict (some of which may be failures,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contract.js","sourceRoot":"","sources":["../../src/generated/contract.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,gXAAgX;AAChX,EAAE;AACF,oEAAoE;AAEpE,wEAAwE;AAExE;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACvB,kBAAkB,EAAE,iFAAiF;IACrG,aAAa,EAAE,+DAA+D;IAC9E,aAAa,EAAE,sFAAsF;IACrG,cAAc,EAAE,gCAAgC;IAChD,eAAe,EAAE,6EAA6E;IAC9F,oFAAoF;IACpF,mFAAmF;IACnF,qFAAqF;IACrF,uFAAuF;IACvF,qFAAqF;IACrF,WAAW,EAAE,2EAA2E;IACxF,WAAW,EAAE,uCAAuC;IACpD,YAAY,EAAE,yEAAyE;IACvF,WAAW,EAAE,uEAAuE;IACpF,mBAAmB,EAAE,0DAA0D;IAC/E,UAAU,EAAE,uCAAuC;IACnD,mBAAmB,EAAE,qDAAqD;IAC1E,sFAAsF;IACtF,yFAAyF;IACzF,wFAAwF;IACxF,oFAAoF;IACpF,kBAAkB,EAAE,gEAAgE;IACpF,aAAa,EAAE,gEAAgE;IAC/E,wFAAwF;IACxF,uFAAuF;IACvF,yFAAyF;IACzF,eAAe,EAAE,iFAAiF;IAClG,qBAAqB,EAAE,mEAAmE;IAC1F,gBAAgB,EAAE,6EAA6E;IAC/F,gBAAgB,EAAE,6EAA6E;IAC/F,qBAAqB,EAAE,wFAAwF;IAC/G,qBAAqB,EAAE,uFAAuF;CACxG,CAAC;AAIX;2FAC2F;AAC3F,MAAM,CAAC,MAAM,YAAY,GAAG;IACxB,gBAAgB,EAAE,GAAG;IACrB,WAAW,EAAE,GAAG;IAChB,WAAW,EAAE,GAAG;IAChB,YAAY,EAAE,GAAG;IACjB,aAAa,EAAE,GAAG;IAClB,SAAS,EAAE,GAAG;IACd,SAAS,EAAE,GAAG;IACd,UAAU,EAAE,GAAG;IACf,SAAS,EAAE,GAAG;IACd,iBAAiB,EAAE,GAAG;IACtB,QAAQ,EAAE,GAAG;IACb,iBAAiB,EAAE,GAAG;IACtB,gBAAgB,EAAE,GAAG;IACrB,WAAW,EAAE,GAAG;IAChB,aAAa,EAAE,GAAG;IAClB,mBAAmB,EAAE,GAAG;IACxB,cAAc,EAAE,GAAG;IACnB,cAAc,EAAE,GAAG;IACnB,mBAAmB,EAAE,GAAG;IACxB,mBAAmB,EAAE,GAAG;CACkB,CAAC;AAE/C
|
|
1
|
+
{"version":3,"file":"contract.js","sourceRoot":"","sources":["../../src/generated/contract.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,gXAAgX;AAChX,EAAE;AACF,oEAAoE;AAEpE,wEAAwE;AAExE;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACvB,kBAAkB,EAAE,iFAAiF;IACrG,aAAa,EAAE,+DAA+D;IAC9E,aAAa,EAAE,sFAAsF;IACrG,cAAc,EAAE,gCAAgC;IAChD,eAAe,EAAE,6EAA6E;IAC9F,oFAAoF;IACpF,mFAAmF;IACnF,qFAAqF;IACrF,uFAAuF;IACvF,qFAAqF;IACrF,WAAW,EAAE,2EAA2E;IACxF,WAAW,EAAE,uCAAuC;IACpD,YAAY,EAAE,yEAAyE;IACvF,WAAW,EAAE,uEAAuE;IACpF,mBAAmB,EAAE,0DAA0D;IAC/E,UAAU,EAAE,uCAAuC;IACnD,mBAAmB,EAAE,qDAAqD;IAC1E,sFAAsF;IACtF,yFAAyF;IACzF,wFAAwF;IACxF,oFAAoF;IACpF,kBAAkB,EAAE,gEAAgE;IACpF,aAAa,EAAE,gEAAgE;IAC/E,wFAAwF;IACxF,uFAAuF;IACvF,yFAAyF;IACzF,eAAe,EAAE,iFAAiF;IAClG,wFAAwF;IACxF,uFAAuF;IACvF,yFAAyF;IACzF,gBAAgB,EAAE,kCAAkC;IACpD,qBAAqB,EAAE,mEAAmE;IAC1F,gBAAgB,EAAE,6EAA6E;IAC/F,gBAAgB,EAAE,6EAA6E;IAC/F,qBAAqB,EAAE,wFAAwF;IAC/G,qBAAqB,EAAE,uFAAuF;CACxG,CAAC;AAIX;2FAC2F;AAC3F,MAAM,CAAC,MAAM,YAAY,GAAG;IACxB,gBAAgB,EAAE,GAAG;IACrB,WAAW,EAAE,GAAG;IAChB,WAAW,EAAE,GAAG;IAChB,YAAY,EAAE,GAAG;IACjB,aAAa,EAAE,GAAG;IAClB,SAAS,EAAE,GAAG;IACd,SAAS,EAAE,GAAG;IACd,UAAU,EAAE,GAAG;IACf,SAAS,EAAE,GAAG;IACd,iBAAiB,EAAE,GAAG;IACtB,QAAQ,EAAE,GAAG;IACb,iBAAiB,EAAE,GAAG;IACtB,gBAAgB,EAAE,GAAG;IACrB,WAAW,EAAE,GAAG;IAChB,aAAa,EAAE,GAAG;IAClB,cAAc,EAAE,GAAG;IACnB,mBAAmB,EAAE,GAAG;IACxB,cAAc,EAAE,GAAG;IACnB,cAAc,EAAE,GAAG;IACnB,mBAAmB,EAAE,GAAG;IACxB,mBAAmB,EAAE,GAAG;CACkB,CAAC;AAE/C;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,UAAU,GAAgE;IACnF,gBAAgB,EAAE,SAAS;IAC3B,WAAW,EAAE,SAAS;IACtB,WAAW,EAAE,QAAQ;IACrB,UAAU,EAAE,QAAQ;IACpB,cAAc,EAAE,QAAQ;IACxB,mBAAmB,EAAE,QAAQ;IAC7B,qFAAqF;IACrF,uFAAuF;IACvF,mBAAmB,EAAE,QAAQ;CAChC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG;IACnB,uEAAuE;IACvE,IAAI,EAAE,gBAAgB;IACtB,+EAA+E;IAC/E,gBAAgB,EAAE,qBAAqB;IACvC,SAAS,EAAE,cAAc;IACzB,sEAAsE;IACtE,MAAM,EAAE,iBAAiB;IACzB,yCAAyC;IACzC,MAAM,EAAE,iBAAiB;IACzB,UAAU,EAAE,aAAa;CACnB,CAAC;AAuCX,wEAAwE;AAExE;;;wDAGwD;AACxD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,QAAQ,CAAU,CAAC;AAI9D;iFACiF;AACjF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAU,CAAC;AAO7D;;;;2DAI2D;AAC3D,MAAM,CAAC,MAAM,cAAc,GAAG;IAC1B,UAAU;IACV,MAAM;IACN,MAAM;IACN,OAAO;IACP,YAAY;IACZ,KAAK;IACL,SAAS;IACT,KAAK;CACC,CAAC;AAIX;;;;4EAI4E;AAC5E,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAU,CAAC;AAI7F;qFACqF;AACrF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,SAAS,EAAE,MAAM,CAAU,CAAC;AAI1D;oEACoE;AACpE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,YAAY,CAAU,CAAC;AAIvE;;;;mEAImE;AACnE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAU,CAAC;AAIhG;;;;;eAKe;AACf,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAU,CAAC;AAI/F;;;;;;8BAM8B;AAC9B,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,iBAAiB,EAAE,sBAAsB,CAAU,CAAC;AAInF;;;kBAGkB;AAClB,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAU,CAAC"}
|
|
@@ -2,7 +2,7 @@ import type { RequestOptions, RequestSpec } from "../client.js";
|
|
|
2
2
|
import type { BatchDto, BatchQuoteDto, CreatedBatchDto, LiveSessionDto, ScrapeQuoteDto, ScrapeResultDto, SearchResultDto, SessionDto, SessionQuoteDto, UsageDto } from "./contract.js";
|
|
3
3
|
import type { CancelBatchParams, CloseSessionParams, CreateBatchParams, CreateSessionParams, GetBatchParams, GetSessionParams, ListBatchesParams, ListSessionsParams, QuoteBatchParams, QuoteScrapeParams, QuoteSessionParams, ScrapeParams, SearchParams } from "./params.js";
|
|
4
4
|
/** The version this client reports in `x-scrapa-client`. */
|
|
5
|
-
export declare const SDK_VERSION = "0.
|
|
5
|
+
export declare const SDK_VERSION = "0.3.0";
|
|
6
6
|
/** The hosted API. */
|
|
7
7
|
export declare const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
8
8
|
export declare abstract class GeneratedOperations {
|
|
@@ -11,12 +11,13 @@ export declare abstract class GeneratedOperations {
|
|
|
11
11
|
* Scrape a page
|
|
12
12
|
*
|
|
13
13
|
* Read one web page and hand it back as markdown, HTML, plain text, its links, a screenshot, a
|
|
14
|
-
* PDF
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
14
|
+
* PDF, or the fields you ask for, read off the page by a model. `engine: "auto"` (the default)
|
|
15
|
+
* makes a plain HTTP request first and re-reads the page in a real Chromium when the body
|
|
16
|
+
* turns out to be a JavaScript shell, so you pay for a browser only when the page needed one.
|
|
17
|
+
* To send a search form or a JSON query, use `method: "POST"` with a `body`; to download a
|
|
18
|
+
* file exactly as the site sent it, ask for `formats: ["raw"]`. Every request leaves through a
|
|
19
|
+
* residential proxy, or through your own with `proxy: { pool: "custom", server }`. The answer
|
|
20
|
+
* says which engine replied, where the page finally was and what it cost.
|
|
20
21
|
*
|
|
21
22
|
* Credits: 1 credit with the fetch engine, 5 with the browser (a screenshot or a PDF is
|
|
22
23
|
* included, never a surcharge), 5 more when you ask for `extract`, plus managed-proxy
|
|
@@ -30,11 +31,11 @@ export declare abstract class GeneratedOperations {
|
|
|
30
31
|
* Price a scrape
|
|
31
32
|
*
|
|
32
33
|
* What the same request would cost, without making it. Free, and refused by the same guards a
|
|
33
|
-
* real call meets
|
|
34
|
+
* real call meets, so a quote is never a price for a call that would be denied. `maxCost` is
|
|
34
35
|
* the ceiling: for `auto` it is the browser's price, which is only charged if the page turns
|
|
35
36
|
* out to need one.
|
|
36
37
|
*
|
|
37
|
-
* Credits: 0 credits
|
|
38
|
+
* Credits: 0 credits. Pricing a call is free.
|
|
38
39
|
*
|
|
39
40
|
* `POST /v1/scrape/quote`
|
|
40
41
|
*/
|
|
@@ -42,12 +43,13 @@ export declare abstract class GeneratedOperations {
|
|
|
42
43
|
/**
|
|
43
44
|
* Search the web
|
|
44
45
|
*
|
|
45
|
-
* Ask the web a question and get back ranked results
|
|
46
|
-
* holds
|
|
47
|
-
* cache. `country` picks both the exit and the market, so `br` returns what
|
|
48
|
-
* would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
49
|
-
* when you want dated journalism, and add `answer: true` when you want a
|
|
50
|
-
* over the results instead of writing it yourself. To read a result, pass
|
|
46
|
+
* Ask the web a question and get back ranked results: title, URL and the snippet the index
|
|
47
|
+
* holds. They are read live off a real search front through a residential exit, never a
|
|
48
|
+
* reseller's cache. `country` picks both the exit and the market, so `br` returns what
|
|
49
|
+
* somebody in Brazil would see. Narrow to your own list of hosts with `includeDomains`, ask
|
|
50
|
+
* for `topic: "news"` when you want dated journalism, and add `answer: true` when you want a
|
|
51
|
+
* paragraph written over the results instead of writing it yourself. To read a result, pass
|
|
52
|
+
* its URL to `scrape`.
|
|
51
53
|
*
|
|
52
54
|
* Credits: 4 credits per 10 results, so a `limit` above 10 costs two rows, plus 1 for `answer:
|
|
53
55
|
* true`. A search every front refused, and anything we broke, cost nothing; a search that ran
|
|
@@ -62,8 +64,8 @@ export declare abstract class GeneratedOperations {
|
|
|
62
64
|
* Start a batch
|
|
63
65
|
*
|
|
64
66
|
* Hand us a list of URLs and stop holding a connection open. Every one is read with the
|
|
65
|
-
* options you pass here
|
|
66
|
-
* `scrape
|
|
67
|
+
* options you pass here. It uses the same engines, the same exit and the same meter as a
|
|
68
|
+
* single `scrape`, and a page the site blocked costs nothing here too. Nothing is charged
|
|
67
69
|
* when you submit: a batch spends as it goes, so use `quote` for the ceiling. Poll `get`
|
|
68
70
|
* for progress, or pass a `webhook` and we will call you instead. Results are readable for
|
|
69
71
|
* 24 hours.
|
|
@@ -82,9 +84,9 @@ export declare abstract class GeneratedOperations {
|
|
|
82
84
|
* What that submission would cost at most, without submitting it. `maxCost` prices every
|
|
83
85
|
* URL at the ceiling one page could reach, so the real bill comes in lower by whatever the
|
|
84
86
|
* web refuses for free and by every page that never needed a browser. Bandwidth is on top
|
|
85
|
-
* and cannot be quoted
|
|
87
|
+
* and cannot be quoted, because it depends on what the pages weigh.
|
|
86
88
|
*
|
|
87
|
-
* Credits: 0 credits
|
|
89
|
+
* Credits: 0 credits. Pricing a batch is free.
|
|
88
90
|
*
|
|
89
91
|
* `POST /v1/batches/quote`
|
|
90
92
|
*/
|
|
@@ -92,10 +94,10 @@ export declare abstract class GeneratedOperations {
|
|
|
92
94
|
/**
|
|
93
95
|
* List your batches
|
|
94
96
|
*
|
|
95
|
-
* Your most recent batches, newest first
|
|
96
|
-
* what it has cost. Results are not included; ask `get` for one batch's pages.
|
|
97
|
+
* Your most recent batches, newest first, with what each one is doing, how far it has got
|
|
98
|
+
* and what it has cost. Results are not included; ask `get` for one batch's pages.
|
|
97
99
|
*
|
|
98
|
-
* Credits: 0 credits
|
|
100
|
+
* Credits: 0 credits. Listing batches is free.
|
|
99
101
|
*
|
|
100
102
|
* `GET /v1/batches`
|
|
101
103
|
*/
|
|
@@ -105,13 +107,13 @@ export declare abstract class GeneratedOperations {
|
|
|
105
107
|
*
|
|
106
108
|
* Where a batch got to: whether it is still running, how many of its URLs now have a
|
|
107
109
|
* verdict, how many of those the web or our policy refused, how many were never attempted,
|
|
108
|
-
* and what it has spent so far. Ask for `items` to get the results themselves
|
|
110
|
+
* and what it has spent so far. Ask for `items` to get the results themselves: the pages
|
|
109
111
|
* and the refusals, in the order you submitted them, each one the same answer a single
|
|
110
|
-
* `scrape` would have given
|
|
111
|
-
* whose results have expired still answers with everything else, because somebody
|
|
112
|
-
* for those pages must never be told the batch does not exist.
|
|
112
|
+
* `scrape` would have given. You get up to 200 at a time, from wherever `from` says. A
|
|
113
|
+
* batch whose results have expired still answers with everything else, because somebody
|
|
114
|
+
* who paid for those pages must never be told the batch does not exist.
|
|
113
115
|
*
|
|
114
|
-
* Credits: 0 credits
|
|
116
|
+
* Credits: 0 credits. Reading a batch is free.
|
|
115
117
|
*
|
|
116
118
|
* `GET /v1/batches/{id}`
|
|
117
119
|
*/
|
|
@@ -119,12 +121,12 @@ export declare abstract class GeneratedOperations {
|
|
|
119
121
|
/**
|
|
120
122
|
* Cancel a batch
|
|
121
123
|
*
|
|
122
|
-
* Stop a batch. A URL already being read finishes and is charged
|
|
123
|
-
* already paying for
|
|
124
|
-
*
|
|
125
|
-
*
|
|
124
|
+
* Stop a batch. A URL already being read finishes and is charged, because it is a page we
|
|
125
|
+
* are already paying for. Every URL not yet started becomes `skipped`, which costs nothing
|
|
126
|
+
* and is not a verdict about the URL: submit those positions again whenever you like. Safe
|
|
127
|
+
* to repeat; a batch that has already ended answers with how it ended.
|
|
126
128
|
*
|
|
127
|
-
* Credits: 0 credits
|
|
129
|
+
* Credits: 0 credits. Cancelling is free, and it stops the spending.
|
|
128
130
|
*
|
|
129
131
|
* `DELETE /v1/batches/{id}`
|
|
130
132
|
*/
|
|
@@ -139,10 +141,10 @@ export declare abstract class GeneratedOperations {
|
|
|
139
141
|
* `puppeteer.connect({ browserWSEndpoint: connectUrl })` in Puppeteer. The browser is
|
|
140
142
|
* yours alone for the whole window, every byte it pulls leaves through the exit you
|
|
141
143
|
* picked, and the socket survives our deploys. The URL carries the session's token and is
|
|
142
|
-
* shown once
|
|
144
|
+
* shown once. It cannot be rebuilt from the id.
|
|
143
145
|
*
|
|
144
146
|
* Credits: 2 credits per minute, reserved for the whole window when the session starts and
|
|
145
|
-
* settled by the minutes actually used when it closes
|
|
147
|
+
* settled by the minutes actually used when it closes, so closing early gives the rest
|
|
146
148
|
* back, and a browser that never came up costs nothing at all. Managed-proxy bandwidth
|
|
147
149
|
* past the first megabyte is added at close; your own proxy is never metered.
|
|
148
150
|
*
|
|
@@ -153,10 +155,10 @@ export declare abstract class GeneratedOperations {
|
|
|
153
155
|
* Price a session
|
|
154
156
|
*
|
|
155
157
|
* What holding a browser for that long would cost, without holding one. `maxCost` is the
|
|
156
|
-
* whole window at the per-minute rate
|
|
157
|
-
* not use come back when the session closes.
|
|
158
|
+
* whole window at the per-minute rate. It is a ceiling, not a bill, because the minutes
|
|
159
|
+
* you do not use come back when the session closes.
|
|
158
160
|
*
|
|
159
|
-
* Credits: 0 credits
|
|
161
|
+
* Credits: 0 credits. Pricing a session is free.
|
|
160
162
|
*
|
|
161
163
|
* `POST /v1/sessions/quote`
|
|
162
164
|
*/
|
|
@@ -164,10 +166,10 @@ export declare abstract class GeneratedOperations {
|
|
|
164
166
|
/**
|
|
165
167
|
* List your sessions
|
|
166
168
|
*
|
|
167
|
-
* Your most recent sessions, newest first
|
|
169
|
+
* Your most recent sessions, newest first: the live ones and the ones that have ended,
|
|
168
170
|
* with what each of them cost.
|
|
169
171
|
*
|
|
170
|
-
* Credits: 0 credits
|
|
172
|
+
* Credits: 0 credits. Listing sessions is free.
|
|
171
173
|
*
|
|
172
174
|
* `GET /v1/sessions`
|
|
173
175
|
*/
|
|
@@ -176,10 +178,10 @@ export declare abstract class GeneratedOperations {
|
|
|
176
178
|
* Read one session
|
|
177
179
|
*
|
|
178
180
|
* Where a session got to: whether it is still live, when it expires, what it has pulled
|
|
179
|
-
* and
|
|
181
|
+
* and, once it has ended, what it finally cost. The connect URL is not repeated here; it
|
|
180
182
|
* is shown once, when the session starts.
|
|
181
183
|
*
|
|
182
|
-
* Credits: 0 credits
|
|
184
|
+
* Credits: 0 credits. Reading a session is free.
|
|
183
185
|
*
|
|
184
186
|
* `GET /v1/sessions/{id}`
|
|
185
187
|
*/
|
|
@@ -188,8 +190,8 @@ export declare abstract class GeneratedOperations {
|
|
|
188
190
|
* Close a session
|
|
189
191
|
*
|
|
190
192
|
* Stop the browser and settle the bill: the minutes you held it are kept and the rest of
|
|
191
|
-
* the reservation goes back. Safe to repeat
|
|
192
|
-
*
|
|
193
|
+
* the reservation goes back. Safe to repeat: a session that has already ended answers with
|
|
194
|
+
* how it ended.
|
|
193
195
|
*
|
|
194
196
|
* Credits: Closing is free. It is where the session's own bill lands: the unused minutes
|
|
195
197
|
* are refunded and the managed pool's bandwidth is charged.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EACR,QAAQ,EACR,aAAa,EACb,eAAe,EACf,cAAc,EACd,cAAc,EACd,eAAe,EACf,eAAe,EACf,UAAU,EACV,eAAe,EACf,QAAQ,EACX,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACR,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACf,MAAM,aAAa,CAAC;AAErB,4DAA4D;AAC5D,eAAO,MAAM,WAAW,UAAU,CAAC;AAEnC,sBAAsB;AACtB,eAAO,MAAM,gBAAgB,8BAA8B,CAAC;AAE5D,8BAAsB,mBAAmB;IACrC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACxB,IAAI,EAAE,WAAW,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,CAAC,CAAC;IAEb
|
|
1
|
+
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../src/generated/operations.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EACR,QAAQ,EACR,aAAa,EACb,eAAe,EACf,cAAc,EACd,cAAc,EACd,eAAe,EACf,eAAe,EACf,UAAU,EACV,eAAe,EACf,QAAQ,EACX,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EACR,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,YAAY,EACf,MAAM,aAAa,CAAC;AAErB,4DAA4D;AAC5D,eAAO,MAAM,WAAW,UAAU,CAAC;AAEnC,sBAAsB;AACtB,eAAO,MAAM,gBAAgB,8BAA8B,CAAC;AAE5D,8BAAsB,mBAAmB;IACrC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACxB,IAAI,EAAE,WAAW,EACjB,MAAM,CAAC,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,cAAc,GACtB,OAAO,CAAC,CAAC,CAAC;IAEb;;;;;;;;;;;;;;;;;;OAkBG;IACH,MAAM,CACF,MAAM,EAAE,YAAY,EACpB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,eAAe,CAAC;IAa3B;;;;;;;;;;;OAWG;IACH,KAAK,CACD,MAAM,EAAE,iBAAiB,EACzB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,cAAc,CAAC;IAa1B;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CACF,MAAM,EAAE,YAAY,EACpB,IAAI,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAC9C,OAAO,CAAC,eAAe,CAAC;IAa3B,QAAQ,CAAC,OAAO;QACZ;;;;;;;;;;;;;;;;WAgBG;yBAES,iBAAiB,SAClB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,eAAe,CAAC;QAW3B;;;;;;;;;;;WAWG;wBAES,gBAAgB,SACjB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,aAAa,CAAC;QAWzB;;;;;;;;;WASG;wBAEU,iBAAiB,SACnB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB;;;;;;;;;;;;;;WAcG;sBAES,cAAc,SACf,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,CAAC;QACpB;;;;;;;;;;;WAWG;yBAES,iBAAiB,SAClB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,QAAQ,CAAC;MAMtB;IAEF,QAAQ,CAAC,QAAQ;QACb;;;;;;;;;;;;;;;;WAgBG;0BAEU,mBAAmB,SACrB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,cAAc,CAAC;QAW1B;;;;;;;;;;WAUG;yBAEU,kBAAkB,SACpB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,eAAe,CAAC;QAW3B;;;;;;;;;WASG;wBAEU,kBAAkB,SACpB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,EAAE,CAAC;QAExB;;;;;;;;;;WAUG;sBAES,gBAAgB,SACjB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,CAAC;QAEtB;;;;;;;;;;;WAWG;wBAES,kBAAkB,SACnB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAC9C,OAAO,CAAC,UAAU,CAAC;MAMxB;IAEF,QAAQ,CAAC,KAAK;QACV;;;;;;;;;;WAUG;qBACU,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,KAAG,OAAO,CAAC,QAAQ,CAAC;MAEzE;CACL"}
|