@scrapa/sdk 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/dist/errors.d.ts +12 -3
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +23 -5
- package/dist/errors.js.map +1 -1
- package/dist/generated/operations.d.ts +1 -1
- package/dist/generated/operations.js +1 -1
- package/dist/generated/params.d.ts +4 -2
- package/dist/generated/params.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/errors.ts +25 -5
- package/src/generated/operations.ts +1 -1
- package/src/generated/params.ts +4 -2
package/README.md
CHANGED
|
@@ -204,10 +204,10 @@ try {
|
|
|
204
204
|
return scrapa.scrape({ url, proxy: { pool: "residential", country: "de" } });
|
|
205
205
|
case "denied": // our policy said no — retrying cannot help, change the URL
|
|
206
206
|
return null;
|
|
207
|
-
case "broken": //
|
|
207
|
+
case "broken": // the API said it broke, and refunded the call — worth one retry
|
|
208
208
|
return scrapa.scrape({ url });
|
|
209
209
|
}
|
|
210
|
-
throw error; // your own mistake (a bad field, a missing key, a spent quota), or
|
|
210
|
+
throw error; // your own mistake (a bad field, a missing key, a spent quota), or error.mayHaveRun
|
|
211
211
|
}
|
|
212
212
|
```
|
|
213
213
|
|
|
@@ -227,6 +227,13 @@ less. A read, a quote or a cancel is also sent again after no answer or a 5xx. A
|
|
|
227
227
|
search, a new batch and a new session are not: one that timed out on your side may already have
|
|
228
228
|
read the page, so you get the error and decide.
|
|
229
229
|
|
|
230
|
+
`error.mayHaveRun` is `true` when the call may have run, and been charged, but no answer from
|
|
231
|
+
the API says so. That happens with every `INVALID_RESPONSE`. On any call but a read, it also
|
|
232
|
+
happens with no answer at all, and with a `5xx` page from something between you and the API,
|
|
233
|
+
such as a gateway that gave up on a slow scrape. Only the API's own answer can say it broke and
|
|
234
|
+
refunded the call, so such an error has no `outcome`, and `retryable` is `false`. Call again
|
|
235
|
+
only if a second run is fine.
|
|
236
|
+
|
|
230
237
|
## Bringing your own proxy
|
|
231
238
|
|
|
232
239
|
```ts
|
package/dist/errors.d.ts
CHANGED
|
@@ -16,6 +16,8 @@ export interface ScrapaErrorInit {
|
|
|
16
16
|
retryAfter?: number;
|
|
17
17
|
/** `X-Request-ID` — quote it to support and we can find the call. */
|
|
18
18
|
requestId?: string;
|
|
19
|
+
/** The call may have run, and been charged, though no answer from the API says so. */
|
|
20
|
+
mayHaveRun?: boolean;
|
|
19
21
|
/** What this call cost. Zero on every refusal the API charges nothing for, which is
|
|
20
22
|
* every refusal except a page that answered. */
|
|
21
23
|
cost?: number;
|
|
@@ -30,6 +32,13 @@ export declare class ScrapaError extends Error {
|
|
|
30
32
|
readonly details?: unknown;
|
|
31
33
|
readonly retryAfter?: number;
|
|
32
34
|
readonly requestId?: string;
|
|
35
|
+
/**
|
|
36
|
+
* True when the call may have run, and been charged, though no answer from the API says
|
|
37
|
+
* so: no answer at all or a `5xx` page from something in front of the API, on a call that
|
|
38
|
+
* is not a `GET`, or a `2xx` that is not an answer. Such an error has no `outcome` and is
|
|
39
|
+
* never `retryable`. Call again only if a second run is fine.
|
|
40
|
+
*/
|
|
41
|
+
readonly mayHaveRun: boolean;
|
|
33
42
|
readonly cost?: number;
|
|
34
43
|
readonly creditsRemaining?: number;
|
|
35
44
|
constructor(init: ScrapaErrorInit);
|
|
@@ -43,12 +52,12 @@ export declare class ScrapaError extends Error {
|
|
|
43
52
|
* "broken" we broke it, and already refunded it. Worth one retry as-is.
|
|
44
53
|
*
|
|
45
54
|
* `undefined` for the refusals that were the caller's own doing — a bad field, a missing
|
|
46
|
-
* key, a spent quota — which are not outcomes of a page at all. `undefined` too
|
|
47
|
-
* `
|
|
55
|
+
* key, a spent quota — which are not outcomes of a page at all. `undefined` too when
|
|
56
|
+
* `mayHaveRun` is true, where nobody said no: the call may have worked and been charged.
|
|
48
57
|
*/
|
|
49
58
|
get outcome(): "blocked" | "denied" | "broken" | undefined;
|
|
50
59
|
/** True when repeating the identical call could work. A limit that clears by waiting
|
|
51
|
-
* and anything we broke, never a verdict on the page itself. Never
|
|
60
|
+
* and anything we broke, never a verdict on the page itself. Never when `mayHaveRun`:
|
|
52
61
|
* that call may already have read the page and charged for it. */
|
|
53
62
|
get retryable(): boolean;
|
|
54
63
|
}
|
package/dist/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAOA,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAE9E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAExD;;4BAE4B;AAC5B,MAAM,MAAM,eAAe,GAAG,eAAe,GAAG,SAAS,GAAG,kBAAkB,CAAC;AAE/E,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,SAAS,GAAG,eAAe,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;qDACiD;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,qBAAa,WAAY,SAAQ,KAAK;IAClC,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,eAAe,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAClD,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;gBAEvB,IAAI,EAAE,eAAe;
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAOA,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAE9E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAExD;;4BAE4B;AAC5B,MAAM,MAAM,eAAe,GAAG,eAAe,GAAG,SAAS,GAAG,kBAAkB,CAAC;AAE/E,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,SAAS,GAAG,eAAe,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;qDACiD;IACjD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,qBAAa,WAAY,SAAQ,KAAK;IAClC,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,eAAe,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IAClD,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;gBAEvB,IAAI,EAAE,eAAe;IAejC;;;;;;;;;;;;OAYG;IACH,IAAI,OAAO,IAAI,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAEzD;IAED;;uEAEmE;IACnE,IAAI,SAAS,IAAI,OAAO,CAQvB;CACJ;AAiBD;;4CAE4C;AAC5C,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAuDvE"}
|
package/dist/errors.js
CHANGED
|
@@ -15,6 +15,13 @@ export class ScrapaError extends Error {
|
|
|
15
15
|
details;
|
|
16
16
|
retryAfter;
|
|
17
17
|
requestId;
|
|
18
|
+
/**
|
|
19
|
+
* True when the call may have run, and been charged, though no answer from the API says
|
|
20
|
+
* so: no answer at all or a `5xx` page from something in front of the API, on a call that
|
|
21
|
+
* is not a `GET`, or a `2xx` that is not an answer. Such an error has no `outcome` and is
|
|
22
|
+
* never `retryable`. Call again only if a second run is fine.
|
|
23
|
+
*/
|
|
24
|
+
mayHaveRun;
|
|
18
25
|
cost;
|
|
19
26
|
creditsRemaining;
|
|
20
27
|
constructor(init) {
|
|
@@ -27,6 +34,7 @@ export class ScrapaError extends Error {
|
|
|
27
34
|
this.details = init.details;
|
|
28
35
|
this.retryAfter = init.retryAfter;
|
|
29
36
|
this.requestId = init.requestId;
|
|
37
|
+
this.mayHaveRun = init.mayHaveRun ?? false;
|
|
30
38
|
this.cost = init.cost;
|
|
31
39
|
this.creditsRemaining = init.creditsRemaining;
|
|
32
40
|
}
|
|
@@ -40,16 +48,18 @@ export class ScrapaError extends Error {
|
|
|
40
48
|
* "broken" we broke it, and already refunded it. Worth one retry as-is.
|
|
41
49
|
*
|
|
42
50
|
* `undefined` for the refusals that were the caller's own doing — a bad field, a missing
|
|
43
|
-
* key, a spent quota — which are not outcomes of a page at all. `undefined` too
|
|
44
|
-
* `
|
|
51
|
+
* key, a spent quota — which are not outcomes of a page at all. `undefined` too when
|
|
52
|
+
* `mayHaveRun` is true, where nobody said no: the call may have worked and been charged.
|
|
45
53
|
*/
|
|
46
54
|
get outcome() {
|
|
47
|
-
return OUTCOME_OF[this.code];
|
|
55
|
+
return this.mayHaveRun ? undefined : OUTCOME_OF[this.code];
|
|
48
56
|
}
|
|
49
57
|
/** True when repeating the identical call could work. A limit that clears by waiting
|
|
50
|
-
* and anything we broke, never a verdict on the page itself. Never
|
|
58
|
+
* and anything we broke, never a verdict on the page itself. Never when `mayHaveRun`:
|
|
51
59
|
* that call may already have read the page and charged for it. */
|
|
52
60
|
get retryable() {
|
|
61
|
+
if (this.mayHaveRun)
|
|
62
|
+
return false;
|
|
53
63
|
return (this.code === "NETWORK_ERROR" ||
|
|
54
64
|
this.code === "TIMEOUT" ||
|
|
55
65
|
this.code === "RATE_LIMIT_EXCEEDED" ||
|
|
@@ -82,6 +92,7 @@ export function scrapaError(failure, host) {
|
|
|
82
92
|
return new ScrapaError({
|
|
83
93
|
code: (failure.timedOut ? "TIMEOUT" : "NETWORK_ERROR"),
|
|
84
94
|
status: 0,
|
|
95
|
+
mayHaveRun: method !== "GET",
|
|
85
96
|
message: failure.timedOut
|
|
86
97
|
? `scrapa: no answer within ${failure.timeoutMs}ms — the page may still have been read and charged.`
|
|
87
98
|
: `scrapa: could not reach ${host}.`,
|
|
@@ -95,18 +106,25 @@ export function scrapaError(failure, host) {
|
|
|
95
106
|
return new ScrapaError({
|
|
96
107
|
code: "INVALID_RESPONSE",
|
|
97
108
|
status,
|
|
109
|
+
mayHaveRun: true,
|
|
98
110
|
message: `scrapa: ${method} ${path} answered ${status}, but the body is not an answer from this API. The call may have run and been charged, so it was not sent again.`,
|
|
99
111
|
details: failure.text,
|
|
100
112
|
requestId: failure.requestId,
|
|
101
113
|
cost: numberHeader(failure.headers, HEADERS.cost),
|
|
102
114
|
creditsRemaining: numberHeader(failure.headers, HEADERS.creditsRemaining),
|
|
103
115
|
});
|
|
116
|
+
// A 5xx that is not our envelope came from something in front of the API: a gateway that
|
|
117
|
+
// gave up on a scrape the API may still have read and charged for. Only the API's own
|
|
118
|
+
// envelope can say it broke and refunded, so on anything but a GET nobody said no.
|
|
119
|
+
const mayHaveRun = !envelope && status >= 500 && method !== "GET";
|
|
104
120
|
return new ScrapaError({
|
|
105
121
|
code: envelope ? envelope.code : statusCode(status),
|
|
106
122
|
status,
|
|
123
|
+
mayHaveRun,
|
|
107
124
|
// A proxy or a cold start can answer HTML; say what actually came back rather than
|
|
108
125
|
// pretending the body was ours.
|
|
109
|
-
message: envelope?.message ??
|
|
126
|
+
message: envelope?.message ??
|
|
127
|
+
`scrapa: ${method} ${path} failed with ${status}.${mayHaveRun ? " The call may have run and been charged. Call again only if a second run is fine." : ""}`,
|
|
110
128
|
messageKey: envelope?.messageKey,
|
|
111
129
|
params: envelope?.params,
|
|
112
130
|
details: envelope ? envelope.details : failure.text,
|
package/dist/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,EAAE;AACF,wFAAwF;AACxF,2FAA2F;AAC3F,qFAAqF;AACrF,yFAAyF;AACzF,2FAA2F;AAC3F,OAAO,EAAE,OAAO,EAAE,UAAU,EAAkB,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,EAAE;AACF,wFAAwF;AACxF,2FAA2F;AAC3F,qFAAqF;AACrF,yFAAyF;AACzF,2FAA2F;AAC3F,OAAO,EAAE,OAAO,EAAE,UAAU,EAAkB,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA6BtD,MAAM,OAAO,WAAY,SAAQ,KAAK;IACzB,IAAI,CAA8B;IAClC,MAAM,CAAS;IACf,UAAU,CAAU;IACpB,MAAM,CAAmC;IACzC,OAAO,CAAW;IAClB,UAAU,CAAU;IACpB,SAAS,CAAU;IAC5B;;;;;OAKG;IACM,UAAU,CAAU;IACpB,IAAI,CAAU;IACd,gBAAgB,CAAU;IAEnC,YAAY,IAAqB;QAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClF,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC;IAClD,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,IAAI,OAAO;QACP,OAAO,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAiB,CAAC,CAAC;IAC5E,CAAC;IAED;;uEAEmE;IACnE,IAAI,SAAS;QACT,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,KAAK,CAAC;QAClC,OAAO,CACH,IAAI,CAAC,IAAI,KAAK,eAAe;YAC7B,IAAI,CAAC,IAAI,KAAK,SAAS;YACvB,IAAI,CAAC,IAAI,KAAK,qBAAqB;YACnC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAC5B,CAAC;IACN,CAAC;CACJ;AAED,SAAS,YAAY,CAAC,OAA4B,EAAE,IAAY;IAC5D,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACxD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,oEAAoE;AACpE,SAAS,UAAU,CAAC,MAAc;IAC9B,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,qBAAqB,CAAC;IACjD,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,WAAW,CAAC;IACvC,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,cAAc,CAAC;IAC1C,OAAO,gBAAgB,CAAC;AAC5B,CAAC;AAED;;4CAE4C;AAC5C,MAAM,UAAU,WAAW,CAAC,OAAgB,EAAE,IAAY;IACtD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAC1D,IAAI,MAAM,KAAK,CAAC;QACZ,OAAO,IAAI,WAAW,CAAC;YACnB,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAA2B;YAChF,MAAM,EAAE,CAAC;YACT,UAAU,EAAE,MAAM,KAAK,KAAK;YAC5B,OAAO,EAAE,OAAO,CAAC,QAAQ;gBACrB,CAAC,CAAC,4BAA4B,OAAO,CAAC,SAAS,qDAAqD;gBACpG,CAAC,CAAC,2BAA2B,IAAI,GAAG;YACxC,KAAK,EAAE,OAAO,CAAC,KAAK;SACvB,CAAC,CAAC;IACP,sFAAsF;IACtF,0FAA0F;IAC1F,wFAAwF;IACxF,uEAAuE;IACvE,IAAI,MAAM,GAAG,GAAG;QACZ,OAAO,IAAI,WAAW,CAAC;YACnB,IAAI,EAAE,kBAA4C;YAClD,MAAM;YACN,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,WAAW,MAAM,IAAI,IAAI,aAAa,MAAM,kHAAkH;YACvK,OAAO,EAAE,OAAO,CAAC,IAAI;YACrB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;YACjD,gBAAgB,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC;SAC5E,CAAC,CAAC;IACP,yFAAyF;IACzF,sFAAsF;IACtF,mFAAmF;IACnF,MAAM,UAAU,GAAG,CAAC,QAAQ,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,KAAK,KAAK,CAAC;IAClE,OAAO,IAAI,WAAW,CAAC;QACnB,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAE,QAAQ,CAAC,IAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAClE,MAAM;QACN,UAAU;QACV,mFAAmF;QACnF,gCAAgC;QAChC,OAAO,EACH,QAAQ,EAAE,OAAO;YACjB,WAAW,MAAM,IAAI,IAAI,gBAAgB,MAAM,IAAI,UAAU,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9J,UAAU,EAAE,QAAQ,EAAE,UAAU;QAChC,MAAM,EAAE,QAAQ,EAAE,MAAM;QACxB,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI;QACnD,sEAAsE;QACtE,UAAU,EACN,cAAc,CAAC;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,OAAO,EAAE,QAAQ,EAAE,OAAO;SAC7B,CAAC,IAAI,SAAS;QACnB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,uFAAuF;QACvF,mFAAmF;QACnF,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC;QACjD,gBAAgB,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,gBAAgB,CAAC;KAC5E,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -2,7 +2,7 @@ import type { RequestOptions, RequestSpec } from "../client.js";
|
|
|
2
2
|
import type { BatchDto, BatchQuoteDto, CreatedBatchDto, LiveSessionDto, ScrapeQuoteDto, ScrapeResultDto, SearchResultDto, SessionDto, SessionQuoteDto, UsageDto } from "./contract.js";
|
|
3
3
|
import type { CancelBatchParams, CloseSessionParams, CreateBatchParams, CreateSessionParams, GetBatchParams, GetSessionParams, ListBatchesParams, ListSessionsParams, QuoteBatchParams, QuoteScrapeParams, QuoteSessionParams, ScrapeParams, SearchParams } from "./params.js";
|
|
4
4
|
/** The version this client reports in `x-scrapa-client`. */
|
|
5
|
-
export declare const SDK_VERSION = "0.2.
|
|
5
|
+
export declare const SDK_VERSION = "0.2.2";
|
|
6
6
|
/** The hosted API. */
|
|
7
7
|
export declare const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
8
8
|
export declare abstract class GeneratedOperations {
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// One method per capability, grouped the way the docs group them, returning the
|
|
5
5
|
// DTO the operation declares. `Scrapa` in ../client.ts supplies request().
|
|
6
6
|
/** The version this client reports in `x-scrapa-client`. */
|
|
7
|
-
export const SDK_VERSION = "0.2.
|
|
7
|
+
export const SDK_VERSION = "0.2.2";
|
|
8
8
|
/** The hosted API. */
|
|
9
9
|
export const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
10
10
|
export class GeneratedOperations {
|
|
@@ -323,7 +323,8 @@ export interface CreateBatchParams {
|
|
|
323
323
|
/** Where to POST when something happens. An https URL you control; private and loopback
|
|
324
324
|
* hosts are refused, the same as any other URL we fetch. */
|
|
325
325
|
url: string;
|
|
326
|
-
/** What to call about. "batch.completed" fires once
|
|
326
|
+
/** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
|
|
327
|
+
* we could not finish the batch. `batch.status` says which: `completed` or `failed`.
|
|
327
328
|
* "batch.item.completed" fires per URL — one request from us per item, which is what makes
|
|
328
329
|
* a long batch usable before it ends, so ask for it deliberately. */
|
|
329
330
|
events?: ("batch.completed" | "batch.item.completed")[];
|
|
@@ -430,7 +431,8 @@ export interface QuoteBatchParams {
|
|
|
430
431
|
/** Where to POST when something happens. An https URL you control; private and loopback
|
|
431
432
|
* hosts are refused, the same as any other URL we fetch. */
|
|
432
433
|
url: string;
|
|
433
|
-
/** What to call about. "batch.completed" fires once
|
|
434
|
+
/** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
|
|
435
|
+
* we could not finish the batch. `batch.status` says which: `completed` or `failed`.
|
|
434
436
|
* "batch.item.completed" fires per URL — one request from us per item, which is what makes
|
|
435
437
|
* a long batch usable before it ends, so ask for it deliberately. */
|
|
436
438
|
events?: ("batch.completed" | "batch.item.completed")[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,iFAAiF;IACjF,GAAG,EAAE,MAAM,CAAC;IACZ;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+BAA+B;AAC/B,MAAM,WAAW,iBAAiB;IAC9B,iFAAiF;IACjF,GAAG,EAAE,MAAM,CAAC;IACZ;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd;;oDAEgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;+CAC2C;IAC3C,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;6BAEyB;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;uFACmF;IACnF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;uDACmD;IACnD,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ
|
|
1
|
+
{"version":3,"file":"params.d.ts","sourceRoot":"","sources":["../../src/generated/params.ts"],"names":[],"mappings":"AAMA,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,iFAAiF;IACjF,GAAG,EAAE,MAAM,CAAC;IACZ;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+BAA+B;AAC/B,MAAM,WAAW,iBAAiB;IAC9B,iFAAiF;IACjF,GAAG,EAAE,MAAM,CAAC;IACZ;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,gCAAgC;AAChC,MAAM,WAAW,YAAY;IACzB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd;;oDAEgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;+CAC2C;IAC3C,KAAK,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC;IAC3B,6EAA6E;IAC7E,SAAS,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;IAC9C;;6BAEyB;IACzB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B;uFACmF;IACnF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;uDACmD;IACnD,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;8EAGsE;QACtE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,gBAAgB;IAC7B;;+DAE2D;IAC3D,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACtC;;sGAEkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;sCAGkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;0DAEsD;IACtD,OAAO,CAAC,EAAE,CAAC,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,YAAY,GAAG,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC;IACxF;;;qGAGiG;IACjG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;gBAGY;IACZ,OAAO,CAAC,EAAE;QACN;uGAC+F;QAC/F,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC;;qBAEa;QACb,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,+EAA+E;IAC/E,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;IACF;iCAC6B;IAC7B,OAAO,CAAC,EAAE;QACN;;;oBAGY;QACZ,SAAS,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;QACxD;;wCAEgC;QAChC,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;0EACkE;QAClE,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;kDAC0C;QAC1C,QAAQ,CAAC,EAAE;YACP,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;SAClB,CAAC;QACF;iFACyE;QACzE,QAAQ,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;IACF;6DACyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC;wBACoB;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qBAEiB;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;4BACwB;IACxB,OAAO,CAAC,EAAE;QACN;qEAC6D;QAC7D,GAAG,EAAE,MAAM,CAAC;QACZ;;;8EAGsE;QACtE,MAAM,CAAC,EAAE,CAAC,iBAAiB,GAAG,sBAAsB,CAAC,EAAE,CAAC;KAC3D,CAAC;CACL;AAED,sCAAsC;AACtC,MAAM,WAAW,iBAAiB;IAC9B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC3B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX;kEAC8D;IAC9D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;uFACmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IAC9B,6CAA6C;IAC7C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,yCAAyC;AACzC,MAAM,WAAW,mBAAmB;IAChC;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B;;yFAEqF;IACrF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;gBACY;IACZ,KAAK,CAAC,EAAE;QACJ;mFAC2E;QAC3E,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAAC;QAChC;mCAC2B;QAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB;;;6BAGqB;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;4DAEoD;QACpD,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;8DAGsD;QACtD,IAAI,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACL;AAED,uCAAuC;AACvC,MAAM,WAAW,kBAAkB;IAC/B,kEAAkE;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sCAAsC;AACtC,MAAM,WAAW,gBAAgB;IAC7B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd;AAED,wCAAwC;AACxC,MAAM,WAAW,kBAAkB;IAC/B,+CAA+C;IAC/C,EAAE,EAAE,MAAM,CAAC;CACd"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@scrapa/sdk",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "Official TypeScript client for scrapa.app — scrape any web page as markdown, HTML, text, links, a screenshot or a PDF, search the web, and drive a real browser over CDP.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"homepage": "https://scrapa.app",
|
package/src/errors.ts
CHANGED
|
@@ -26,6 +26,8 @@ export interface ScrapaErrorInit {
|
|
|
26
26
|
retryAfter?: number;
|
|
27
27
|
/** `X-Request-ID` — quote it to support and we can find the call. */
|
|
28
28
|
requestId?: string;
|
|
29
|
+
/** The call may have run, and been charged, though no answer from the API says so. */
|
|
30
|
+
mayHaveRun?: boolean;
|
|
29
31
|
/** What this call cost. Zero on every refusal the API charges nothing for, which is
|
|
30
32
|
* every refusal except a page that answered. */
|
|
31
33
|
cost?: number;
|
|
@@ -41,6 +43,13 @@ export class ScrapaError extends Error {
|
|
|
41
43
|
readonly details?: unknown;
|
|
42
44
|
readonly retryAfter?: number;
|
|
43
45
|
readonly requestId?: string;
|
|
46
|
+
/**
|
|
47
|
+
* True when the call may have run, and been charged, though no answer from the API says
|
|
48
|
+
* so: no answer at all or a `5xx` page from something in front of the API, on a call that
|
|
49
|
+
* is not a `GET`, or a `2xx` that is not an answer. Such an error has no `outcome` and is
|
|
50
|
+
* never `retryable`. Call again only if a second run is fine.
|
|
51
|
+
*/
|
|
52
|
+
readonly mayHaveRun: boolean;
|
|
44
53
|
readonly cost?: number;
|
|
45
54
|
readonly creditsRemaining?: number;
|
|
46
55
|
|
|
@@ -54,6 +63,7 @@ export class ScrapaError extends Error {
|
|
|
54
63
|
this.details = init.details;
|
|
55
64
|
this.retryAfter = init.retryAfter;
|
|
56
65
|
this.requestId = init.requestId;
|
|
66
|
+
this.mayHaveRun = init.mayHaveRun ?? false;
|
|
57
67
|
this.cost = init.cost;
|
|
58
68
|
this.creditsRemaining = init.creditsRemaining;
|
|
59
69
|
}
|
|
@@ -68,17 +78,18 @@ export class ScrapaError extends Error {
|
|
|
68
78
|
* "broken" we broke it, and already refunded it. Worth one retry as-is.
|
|
69
79
|
*
|
|
70
80
|
* `undefined` for the refusals that were the caller's own doing — a bad field, a missing
|
|
71
|
-
* key, a spent quota — which are not outcomes of a page at all. `undefined` too
|
|
72
|
-
* `
|
|
81
|
+
* key, a spent quota — which are not outcomes of a page at all. `undefined` too when
|
|
82
|
+
* `mayHaveRun` is true, where nobody said no: the call may have worked and been charged.
|
|
73
83
|
*/
|
|
74
84
|
get outcome(): "blocked" | "denied" | "broken" | undefined {
|
|
75
|
-
return OUTCOME_OF[this.code as ErrorCode];
|
|
85
|
+
return this.mayHaveRun ? undefined : OUTCOME_OF[this.code as ErrorCode];
|
|
76
86
|
}
|
|
77
87
|
|
|
78
88
|
/** True when repeating the identical call could work. A limit that clears by waiting
|
|
79
|
-
* and anything we broke, never a verdict on the page itself. Never
|
|
89
|
+
* and anything we broke, never a verdict on the page itself. Never when `mayHaveRun`:
|
|
80
90
|
* that call may already have read the page and charged for it. */
|
|
81
91
|
get retryable(): boolean {
|
|
92
|
+
if (this.mayHaveRun) return false;
|
|
82
93
|
return (
|
|
83
94
|
this.code === "NETWORK_ERROR" ||
|
|
84
95
|
this.code === "TIMEOUT" ||
|
|
@@ -112,6 +123,7 @@ export function scrapaError(failure: Failure, host: string): ScrapaError {
|
|
|
112
123
|
return new ScrapaError({
|
|
113
124
|
code: (failure.timedOut ? "TIMEOUT" : "NETWORK_ERROR") satisfies ClientErrorCode,
|
|
114
125
|
status: 0,
|
|
126
|
+
mayHaveRun: method !== "GET",
|
|
115
127
|
message: failure.timedOut
|
|
116
128
|
? `scrapa: no answer within ${failure.timeoutMs}ms — the page may still have been read and charged.`
|
|
117
129
|
: `scrapa: could not reach ${host}.`,
|
|
@@ -125,18 +137,26 @@ export function scrapaError(failure: Failure, host: string): ScrapaError {
|
|
|
125
137
|
return new ScrapaError({
|
|
126
138
|
code: "INVALID_RESPONSE" satisfies ClientErrorCode,
|
|
127
139
|
status,
|
|
140
|
+
mayHaveRun: true,
|
|
128
141
|
message: `scrapa: ${method} ${path} answered ${status}, but the body is not an answer from this API. The call may have run and been charged, so it was not sent again.`,
|
|
129
142
|
details: failure.text,
|
|
130
143
|
requestId: failure.requestId,
|
|
131
144
|
cost: numberHeader(failure.headers, HEADERS.cost),
|
|
132
145
|
creditsRemaining: numberHeader(failure.headers, HEADERS.creditsRemaining),
|
|
133
146
|
});
|
|
147
|
+
// A 5xx that is not our envelope came from something in front of the API: a gateway that
|
|
148
|
+
// gave up on a scrape the API may still have read and charged for. Only the API's own
|
|
149
|
+
// envelope can say it broke and refunded, so on anything but a GET nobody said no.
|
|
150
|
+
const mayHaveRun = !envelope && status >= 500 && method !== "GET";
|
|
134
151
|
return new ScrapaError({
|
|
135
152
|
code: envelope ? (envelope.code as ErrorCode) : statusCode(status),
|
|
136
153
|
status,
|
|
154
|
+
mayHaveRun,
|
|
137
155
|
// A proxy or a cold start can answer HTML; say what actually came back rather than
|
|
138
156
|
// pretending the body was ours.
|
|
139
|
-
message:
|
|
157
|
+
message:
|
|
158
|
+
envelope?.message ??
|
|
159
|
+
`scrapa: ${method} ${path} failed with ${status}.${mayHaveRun ? " The call may have run and been charged. Call again only if a second run is fine." : ""}`,
|
|
140
160
|
messageKey: envelope?.messageKey,
|
|
141
161
|
params: envelope?.params,
|
|
142
162
|
details: envelope ? envelope.details : failure.text,
|
|
@@ -34,7 +34,7 @@ import type {
|
|
|
34
34
|
} from "./params.ts";
|
|
35
35
|
|
|
36
36
|
/** The version this client reports in `x-scrapa-client`. */
|
|
37
|
-
export const SDK_VERSION = "0.2.
|
|
37
|
+
export const SDK_VERSION = "0.2.2";
|
|
38
38
|
|
|
39
39
|
/** The hosted API. */
|
|
40
40
|
export const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
package/src/generated/params.ts
CHANGED
|
@@ -332,7 +332,8 @@ export interface CreateBatchParams {
|
|
|
332
332
|
/** Where to POST when something happens. An https URL you control; private and loopback
|
|
333
333
|
* hosts are refused, the same as any other URL we fetch. */
|
|
334
334
|
url: string;
|
|
335
|
-
/** What to call about. "batch.completed" fires once
|
|
335
|
+
/** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
|
|
336
|
+
* we could not finish the batch. `batch.status` says which: `completed` or `failed`.
|
|
336
337
|
* "batch.item.completed" fires per URL — one request from us per item, which is what makes
|
|
337
338
|
* a long batch usable before it ends, so ask for it deliberately. */
|
|
338
339
|
events?: ("batch.completed" | "batch.item.completed")[];
|
|
@@ -440,7 +441,8 @@ export interface QuoteBatchParams {
|
|
|
440
441
|
/** Where to POST when something happens. An https URL you control; private and loopback
|
|
441
442
|
* hosts are refused, the same as any other URL we fetch. */
|
|
442
443
|
url: string;
|
|
443
|
-
/** What to call about. "batch.completed" fires once
|
|
444
|
+
/** What to call about. "batch.completed" fires once: when every URL has a verdict, or when
|
|
445
|
+
* we could not finish the batch. `batch.status` says which: `completed` or `failed`.
|
|
444
446
|
* "batch.item.completed" fires per URL — one request from us per item, which is what makes
|
|
445
447
|
* a long batch usable before it ends, so ask for it deliberately. */
|
|
446
448
|
events?: ("batch.completed" | "batch.item.completed")[];
|