@scrapa/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +232 -0
- package/dist/client.d.ts +47 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +182 -0
- package/dist/client.js.map +1 -0
- package/dist/errors.d.ts +50 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +56 -0
- package/dist/errors.js.map +1 -0
- package/dist/generated/contract.d.ts +470 -0
- package/dist/generated/contract.d.ts.map +1 -0
- package/dist/generated/contract.js +161 -0
- package/dist/generated/contract.js.map +1 -0
- package/dist/generated/operations.d.ts +213 -0
- package/dist/generated/operations.d.ts.map +1 -0
- package/dist/generated/operations.js +220 -0
- package/dist/generated/operations.js.map +1 -0
- package/dist/generated/params.d.ts +420 -0
- package/dist/generated/params.d.ts.map +1 -0
- package/dist/generated/params.js +7 -0
- package/dist/generated/params.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/package.json +55 -0
- package/src/client.ts +227 -0
- package/src/errors.ts +83 -0
- package/src/generated/contract.ts +564 -0
- package/src/generated/operations.ts +293 -0
- package/src/generated/params.ts +437 -0
- package/src/index.ts +9 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
// GENERATED FILE — do not edit. Run `bun run sdk:gen`.
|
|
2
|
+
// Source: apps/api/src/routes/v1 (the mounted operations)
|
|
3
|
+
//
|
|
4
|
+
// One method per capability, grouped the way the docs group them, returning the
|
|
5
|
+
// DTO the operation declares. `Scrapa` in ../client.ts supplies request().
|
|
6
|
+
|
|
7
|
+
import type { RequestOptions, RequestSpec } from "../client.ts";
|
|
8
|
+
import type {
|
|
9
|
+
BatchDto,
|
|
10
|
+
BatchQuoteDto,
|
|
11
|
+
CreatedBatchDto,
|
|
12
|
+
LiveSessionDto,
|
|
13
|
+
ScrapeQuoteDto,
|
|
14
|
+
ScrapeResultDto,
|
|
15
|
+
SearchResultDto,
|
|
16
|
+
SessionDto,
|
|
17
|
+
SessionQuoteDto,
|
|
18
|
+
UsageDto,
|
|
19
|
+
} from "./contract.ts";
|
|
20
|
+
import type {
|
|
21
|
+
CancelBatchParams,
|
|
22
|
+
CloseSessionParams,
|
|
23
|
+
CreateBatchParams,
|
|
24
|
+
CreateSessionParams,
|
|
25
|
+
GetBatchParams,
|
|
26
|
+
GetSessionParams,
|
|
27
|
+
ListBatchesParams,
|
|
28
|
+
ListSessionsParams,
|
|
29
|
+
QuoteBatchParams,
|
|
30
|
+
QuoteScrapeParams,
|
|
31
|
+
QuoteSessionParams,
|
|
32
|
+
ScrapeParams,
|
|
33
|
+
SearchParams,
|
|
34
|
+
} from "./params.ts";
|
|
35
|
+
|
|
36
|
+
/** The version this client reports in `x-scrapa-client`. */
|
|
37
|
+
export const SDK_VERSION = "0.1.0";
|
|
38
|
+
|
|
39
|
+
/** The hosted API. */
|
|
40
|
+
export const DEFAULT_BASE_URL = "https://api.scrapa.app/v1";
|
|
41
|
+
|
|
42
|
+
export abstract class GeneratedOperations {
|
|
43
|
+
protected abstract request<T>(
|
|
44
|
+
spec: RequestSpec,
|
|
45
|
+
params?: object,
|
|
46
|
+
opts?: RequestOptions,
|
|
47
|
+
): Promise<T>;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Scrape a page
|
|
51
|
+
*
|
|
52
|
+
* Read one web page and hand it back as markdown, HTML, plain text, its links, a screenshot or
|
|
53
|
+
* a PDF. `engine: "auto"` (the default) makes a plain HTTP request first and re-reads the page
|
|
54
|
+
* in a real Chromium when the body turns out to be a JavaScript shell, so you pay for a
|
|
55
|
+
* browser only when the page needed one. Every request leaves through a residential proxy, or
|
|
56
|
+
* through your own with `proxy: { pool: "custom", server }`. The answer says which engine
|
|
57
|
+
* replied, where the page finally was and what it cost.
|
|
58
|
+
*
|
|
59
|
+
* Credits: 1 credit with the fetch engine, 5 with the browser (a screenshot or a PDF is included, never
|
|
60
|
+
* a surcharge), plus managed-proxy bandwidth past the first megabyte. A page the site blocked,
|
|
61
|
+
* a URL our policy refused and anything we broke all cost nothing.
|
|
62
|
+
*
|
|
63
|
+
* `POST /v1/scrape`
|
|
64
|
+
*/
|
|
65
|
+
scrape(params: ScrapeParams, opts?: RequestOptions): Promise<ScrapeResultDto> {
|
|
66
|
+
return this.request({ method: "POST", path: "/scrape", idempotent: true }, params, opts);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Price a scrape
|
|
71
|
+
*
|
|
72
|
+
* What the same request would cost, without making it. Free, and refused by the same guards a
|
|
73
|
+
* real call meets — so a quote is never a price for a call that would be denied. `maxCost` is
|
|
74
|
+
* the ceiling: for `auto` it is the browser's price, which is only charged if the page turns
|
|
75
|
+
* out to need one.
|
|
76
|
+
*
|
|
77
|
+
* Credits: 0 credits — pricing a call is free.
|
|
78
|
+
*
|
|
79
|
+
* `POST /v1/scrape/quote`
|
|
80
|
+
*/
|
|
81
|
+
quote(params: QuoteScrapeParams, opts?: RequestOptions): Promise<ScrapeQuoteDto> {
|
|
82
|
+
return this.request(
|
|
83
|
+
{ method: "POST", path: "/scrape/quote", idempotent: true },
|
|
84
|
+
params,
|
|
85
|
+
opts,
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Search the web
|
|
91
|
+
*
|
|
92
|
+
* Ask the web a question and get back ranked results — title, URL and the snippet the index
|
|
93
|
+
* holds — read live off a real search front through a residential exit, never a reseller's
|
|
94
|
+
* cache. `country` picks both the exit and the market, so `br` returns what somebody in Brazil
|
|
95
|
+
* would see. Narrow to your own list of hosts with `includeDomains`, ask for `topic: "news"`
|
|
96
|
+
* when you want dated journalism, and add `answer: true` when you want a paragraph written
|
|
97
|
+
* over the results instead of writing it yourself. To read a result, pass its URL to `scrape`.
|
|
98
|
+
*
|
|
99
|
+
* Credits: 4 credits per 10 results, so a `limit` above 10 costs two rows, plus 1 for `answer: true`. A
|
|
100
|
+
* search every front refused, and anything we broke, cost nothing; a search that ran and found
|
|
101
|
+
* nothing is charged, because the index answered. A repeat of the same search within a few
|
|
102
|
+
* minutes is free and says so with `cached: true`.
|
|
103
|
+
*
|
|
104
|
+
* `POST /v1/search`
|
|
105
|
+
*/
|
|
106
|
+
search(params: SearchParams, opts?: RequestOptions): Promise<SearchResultDto> {
|
|
107
|
+
return this.request({ method: "POST", path: "/search", idempotent: true }, params, opts);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
readonly batches = {
|
|
111
|
+
/**
|
|
112
|
+
* Start a batch
|
|
113
|
+
*
|
|
114
|
+
* Hand us a list of URLs and stop holding a connection open. Every one is read with the
|
|
115
|
+
* options you pass here — the same engines, the same exit and the same meter a single
|
|
116
|
+
* `scrape` uses — and a page the site blocked costs nothing here too. Nothing is charged
|
|
117
|
+
* when you submit: a batch spends as it goes, so use `quote` for the ceiling. Poll `get`
|
|
118
|
+
* for progress, or pass a `webhook` and we will call you instead. Results are readable for
|
|
119
|
+
* 24 hours.
|
|
120
|
+
*
|
|
121
|
+
* Credits: Creating a batch is free, and nothing is held: each URL then costs exactly what the same
|
|
122
|
+
* call would have cost on its own, and the pages the web refused cost nothing here too.
|
|
123
|
+
* What the batch has spent so far is on the batch itself, and it grows while it runs.
|
|
124
|
+
*
|
|
125
|
+
* `POST /v1/batches`
|
|
126
|
+
*/
|
|
127
|
+
create: (params: CreateBatchParams, opts?: RequestOptions): Promise<CreatedBatchDto> =>
|
|
128
|
+
this.request({ method: "POST", path: "/batches", idempotent: false }, params, opts),
|
|
129
|
+
/**
|
|
130
|
+
* Price a batch
|
|
131
|
+
*
|
|
132
|
+
* What that submission would cost at most, without submitting it. `maxCost` prices every
|
|
133
|
+
* URL at the ceiling one page could reach, so the real bill comes in lower by whatever the
|
|
134
|
+
* web refuses for free and by every page that never needed a browser. Bandwidth is on top
|
|
135
|
+
* and cannot be quoted — it depends on what the pages weigh.
|
|
136
|
+
*
|
|
137
|
+
* Credits: 0 credits — pricing a batch is free.
|
|
138
|
+
*
|
|
139
|
+
* `POST /v1/batches/quote`
|
|
140
|
+
*/
|
|
141
|
+
quote: (params: QuoteBatchParams, opts?: RequestOptions): Promise<BatchQuoteDto> =>
|
|
142
|
+
this.request(
|
|
143
|
+
{ method: "POST", path: "/batches/quote", idempotent: true },
|
|
144
|
+
params,
|
|
145
|
+
opts,
|
|
146
|
+
),
|
|
147
|
+
/**
|
|
148
|
+
* List your batches
|
|
149
|
+
*
|
|
150
|
+
* Your most recent batches, newest first — what each one is doing, how far it has got and
|
|
151
|
+
* what it has cost. Results are not included; ask `get` for one batch's pages.
|
|
152
|
+
*
|
|
153
|
+
* Credits: 0 credits — listing batches is free.
|
|
154
|
+
*
|
|
155
|
+
* `GET /v1/batches`
|
|
156
|
+
*/
|
|
157
|
+
list: (params?: ListBatchesParams, opts?: RequestOptions): Promise<BatchDto[]> =>
|
|
158
|
+
this.request({ method: "GET", path: "/batches", idempotent: true }, params, opts),
|
|
159
|
+
/**
|
|
160
|
+
* Read one batch
|
|
161
|
+
*
|
|
162
|
+
* Where a batch got to: whether it is still running, how many of its URLs now have a
|
|
163
|
+
* verdict, how many of those the web or our policy refused, how many were never attempted,
|
|
164
|
+
* and what it has spent so far. Ask for `items` to get the results themselves — the pages
|
|
165
|
+
* and the refusals, in the order you submitted them, each one the same answer a single
|
|
166
|
+
* `scrape` would have given — up to 200 at a time, from wherever `from` says. A batch
|
|
167
|
+
* whose results have expired still answers with everything else, because somebody who paid
|
|
168
|
+
* for those pages must never be told the batch does not exist.
|
|
169
|
+
*
|
|
170
|
+
* Credits: 0 credits — reading a batch is free.
|
|
171
|
+
*
|
|
172
|
+
* `GET /v1/batches/{id}`
|
|
173
|
+
*/
|
|
174
|
+
get: (params: GetBatchParams, opts?: RequestOptions): Promise<BatchDto> =>
|
|
175
|
+
this.request({ method: "GET", path: "/batches/{id}", idempotent: true }, params, opts),
|
|
176
|
+
/**
|
|
177
|
+
* Cancel a batch
|
|
178
|
+
*
|
|
179
|
+
* Stop a batch. A URL already being read finishes and is charged — it is a page we are
|
|
180
|
+
* already paying for — and every URL not yet started becomes `skipped`, which costs
|
|
181
|
+
* nothing and is not a verdict about the URL: submit those positions again whenever you
|
|
182
|
+
* like. Safe to repeat; a batch that has already ended answers with how it ended.
|
|
183
|
+
*
|
|
184
|
+
* Credits: 0 credits — cancelling is free, and it stops the spending.
|
|
185
|
+
*
|
|
186
|
+
* `DELETE /v1/batches/{id}`
|
|
187
|
+
*/
|
|
188
|
+
cancel: (params: CancelBatchParams, opts?: RequestOptions): Promise<BatchDto> =>
|
|
189
|
+
this.request(
|
|
190
|
+
{ method: "DELETE", path: "/batches/{id}", idempotent: true },
|
|
191
|
+
params,
|
|
192
|
+
opts,
|
|
193
|
+
),
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
readonly sessions = {
|
|
197
|
+
/**
|
|
198
|
+
* Start a browser session
|
|
199
|
+
*
|
|
200
|
+
* Hold a real Chromium for as long as you asked for and answer with a CDP URL your own
|
|
201
|
+
* tools connect to: `chromium.connectOverCDP(connectUrl)` in Playwright,
|
|
202
|
+
* `puppeteer.connect({ browserWSEndpoint: connectUrl })` in Puppeteer. The browser is
|
|
203
|
+
* yours alone for the whole window, every byte it pulls leaves through the exit you
|
|
204
|
+
* picked, and the socket survives our deploys. The URL carries the session's token and is
|
|
205
|
+
* shown once — it cannot be rebuilt from the id.
|
|
206
|
+
*
|
|
207
|
+
* Credits: 2 credits per minute, reserved for the whole window when the session starts and settled
|
|
208
|
+
* by the minutes actually used when it closes — so closing early gives the rest back, and
|
|
209
|
+
* a browser that never came up costs nothing at all. Managed-proxy bandwidth past the
|
|
210
|
+
* first megabyte is added at close; your own proxy is never metered.
|
|
211
|
+
*
|
|
212
|
+
* `POST /v1/sessions`
|
|
213
|
+
*/
|
|
214
|
+
create: (params?: CreateSessionParams, opts?: RequestOptions): Promise<LiveSessionDto> =>
|
|
215
|
+
this.request({ method: "POST", path: "/sessions", idempotent: false }, params, opts),
|
|
216
|
+
/**
|
|
217
|
+
* Price a session
|
|
218
|
+
*
|
|
219
|
+
* What holding a browser for that long would cost, without holding one. `maxCost` is the
|
|
220
|
+
* whole window at the per-minute rate — a ceiling, not a bill, because the minutes you do
|
|
221
|
+
* not use come back when the session closes.
|
|
222
|
+
*
|
|
223
|
+
* Credits: 0 credits — pricing a session is free.
|
|
224
|
+
*
|
|
225
|
+
* `POST /v1/sessions/quote`
|
|
226
|
+
*/
|
|
227
|
+
quote: (params?: QuoteSessionParams, opts?: RequestOptions): Promise<SessionQuoteDto> =>
|
|
228
|
+
this.request(
|
|
229
|
+
{ method: "POST", path: "/sessions/quote", idempotent: true },
|
|
230
|
+
params,
|
|
231
|
+
opts,
|
|
232
|
+
),
|
|
233
|
+
/**
|
|
234
|
+
* List your sessions
|
|
235
|
+
*
|
|
236
|
+
* Your most recent sessions, newest first — the live ones and the ones that have ended,
|
|
237
|
+
* with what each of them cost.
|
|
238
|
+
*
|
|
239
|
+
* Credits: 0 credits — listing sessions is free.
|
|
240
|
+
*
|
|
241
|
+
* `GET /v1/sessions`
|
|
242
|
+
*/
|
|
243
|
+
list: (params?: ListSessionsParams, opts?: RequestOptions): Promise<SessionDto[]> =>
|
|
244
|
+
this.request({ method: "GET", path: "/sessions", idempotent: true }, params, opts),
|
|
245
|
+
/**
|
|
246
|
+
* Read one session
|
|
247
|
+
*
|
|
248
|
+
* Where a session got to: whether it is still live, when it expires, what it has pulled
|
|
249
|
+
* and — once it has ended — what it finally cost. The connect URL is not repeated here; it
|
|
250
|
+
* is shown once, when the session starts.
|
|
251
|
+
*
|
|
252
|
+
* Credits: 0 credits — reading a session is free.
|
|
253
|
+
*
|
|
254
|
+
* `GET /v1/sessions/{id}`
|
|
255
|
+
*/
|
|
256
|
+
get: (params: GetSessionParams, opts?: RequestOptions): Promise<SessionDto> =>
|
|
257
|
+
this.request({ method: "GET", path: "/sessions/{id}", idempotent: true }, params, opts),
|
|
258
|
+
/**
|
|
259
|
+
* Close a session
|
|
260
|
+
*
|
|
261
|
+
* Stop the browser and settle the bill: the minutes you held it are kept and the rest of
|
|
262
|
+
* the reservation goes back. Safe to repeat — a session that has already ended answers
|
|
263
|
+
* with how it ended.
|
|
264
|
+
*
|
|
265
|
+
* Credits: Closing is free. It is where the session's own bill lands: the unused minutes are
|
|
266
|
+
* refunded and the managed pool's bandwidth is charged.
|
|
267
|
+
*
|
|
268
|
+
* `DELETE /v1/sessions/{id}`
|
|
269
|
+
*/
|
|
270
|
+
close: (params: CloseSessionParams, opts?: RequestOptions): Promise<SessionDto> =>
|
|
271
|
+
this.request(
|
|
272
|
+
{ method: "DELETE", path: "/sessions/{id}", idempotent: true },
|
|
273
|
+
params,
|
|
274
|
+
opts,
|
|
275
|
+
),
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
readonly usage = {
|
|
279
|
+
/**
|
|
280
|
+
* Usage this month
|
|
281
|
+
*
|
|
282
|
+
* Credits spent this billing month against the plan's grant, the purchased balance on top
|
|
283
|
+
* of it, managed-proxy megabytes against the plan's ceiling, when the month rolls over,
|
|
284
|
+
* and the same split per API key. Free, and worth reading before a bulk run.
|
|
285
|
+
*
|
|
286
|
+
* Credits: 0 credits.
|
|
287
|
+
*
|
|
288
|
+
* `GET /v1/usage`
|
|
289
|
+
*/
|
|
290
|
+
get: (opts?: RequestOptions): Promise<UsageDto> =>
|
|
291
|
+
this.request({ method: "GET", path: "/usage", idempotent: true }, undefined, opts),
|
|
292
|
+
};
|
|
293
|
+
}
|