@scrapa/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +232 -0
- package/dist/client.d.ts +47 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +182 -0
- package/dist/client.js.map +1 -0
- package/dist/errors.d.ts +50 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +56 -0
- package/dist/errors.js.map +1 -0
- package/dist/generated/contract.d.ts +470 -0
- package/dist/generated/contract.d.ts.map +1 -0
- package/dist/generated/contract.js +161 -0
- package/dist/generated/contract.js.map +1 -0
- package/dist/generated/operations.d.ts +213 -0
- package/dist/generated/operations.d.ts.map +1 -0
- package/dist/generated/operations.js +220 -0
- package/dist/generated/operations.js.map +1 -0
- package/dist/generated/params.d.ts +420 -0
- package/dist/generated/params.d.ts.map +1 -0
- package/dist/generated/params.js +7 -0
- package/dist/generated/params.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/package.json +55 -0
- package/src/client.ts +227 -0
- package/src/errors.ts +83 -0
- package/src/generated/contract.ts +564 -0
- package/src/generated/operations.ts +293 -0
- package/src/generated/params.ts +437 -0
- package/src/index.ts +9 -0
|
@@ -0,0 +1,564 @@
|
|
|
1
|
+
// GENERATED FILE — do not edit. Run `bun run sdk:gen`.
|
|
2
|
+
// Source: packages/shared/src/http/index.ts, packages/shared/src/locale/index.ts, packages/shared/src/enums/index.ts, packages/shared/src/billing/catalog.ts, packages/shared/src/dto/account.ts, packages/shared/src/dto/scrape.ts, packages/shared/src/dto/batch.ts, packages/shared/src/dto/search.ts, packages/shared/src/dto/sessions.ts, packages/shared/src/dto/usage.ts
|
|
3
|
+
//
|
|
4
|
+
// The shapes the API speaks, lifted from the server's own contract.
|
|
5
|
+
|
|
6
|
+
// ── from packages/shared/src/http/index.ts ───────────────────────────
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The closed set of error codes. A runtime list, not a bare union, so the SDK can narrow
|
|
10
|
+
* untrusted JSON into it and every surface can enumerate it (the docs errors table, the
|
|
11
|
+
* locale catalogs, exhaustiveness checks).
|
|
12
|
+
*
|
|
13
|
+
* THE WHOSE-FAULT VOCABULARY (DESIGN.md) is the organising principle: a caller's next move
|
|
14
|
+
* depends entirely on who said no, so the code says. `UPSTREAM_BLOCKED` / `NAV_TIMEOUT` are
|
|
15
|
+
* "the site said no" (retry differently — another engine, pool or country); `SSRF_DENIED` /
|
|
16
|
+
* `AUP_DENIED` are "our policy said no" (retrying cannot help, change the URL); a 5xx is
|
|
17
|
+
* "we broke it" (retry as-is, already refunded).
|
|
18
|
+
*/
|
|
19
|
+
export const ERROR_CODES = [
|
|
20
|
+
"VALIDATION_ERROR", // 400 — zod issues; details carry {path, code}[] only, never the rejected values
|
|
21
|
+
"INVALID_URL", // 400 — not an http(s) URL we can fetch; details {input, hint}
|
|
22
|
+
"SSRF_DENIED", // 400 — private/loopback/link-local/dotless host, the box itself, or a redirect there
|
|
23
|
+
"UNAUTHORIZED", // 401 — no credential presented
|
|
24
|
+
"INVALID_TOKEN", // 401 — a credential was presented but is unknown, revoked or the wrong kind
|
|
25
|
+
// 402 is PLAN_GATE alone. There is deliberately no PAYMENT_REQUIRED: every quantity
|
|
26
|
+
// scrapa sells — monthly credits, managed-proxy megabytes, concurrency, sessions —
|
|
27
|
+
// recurs, so WAITING always clears it and the honest answer is a 429 that says when.
|
|
28
|
+
// 402 is reserved for the one thing waiting cannot fix: a capability the plan does not
|
|
29
|
+
// include at all. A code with no caller is a promise in the docs nothing implements.
|
|
30
|
+
"PLAN_GATE", // 402 — a capability this plan does not include; details {gate, upgradeTo}
|
|
31
|
+
"FORBIDDEN", // 403 — suspended account / staff-only
|
|
32
|
+
"AUP_DENIED", // 403 — the target is on the acceptable-use denylist; details {category}
|
|
33
|
+
"NOT_FOUND", // 404 — generic, including "exists but not yours" (no existence leaks)
|
|
34
|
+
"SESSION_NOT_FOUND", // 404 — the browser session is unknown, closed or expired
|
|
35
|
+
"CONFLICT", // 409 — e.g. the api-key name cap race
|
|
36
|
+
"PAYLOAD_TOO_LARGE", // 413 — the request body is over the surface's limit
|
|
37
|
+
// ── 422: the request was fine, the WEB did not cooperate. Deliberately 4xx, not 5xx:
|
|
38
|
+
// "5xx is reserved for what we broke" (AGENTS.md), and these cost the caller nothing.
|
|
39
|
+
// 422 also tells an HTTP client the truth — replaying the identical request gets the
|
|
40
|
+
// identical refusal, so the fix is a different engine/pool/country, not a retry.
|
|
41
|
+
"UPSTREAM_BLOCKED", // 422 — an anti-bot wall, a captcha, a hard 403 from the target
|
|
42
|
+
"NAV_TIMEOUT", // 422 — the page never reached a usable state inside the budget
|
|
43
|
+
"RATE_LIMIT_EXCEEDED", // 429 — the per-minute burst bucket; Retry-After + details {limit}
|
|
44
|
+
"QUOTA_EXCEEDED", // 429 — a durable bound that WAITING clears; details {scope, limit, resetAt}
|
|
45
|
+
"INTERNAL_ERROR", // 500 — we broke it; the body is always generic, the cause stays server-side
|
|
46
|
+
"SERVICE_UNAVAILABLE", // 503 — a dependency (Postgres, Redis, the driver) is down; the meter fails closed here
|
|
47
|
+
"BILLING_UNAVAILABLE", // 503 — Stripe is not configured on this deployment: the catalog shows but cannot sell
|
|
48
|
+
] as const;
|
|
49
|
+
|
|
50
|
+
export type ErrorCode = (typeof ERROR_CODES)[number];
|
|
51
|
+
|
|
52
|
+
/** The status each code answers with. The server's error factory, the SDK's fallback and
|
|
53
|
+
* the docs' errors table all read this one map — three copies could drift, one cannot. */
|
|
54
|
+
export const ERROR_STATUS = {
|
|
55
|
+
VALIDATION_ERROR: 400,
|
|
56
|
+
INVALID_URL: 400,
|
|
57
|
+
SSRF_DENIED: 400,
|
|
58
|
+
UNAUTHORIZED: 401,
|
|
59
|
+
INVALID_TOKEN: 401,
|
|
60
|
+
PLAN_GATE: 402,
|
|
61
|
+
FORBIDDEN: 403,
|
|
62
|
+
AUP_DENIED: 403,
|
|
63
|
+
NOT_FOUND: 404,
|
|
64
|
+
SESSION_NOT_FOUND: 404,
|
|
65
|
+
CONFLICT: 409,
|
|
66
|
+
PAYLOAD_TOO_LARGE: 413,
|
|
67
|
+
UPSTREAM_BLOCKED: 422,
|
|
68
|
+
NAV_TIMEOUT: 422,
|
|
69
|
+
RATE_LIMIT_EXCEEDED: 429,
|
|
70
|
+
QUOTA_EXCEEDED: 429,
|
|
71
|
+
INTERNAL_ERROR: 500,
|
|
72
|
+
SERVICE_UNAVAILABLE: 503,
|
|
73
|
+
BILLING_UNAVAILABLE: 503,
|
|
74
|
+
} as const satisfies Record<ErrorCode, number>;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The three no-charge outcomes, as DESIGN.md names them. Every code that costs the caller
|
|
78
|
+
* nothing maps to exactly one — this is the map the console reads to pick the hue and the
|
|
79
|
+
* word, and the SDK reads to tell a caller what to do next. A code absent from here was
|
|
80
|
+
* the caller's own mistake (a bad body, a missing key) and gets no colour.
|
|
81
|
+
*
|
|
82
|
+
* blocked → the site said no → try another engine, pool or country
|
|
83
|
+
* denied → our policy said no → change the URL; retrying cannot help
|
|
84
|
+
* broken → we broke it → retry as-is; it was already refunded
|
|
85
|
+
*/
|
|
86
|
+
export const OUTCOME_OF: Partial<Record<ErrorCode, "blocked" | "denied" | "broken">> = {
|
|
87
|
+
UPSTREAM_BLOCKED: "blocked",
|
|
88
|
+
NAV_TIMEOUT: "blocked",
|
|
89
|
+
SSRF_DENIED: "denied",
|
|
90
|
+
AUP_DENIED: "denied",
|
|
91
|
+
INTERNAL_ERROR: "broken",
|
|
92
|
+
SERVICE_UNAVAILABLE: "broken",
|
|
93
|
+
// Stripe unconfigured is entirely our side of the line: the catalog shows and cannot
|
|
94
|
+
// sell. The customer did nothing wrong, so it is the same hue and costs the same zero.
|
|
95
|
+
BILLING_UNAVAILABLE: "broken",
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Response headers a client may read. One spelling for the server (CORS `exposeHeaders`
|
|
100
|
+
* is `Object.values(HEADERS)`), the SDK and the docs.
|
|
101
|
+
*
|
|
102
|
+
* `cost` and `creditsRemaining` ride on EVERY metered response including the free ones —
|
|
103
|
+
* a blocked page reports `x-request-cost: 0`, which is how the promise "blocked pages cost
|
|
104
|
+
* nothing" becomes something a caller can verify instead of something we assert.
|
|
105
|
+
*/
|
|
106
|
+
export const HEADERS = {
|
|
107
|
+
/** Credits this call actually cost. `0` on every no-charge outcome. */
|
|
108
|
+
cost: "x-request-cost",
|
|
109
|
+
/** Credits left after it — plan quota remaining plus the purchased balance. */
|
|
110
|
+
creditsRemaining: "x-credits-remaining",
|
|
111
|
+
requestId: "x-request-id",
|
|
112
|
+
/** Which engine answered (`fetch` | `browser`) — set from Phase 3. */
|
|
113
|
+
engine: "x-scrapa-engine",
|
|
114
|
+
/** What the SDK identifies itself as. */
|
|
115
|
+
client: "x-scrapa-client",
|
|
116
|
+
retryAfter: "retry-after",
|
|
117
|
+
} as const;
|
|
118
|
+
|
|
119
|
+
export interface PaginationMeta {
|
|
120
|
+
total: number;
|
|
121
|
+
limit: number;
|
|
122
|
+
offset: number;
|
|
123
|
+
hasMore: boolean;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Rides on every metered 2xx, mirroring the `x-request-cost` / `x-credits-remaining`
|
|
127
|
+
* headers for clients that would rather read the body. `dryRun` marks a priced-but-not-run
|
|
128
|
+
* call (`dryRun: true` on the request), which always costs 0. */
|
|
129
|
+
export interface UsageMeta {
|
|
130
|
+
/** Credits charged by THIS call. 0 on a no-charge outcome and on a dry run. */
|
|
131
|
+
cost: number;
|
|
132
|
+
creditsRemaining: number;
|
|
133
|
+
requestId: string;
|
|
134
|
+
dryRun?: true;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Success envelope — every 2xx JSON body is exactly this shape. */
|
|
138
|
+
export interface ApiSuccess<T> {
|
|
139
|
+
data: T;
|
|
140
|
+
meta?: PaginationMeta | UsageMeta;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Error envelope — byte-compatible with the server's `AppError.toJSON()`. */
|
|
144
|
+
export interface ApiError {
|
|
145
|
+
error: {
|
|
146
|
+
code: ErrorCode;
|
|
147
|
+
/** English fallback, for logs, curl and agents. Not shown when `messageKey` resolves. */
|
|
148
|
+
message: string;
|
|
149
|
+
/** Stable key for clients that localize or brand their copy of this failure. */
|
|
150
|
+
messageKey?: string;
|
|
151
|
+
/** Interpolation values for `messageKey`. */
|
|
152
|
+
params?: Record<string, string | number>;
|
|
153
|
+
details?: unknown;
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
// ── from packages/shared/src/enums/index.ts ──────────────────────────
|
|
157
|
+
|
|
158
|
+
/** Which exit a call leaves through. `residential` is the managed pool (metered bytes);
|
|
159
|
+
* `custom` is the caller's own proxy URL — free on every plan, bytes never billed.
|
|
160
|
+
* `datacenter` is a ROADMAP pool and deliberately absent: an enum value with nothing
|
|
161
|
+
* behind it is a documented promise we cannot keep. */
|
|
162
|
+
export const PROXY_POOLS = ["residential", "custom"] as const;
|
|
163
|
+
|
|
164
|
+
export type ProxyPool = (typeof PROXY_POOLS)[number];
|
|
165
|
+
|
|
166
|
+
/** How a page is fetched. `auto` tries a plain fetch and escalates to a real browser when
|
|
167
|
+
* the body looks like a JavaScript shell, billing whichever engine answered. */
|
|
168
|
+
export const ENGINES = ["auto", "fetch", "browser"] as const;
|
|
169
|
+
|
|
170
|
+
export type Engine = (typeof ENGINES)[number];
|
|
171
|
+
|
|
172
|
+
/** The engine that actually answered — never `auto`, which is a strategy, not an engine. */
|
|
173
|
+
export type ResolvedEngine = Exclude<Engine, "auto">;
|
|
174
|
+
|
|
175
|
+
/** What a scrape can hand back. `markdown` is the default because it is what an LLM, a
|
|
176
|
+
* notebook and a human all read; `html` is the page as fetched; `text` is prose with the
|
|
177
|
+
* markup gone; `links` every absolute anchor in document order. */
|
|
178
|
+
export const SCRAPE_FORMATS = ["markdown", "html", "text", "links", "screenshot", "pdf"] as const;
|
|
179
|
+
|
|
180
|
+
export type ScrapeFormat = (typeof SCRAPE_FORMATS)[number];
|
|
181
|
+
|
|
182
|
+
/** A session's life. `starting` is the driver bringing Chromium up; `live` is the only
|
|
183
|
+
* status whose `connectUrl` accepts a CDP client. The three end states answer one
|
|
184
|
+
* question — who ended it — because that is what decides whether the reservation is
|
|
185
|
+
* settled by the minutes used (`closed`, `expired`) or handed back whole (`failed`, which
|
|
186
|
+
* is ours). A session that never reached `live` never charged a minute. */
|
|
187
|
+
export const SESSION_STATUSES = ["starting", "live", "closed", "expired", "failed"] as const;
|
|
188
|
+
|
|
189
|
+
export type SessionStatus = (typeof SESSION_STATUSES)[number];
|
|
190
|
+
|
|
191
|
+
/** What a search is looking for. `general` is the web; `news` is dated journalism — a
|
|
192
|
+
* different door at every engine, and the only one that reliably carries a date. */
|
|
193
|
+
export const SEARCH_TOPICS = ["general", "news"] as const;
|
|
194
|
+
|
|
195
|
+
export type SearchTopic = (typeof SEARCH_TOPICS)[number];
|
|
196
|
+
|
|
197
|
+
/** Which front answered a search. Not a caller's choice — the ladder picks, and says which
|
|
198
|
+
* one it was, because a result set's flavour follows its index. */
|
|
199
|
+
export const SEARCH_ENGINES = ["bing", "brave"] as const;
|
|
200
|
+
|
|
201
|
+
export type SearchEngine = (typeof SEARCH_ENGINES)[number];
|
|
202
|
+
|
|
203
|
+
/** A batch's life. `queued` is accepted but not yet picked up; `running` has at least one
|
|
204
|
+
* item in flight. The three end states answer the same question the session statuses do —
|
|
205
|
+
* who ended it: `completed` ran every item to a verdict (some of which may be failures,
|
|
206
|
+
* which is not the batch failing), `canceled` is the caller's own call, and `failed` is
|
|
207
|
+
* ours and only ours. A batch does NOT fail because pages did. */
|
|
208
|
+
export const BATCH_STATUSES = ["queued", "running", "completed", "canceled", "failed"] as const;
|
|
209
|
+
|
|
210
|
+
export type BatchStatus = (typeof BATCH_STATUSES)[number];
|
|
211
|
+
|
|
212
|
+
/** One URL's life inside a batch. `done` means a page came back and was charged for;
|
|
213
|
+
* `failed` means it did not, and carries the same error a synchronous call would have
|
|
214
|
+
* thrown — including the free outcomes, which cost nothing here exactly as they do there.
|
|
215
|
+
* `skipped` is the item nobody ran: the batch was canceled, or the account's credits ran
|
|
216
|
+
* out partway through, and neither is a verdict about the URL. */
|
|
217
|
+
export const BATCH_ITEM_STATUSES = ["queued", "running", "done", "failed", "skipped"] as const;
|
|
218
|
+
|
|
219
|
+
export type BatchItemStatus = (typeof BATCH_ITEM_STATUSES)[number];
|
|
220
|
+
|
|
221
|
+
/** What we will call a webhook about. Two, because they answer two different questions:
|
|
222
|
+
* `batch.completed` is "stop polling", and `batch.item.completed` is "you can start
|
|
223
|
+
* working on this one now" — the reason a 5,000-URL batch is usable before it ends.
|
|
224
|
+
*
|
|
225
|
+
* Per-item delivery is OPT-IN and completion is the default, because the per-item choice
|
|
226
|
+
* is a caller asking for one HTTP request per URL from us and it should be a decision
|
|
227
|
+
* rather than a surprise. */
|
|
228
|
+
export const WEBHOOK_EVENTS = ["batch.completed", "batch.item.completed"] as const;
|
|
229
|
+
|
|
230
|
+
export type WebhookEvent = (typeof WEBHOOK_EVENTS)[number];
|
|
231
|
+
|
|
232
|
+
/** How a delivery ended. `delivered` is any 2xx. `failed` is every attempt spent — a
|
|
233
|
+
* non-2xx, a timeout, a refused connection — and it is terminal: we do not hold a
|
|
234
|
+
* customer's results hostage to their endpoint, the batch is readable over the API
|
|
235
|
+
* regardless. */
|
|
236
|
+
export const WEBHOOK_DELIVERY_STATUSES = ["pending", "delivered", "failed"] as const;
|
|
237
|
+
|
|
238
|
+
export type WebhookDeliveryStatus = (typeof WEBHOOK_DELIVERY_STATUSES)[number];
|
|
239
|
+
// ── from packages/shared/src/billing/catalog.ts ──────────────────────
|
|
240
|
+
|
|
241
|
+
export type PlanId = "free" | "starter" | "pro" | "scale" | "internal";
|
|
242
|
+
// ── from packages/shared/src/dto/scrape.ts ───────────────────────────
|
|
243
|
+
|
|
244
|
+
/** What the page said about itself. Every field is null when the page did not say — never
|
|
245
|
+
* guessed from the body, so a caller can trust a non-null value. */
|
|
246
|
+
export interface ScrapeMetadataDto {
|
|
247
|
+
title: string | null;
|
|
248
|
+
description: string | null;
|
|
249
|
+
/** The document's `lang`, as written (`en`, `pt-BR`). */
|
|
250
|
+
language: string | null;
|
|
251
|
+
canonicalUrl: string | null;
|
|
252
|
+
siteName: string | null;
|
|
253
|
+
ogTitle: string | null;
|
|
254
|
+
ogDescription: string | null;
|
|
255
|
+
ogImage: string | null;
|
|
256
|
+
/** The response's `content-type`, media type only. */
|
|
257
|
+
contentType: string | null;
|
|
258
|
+
/** True when the fetched body looks like a JavaScript shell — a page whose content only
|
|
259
|
+
* exists after scripts run. `auto` escalates to the browser engine on it; `fetch` reports
|
|
260
|
+
* it so a caller can retry with `engine: "browser"` instead of trusting an empty page.
|
|
261
|
+
* Always false from the browser engine, which ran the scripts. */
|
|
262
|
+
needsJs: boolean;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** `POST /v1/scrape`'s answer. Only the requested formats are present; everything else is
|
|
266
|
+
* always there. Artifacts (`screenshot`, `pdf`) are base64 in v1. */
|
|
267
|
+
export interface ScrapeResultDto {
|
|
268
|
+
/** The URL as requested. */
|
|
269
|
+
url: string;
|
|
270
|
+
/** Where the page actually was, after redirects. */
|
|
271
|
+
finalUrl: string;
|
|
272
|
+
/** The target's HTTP status. A 404 page that answered is still a page that answered. */
|
|
273
|
+
status: number;
|
|
274
|
+
engine: ResolvedEngine;
|
|
275
|
+
pool: ProxyPool;
|
|
276
|
+
/** True when the first attempt did not answer and a second one did — an `auto` call
|
|
277
|
+
* whose page turned out to need a browser, or a blocked page another renderer got
|
|
278
|
+
* past. It is why the call cost what it cost; `escalate: false` is how you stop it.
|
|
279
|
+
* Absent means the first attempt answered. */
|
|
280
|
+
escalated?: boolean;
|
|
281
|
+
/** How many times the page was asked for, when it took more than once — you asked for
|
|
282
|
+
* `retry`, the site blocked us, and a later address got through. It is why the call took
|
|
283
|
+
* as long as it did. Absent means once. */
|
|
284
|
+
attempts?: number;
|
|
285
|
+
/** What the managed pool carried for this call, and meters past the free megabyte.
|
|
286
|
+
* `fetch`: the body, decompressed — what the caller received. `browser`: every response
|
|
287
|
+
* the page pulled, as carried on the wire. An `auto` call that escalated counts both. */
|
|
288
|
+
bytes: number;
|
|
289
|
+
markdown?: string;
|
|
290
|
+
html?: string;
|
|
291
|
+
text?: string;
|
|
292
|
+
links?: string[];
|
|
293
|
+
screenshot?: string;
|
|
294
|
+
pdf?: string;
|
|
295
|
+
metadata: ScrapeMetadataDto;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** `dryRun: true`'s answer — the price of the call, without making it. */
|
|
299
|
+
export interface ScrapeQuoteDto {
|
|
300
|
+
engine: ResolvedEngine;
|
|
301
|
+
pool: ProxyPool;
|
|
302
|
+
formats: ScrapeFormat[];
|
|
303
|
+
/** Credits the engine charges when the page answers. For `auto`, the fetch row. */
|
|
304
|
+
cost: number;
|
|
305
|
+
/** The most the engine can charge before bandwidth: `cost` when the engine is fixed; the
|
|
306
|
+
* browser row for `auto`, which escalates when the page needs a real browser. */
|
|
307
|
+
maxCost: number;
|
|
308
|
+
/** Managed-pool megabytes included per request before `perMb` applies. 0 on BYO. */
|
|
309
|
+
freeMb: number;
|
|
310
|
+
/** Credits per additional megabyte through the managed pool. 0 on BYO. */
|
|
311
|
+
perMb: number;
|
|
312
|
+
}
|
|
313
|
+
// ── from packages/shared/src/dto/batch.ts ────────────────────────────
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* A batch: one set of URLs, read with one set of options, on our schedule rather than the
|
|
317
|
+
* caller's connection.
|
|
318
|
+
*
|
|
319
|
+
* The rule that shapes everything here is that **a batch is N scrapes and nothing more**.
|
|
320
|
+
* Each item is charged exactly what the same call would have cost synchronously, on the
|
|
321
|
+
* same terms — a blocked page is free inside a batch too — so there is no batch price, no
|
|
322
|
+
* discount and no surcharge to reason about. What the caller buys is not having to hold a
|
|
323
|
+
* connection open for an hour.
|
|
324
|
+
*
|
|
325
|
+
* `cost` and `bytes` therefore GROW while the batch runs. They are what it has spent so
|
|
326
|
+
* far, not an estimate and not a reservation: nothing is held up front, which is why a
|
|
327
|
+
* batch can outlive a month's credits and report honestly that it did (the remaining items
|
|
328
|
+
* go `skipped`, and skipped is not a verdict about the URL).
|
|
329
|
+
*/
|
|
330
|
+
export interface BatchDto {
|
|
331
|
+
id: string;
|
|
332
|
+
status: BatchStatus;
|
|
333
|
+
/** URLs accepted. Fixed at create — a batch never grows. */
|
|
334
|
+
total: number;
|
|
335
|
+
/** Items that have reached an end state, `skipped` included. `completed === total` is
|
|
336
|
+
* what makes the batch itself complete. */
|
|
337
|
+
completed: number;
|
|
338
|
+
/** Of those, how many the web or our policy refused. A `failed` count equal to `total`
|
|
339
|
+
* is still a `completed` batch: every URL got a verdict, and the verdicts were
|
|
340
|
+
* refusals. Free ones — a blocked page costs nothing here too. */
|
|
341
|
+
failed: number;
|
|
342
|
+
/** Of those, how many were never attempted — the batch was cancelled, or the month's
|
|
343
|
+
* credits ran out under it. Counted APART from `failed` because it is not a verdict
|
|
344
|
+
* about the URL: submit these positions again and they will be read. */
|
|
345
|
+
skipped: number;
|
|
346
|
+
/** Which exit every item leaves through, fixed at create like a session's. */
|
|
347
|
+
pool: ProxyPool;
|
|
348
|
+
/** Credits spent SO FAR. Grows while the batch runs. */
|
|
349
|
+
cost: number;
|
|
350
|
+
/** Managed-pool bytes so far, on the same terms as a synchronous scrape's. */
|
|
351
|
+
bytes: number;
|
|
352
|
+
createdAt: string;
|
|
353
|
+
/** When the first item was picked up. */
|
|
354
|
+
startedAt?: string;
|
|
355
|
+
endedAt?: string;
|
|
356
|
+
/** When the RESULTS stop being readable. The batch's own row OUTLIVES them, which is
|
|
357
|
+
* why there is no expiry error code: a batch past this date still answers with its
|
|
358
|
+
* status, its counts and what it cost, just with no `items`. Somebody who paid for
|
|
359
|
+
* five thousand pages must never be told "not found". */
|
|
360
|
+
expiresAt: string;
|
|
361
|
+
/** The items, when they were asked for. Absent — not empty — when they were not:
|
|
362
|
+
* an empty array would read like a batch with no URLs. */
|
|
363
|
+
items?: BatchItemDto[];
|
|
364
|
+
/** Where we will call back, when one was configured. The secret is never echoed. */
|
|
365
|
+
webhook?: BatchWebhookDto;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/** One URL's outcome. Exactly the answer the same URL would have given synchronously,
|
|
369
|
+
* including which of the two ways it could refuse. */
|
|
370
|
+
export interface BatchItemDto {
|
|
371
|
+
/** Position in the submitted list, 0-based. The list's ORDER is preserved — it is how
|
|
372
|
+
* a caller lines results back up against whatever they submitted. */
|
|
373
|
+
position: number;
|
|
374
|
+
url: string;
|
|
375
|
+
status: BatchItemStatus;
|
|
376
|
+
/** The page, when one answered. The same DTO `scrape()` returns. */
|
|
377
|
+
result?: ScrapeResultDto;
|
|
378
|
+
/** Why not, when it did not — the same code and sentence a synchronous call would have
|
|
379
|
+
* thrown, so one error handler serves both doors. */
|
|
380
|
+
error?: { code: ErrorCode; message: string };
|
|
381
|
+
/** What this item cost. 0 on every free outcome, exactly as synchronously. */
|
|
382
|
+
cost: number;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/** The callback a batch was created with. */
|
|
386
|
+
export interface BatchWebhookDto {
|
|
387
|
+
url: string;
|
|
388
|
+
events: WebhookEvent[];
|
|
389
|
+
/** The most recent attempt's verdict, so a caller debugging a silent endpoint can see
|
|
390
|
+
* whether we ever reached it without opening a support ticket. */
|
|
391
|
+
status: WebhookDeliveryStatus;
|
|
392
|
+
/** The last attempt's HTTP status, when we got one at all. Absent means we never
|
|
393
|
+
* reached the endpoint — DNS, TLS, a refused connection or a timeout. */
|
|
394
|
+
lastStatus?: number;
|
|
395
|
+
lastAttemptAt?: string;
|
|
396
|
+
/** Attempts spent on the MOST RECENT delivery, which is what `lastStatus` describes too.
|
|
397
|
+
* A delivery is retried on a schedule; this is where that one got to. With per-item
|
|
398
|
+
* events there are many deliveries against one batch, and this is not their sum. */
|
|
399
|
+
attempts: number;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** `POST /v1/batches/quote` — the ceiling, not a bill. Every item is priced as the same
|
|
403
|
+
* synchronous call would be, so this is `total` × that, and the real cost is lower by
|
|
404
|
+
* whatever the web refuses for free. */
|
|
405
|
+
export interface BatchQuoteDto {
|
|
406
|
+
total: number;
|
|
407
|
+
pool: ProxyPool;
|
|
408
|
+
/** What one item would cost at most — the browser row for `auto`, since that is the
|
|
409
|
+
* ceiling a page can escalate to. */
|
|
410
|
+
perItemMaxCost: number;
|
|
411
|
+
/** `total` × `perItemMaxCost`. Bandwidth is on top and cannot be quoted: it depends on
|
|
412
|
+
* what the pages weigh. */
|
|
413
|
+
maxCost: number;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* What creating a batch answers with: the batch, plus the one thing that is never said
|
|
418
|
+
* again.
|
|
419
|
+
*
|
|
420
|
+
* We mint the signing secret rather than taking one, so a customer never puts a credential
|
|
421
|
+
* on the wire to us, and we keep it because signing a delivery needs the key itself. What
|
|
422
|
+
* they get is this single sight of it — the same bargain `sc_` keys make. Losing it means
|
|
423
|
+
* creating another batch, not reading it back.
|
|
424
|
+
*/
|
|
425
|
+
export interface CreatedBatchDto extends BatchDto {
|
|
426
|
+
/** `whsec_…`, present exactly when a webhook was configured. Verify a delivery with it:
|
|
427
|
+
* HMAC-SHA256 over `<timestamp>.<raw body>`, compared against the `v1=` value in the
|
|
428
|
+
* `scrapa-signature` header. */
|
|
429
|
+
webhookSecret?: string;
|
|
430
|
+
}
|
|
431
|
+
// ── from packages/shared/src/dto/search.ts ───────────────────────────
|
|
432
|
+
|
|
433
|
+
/** One hit, as the index ranked it. The array's ORDER is the ranking — there is no score,
|
|
434
|
+
* because no engine we read publishes one and a rank-derived float would be a made-up
|
|
435
|
+
* number in a public contract. */
|
|
436
|
+
export interface SearchHitDto {
|
|
437
|
+
/** 1-based rank within this answer. */
|
|
438
|
+
position: number;
|
|
439
|
+
url: string;
|
|
440
|
+
title: string;
|
|
441
|
+
/** The engine's own summary of the page, a couple of lines. Never the page itself —
|
|
442
|
+
* read that with `POST /v1/scrape` on the URL. */
|
|
443
|
+
snippet: string;
|
|
444
|
+
/** When the engine dated it, ISO 8601. Null whenever it did not, which on the general
|
|
445
|
+
* topic is most of the time — never guessed from the snippet. */
|
|
446
|
+
publishedAt: string | null;
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/** `POST /v1/search`'s answer. */
|
|
450
|
+
export interface SearchResultDto {
|
|
451
|
+
/** The query as asked, before any engine operator was added to it. */
|
|
452
|
+
query: string;
|
|
453
|
+
topic: SearchTopic;
|
|
454
|
+
/** Which front answered. */
|
|
455
|
+
engine: SearchEngine;
|
|
456
|
+
pool: ProxyPool;
|
|
457
|
+
/** What the managed pool carried, decompressed. 0 when the cheap rung answered, which
|
|
458
|
+
* for the common query it does — a SERP is not billed bandwidth. */
|
|
459
|
+
bytes: number;
|
|
460
|
+
/** True when this answer came from the short-lived cache of an identical request, in
|
|
461
|
+
* which case it cost nothing. */
|
|
462
|
+
cached: boolean;
|
|
463
|
+
/** The one-paragraph answer, when `answer: true` was asked for. Written by a model over
|
|
464
|
+
* the hits below, in the query's language, and never a substitute for reading them. */
|
|
465
|
+
answer: string | null;
|
|
466
|
+
results: SearchHitDto[];
|
|
467
|
+
}
|
|
468
|
+
// ── from packages/shared/src/dto/sessions.ts ─────────────────────────
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* A managed browser session: one Chromium PROCESS, held for the caller, driven over raw CDP
|
|
472
|
+
* by their own Playwright or Puppeteer. We hand back a URL, not a client — everything a
|
|
473
|
+
* customer already knows how to do with `connectOverCDP` keeps working, which is the whole
|
|
474
|
+
* point of the product.
|
|
475
|
+
*/
|
|
476
|
+
export interface SessionDto {
|
|
477
|
+
id: string;
|
|
478
|
+
status: SessionStatus;
|
|
479
|
+
/** Where a CDP client connects: `wss://api.scrapa.app/v1/sessions/<id>/cdp?token=…`.
|
|
480
|
+
* Present only while the session can still be connected to — an ended session's URL
|
|
481
|
+
* would be a link that looks alive and is not. The token is in the URL because that is
|
|
482
|
+
* the only place a WebSocket client can put a credential. */
|
|
483
|
+
connectUrl?: string;
|
|
484
|
+
/** Which exit every byte of this session leaves through, fixed at create. A tenant
|
|
485
|
+
* cannot change it later: the one CDP command we rewrite is the one that would try. */
|
|
486
|
+
pool: ProxyPool;
|
|
487
|
+
/** Minutes reserved at create, and the wall the reaper enforces. */
|
|
488
|
+
timeoutMin: number;
|
|
489
|
+
/** When the reaper will kill it, if nothing else does. */
|
|
490
|
+
expiresAt: string;
|
|
491
|
+
/** Credits held at create: `timeoutMin` × the session-minute row. Not yet spent. */
|
|
492
|
+
creditsReserved: number;
|
|
493
|
+
/** What the session actually cost, once it ended. Absent while it is still running —
|
|
494
|
+
* a running session has no final price, and a zero would read like a free one. */
|
|
495
|
+
creditsSettled?: number;
|
|
496
|
+
/** Managed-pool bytes this session pulled. Metered at settle, like a scrape's. */
|
|
497
|
+
bytes: number;
|
|
498
|
+
createdAt: string;
|
|
499
|
+
/** When Chromium came up and the URL started accepting clients. */
|
|
500
|
+
startedAt?: string;
|
|
501
|
+
endedAt?: string;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* What starting a session answers with: the same session, except that its URL is not
|
|
506
|
+
* optional. A create only returns once Chromium is up, so the one call whose whole point is
|
|
507
|
+
* the connect URL hands back a type that has one — `chromium.connectOverCDP(session.connectUrl)`
|
|
508
|
+
* with no `!` and no check for a field that cannot be missing.
|
|
509
|
+
*/
|
|
510
|
+
export interface LiveSessionDto extends SessionDto {
|
|
511
|
+
connectUrl: string;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/** `POST /v1/sessions` with `dryRun: true` — what holding a session would cost, without
|
|
515
|
+
* starting one. The reservation is the whole `timeoutMin`; closing early hands the rest
|
|
516
|
+
* back, so this is a ceiling, not a bill. */
|
|
517
|
+
export interface SessionQuoteDto {
|
|
518
|
+
pool: ProxyPool;
|
|
519
|
+
timeoutMin: number;
|
|
520
|
+
/** Credits per minute held. */
|
|
521
|
+
perMinute: number;
|
|
522
|
+
/** `timeoutMin` × `perMinute` — reserved at create, settled by minutes used at close. */
|
|
523
|
+
maxCost: number;
|
|
524
|
+
/** Managed-pool megabytes included before `perMb` applies. 0 on BYO. */
|
|
525
|
+
freeMb: number;
|
|
526
|
+
/** Credits per additional megabyte through the managed pool. 0 on BYO. */
|
|
527
|
+
perMb: number;
|
|
528
|
+
}
|
|
529
|
+
// ── from packages/shared/src/dto/usage.ts ────────────────────────────
|
|
530
|
+
|
|
531
|
+
/** One key's share of the month, for the console's per-key breakdown. Revoked keys stay in
|
|
532
|
+
* the list: their spend still counts toward the account's month, and hiding them would make
|
|
533
|
+
* the numbers stop adding up. */
|
|
534
|
+
export interface UsageKeyDto {
|
|
535
|
+
id: string;
|
|
536
|
+
name: string;
|
|
537
|
+
prefix: string;
|
|
538
|
+
revoked: boolean;
|
|
539
|
+
/** Credits this key spent in the current billing month. */
|
|
540
|
+
credits: number;
|
|
541
|
+
/** Metered calls it made, free outcomes included — `calls` minus `credits` is where a
|
|
542
|
+
* customer sees how much of their month cost them nothing. */
|
|
543
|
+
calls: number;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/** What `GET /v1/usage` and the console's meter both answer. One shape, two doors. */
|
|
547
|
+
export interface UsageDto {
|
|
548
|
+
plan: PlanId;
|
|
549
|
+
/** The billing month this covers, `YYYY-MM`. */
|
|
550
|
+
month: string;
|
|
551
|
+
/** Credits spent against the plan quota this month. */
|
|
552
|
+
creditsUsed: number;
|
|
553
|
+
/** The plan's monthly grant. */
|
|
554
|
+
creditsIncluded: number;
|
|
555
|
+
/** Purchased credits left over (packs never expire). A separate pool from the quota:
|
|
556
|
+
* collapsing the two would tell someone on packs they have 0 left while calls succeed. */
|
|
557
|
+
creditBalance: number;
|
|
558
|
+
/** Managed-proxy bytes used this month, MB, against the plan's hard ceiling. */
|
|
559
|
+
proxyMbUsed: number;
|
|
560
|
+
proxyMbIncluded: number;
|
|
561
|
+
/** When the quota rolls over — ISO. There is no cron: the month key simply changes. */
|
|
562
|
+
resetAt: string;
|
|
563
|
+
keys: UsageKeyDto[];
|
|
564
|
+
}
|