@dealcrawl/sdk 2.1.0 → 2.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +2335 -0
- package/dist/index.d.ts +2314 -37
- package/dist/index.js +2280 -64
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2260 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +13 -5
- package/dist/client.d.ts +0 -285
- package/dist/client.d.ts.map +0 -1
- package/dist/client.js +0 -336
- package/dist/client.js.map +0 -1
- package/dist/error.d.ts +0 -55
- package/dist/error.d.ts.map +0 -1
- package/dist/error.js +0 -128
- package/dist/error.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/resources/account.d.ts +0 -143
- package/dist/resources/account.d.ts.map +0 -1
- package/dist/resources/account.js +0 -186
- package/dist/resources/account.js.map +0 -1
- package/dist/resources/crawl.d.ts +0 -101
- package/dist/resources/crawl.d.ts.map +0 -1
- package/dist/resources/crawl.js +0 -234
- package/dist/resources/crawl.js.map +0 -1
- package/dist/resources/data.d.ts +0 -157
- package/dist/resources/data.d.ts.map +0 -1
- package/dist/resources/data.js +0 -245
- package/dist/resources/data.js.map +0 -1
- package/dist/resources/dork.d.ts +0 -104
- package/dist/resources/dork.d.ts.map +0 -1
- package/dist/resources/dork.js +0 -163
- package/dist/resources/dork.js.map +0 -1
- package/dist/resources/extract.d.ts +0 -105
- package/dist/resources/extract.d.ts.map +0 -1
- package/dist/resources/extract.js +0 -246
- package/dist/resources/extract.js.map +0 -1
- package/dist/resources/index.d.ts +0 -14
- package/dist/resources/index.d.ts.map +0 -1
- package/dist/resources/index.js +0 -14
- package/dist/resources/index.js.map +0 -1
- package/dist/resources/keys.d.ts +0 -124
- package/dist/resources/keys.d.ts.map +0 -1
- package/dist/resources/keys.js +0 -168
- package/dist/resources/keys.js.map +0 -1
- package/dist/resources/scrape.d.ts +0 -53
- package/dist/resources/scrape.d.ts.map +0 -1
- package/dist/resources/scrape.js +0 -85
- package/dist/resources/scrape.js.map +0 -1
- package/dist/resources/status.d.ts +0 -100
- package/dist/resources/status.d.ts.map +0 -1
- package/dist/resources/status.js +0 -133
- package/dist/resources/status.js.map +0 -1
- package/dist/resources/webhooks.d.ts +0 -126
- package/dist/resources/webhooks.d.ts.map +0 -1
- package/dist/resources/webhooks.js +0 -167
- package/dist/resources/webhooks.js.map +0 -1
- package/dist/types/config.d.ts +0 -45
- package/dist/types/config.d.ts.map +0 -1
- package/dist/types/config.js +0 -10
- package/dist/types/config.js.map +0 -1
- package/dist/types/index.d.ts +0 -8
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -8
- package/dist/types/index.js.map +0 -1
- package/dist/types/options.d.ts +0 -328
- package/dist/types/options.d.ts.map +0 -1
- package/dist/types/options.js +0 -6
- package/dist/types/options.js.map +0 -1
- package/dist/types/responses.d.ts +0 -422
- package/dist/types/responses.d.ts.map +0 -1
- package/dist/types/responses.js +0 -6
- package/dist/types/responses.js.map +0 -1
- package/dist/types/shared.d.ts +0 -234
- package/dist/types/shared.d.ts.map +0 -1
- package/dist/types/shared.js +0 -37
- package/dist/types/shared.js.map +0 -1
- package/dist/utils/polling.d.ts +0 -57
- package/dist/utils/polling.d.ts.map +0 -1
- package/dist/utils/polling.js +0 -110
- package/dist/utils/polling.js.map +0 -1
- package/dist/utils/request.d.ts +0 -47
- package/dist/utils/request.d.ts.map +0 -1
- package/dist/utils/request.js +0 -192
- package/dist/utils/request.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,65 +1,2281 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
*/
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
|
+
|
|
5
|
+
// src/types/config.ts
|
|
6
|
+
var DEFAULT_CONFIG = {
|
|
7
|
+
baseUrl: "https://api.dealcrawl.dev",
|
|
8
|
+
timeout: 3e4,
|
|
9
|
+
maxRetries: 3,
|
|
10
|
+
retryDelay: 1e3
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
// src/error.ts
|
|
14
|
+
var ERROR_CODES = {
|
|
15
|
+
// Authentication
|
|
16
|
+
INVALID_API_KEY: "INVALID_API_KEY",
|
|
17
|
+
MISSING_API_KEY: "MISSING_API_KEY",
|
|
18
|
+
API_KEY_EXPIRED: "API_KEY_EXPIRED",
|
|
19
|
+
ACCOUNT_SUSPENDED: "ACCOUNT_SUSPENDED",
|
|
20
|
+
// Rate limiting
|
|
21
|
+
RATE_LIMIT_EXCEEDED: "RATE_LIMIT_EXCEEDED",
|
|
22
|
+
QUOTA_EXCEEDED: "QUOTA_EXCEEDED",
|
|
23
|
+
// Validation
|
|
24
|
+
INVALID_URL: "INVALID_URL",
|
|
25
|
+
INVALID_REQUEST: "INVALID_REQUEST",
|
|
26
|
+
MISSING_REQUIRED_FIELD: "MISSING_REQUIRED_FIELD",
|
|
27
|
+
// Job errors
|
|
28
|
+
JOB_NOT_FOUND: "JOB_NOT_FOUND",
|
|
29
|
+
JOB_FAILED: "JOB_FAILED",
|
|
30
|
+
JOB_TIMEOUT: "JOB_TIMEOUT",
|
|
31
|
+
// Scraping errors
|
|
32
|
+
FETCH_FAILED: "FETCH_FAILED",
|
|
33
|
+
PARSE_FAILED: "PARSE_FAILED",
|
|
34
|
+
BLOCKED_BY_ROBOTS: "BLOCKED_BY_ROBOTS",
|
|
35
|
+
CAPTCHA_DETECTED: "CAPTCHA_DETECTED",
|
|
36
|
+
SITE_UNREACHABLE: "SITE_UNREACHABLE",
|
|
37
|
+
// System errors
|
|
38
|
+
INTERNAL_ERROR: "INTERNAL_ERROR",
|
|
39
|
+
SERVICE_UNAVAILABLE: "SERVICE_UNAVAILABLE",
|
|
40
|
+
REDIS_ERROR: "REDIS_ERROR",
|
|
41
|
+
DATABASE_ERROR: "DATABASE_ERROR"
|
|
42
|
+
};
|
|
43
|
+
var DealCrawlError = class _DealCrawlError extends Error {
|
|
44
|
+
/** Error code from ERROR_CODES */
|
|
45
|
+
code;
|
|
46
|
+
/** HTTP status code from the API response */
|
|
47
|
+
statusCode;
|
|
48
|
+
/** Additional error details */
|
|
49
|
+
details;
|
|
50
|
+
/** Retry-After header value in seconds (for rate limiting) */
|
|
51
|
+
retryAfter;
|
|
52
|
+
constructor(options) {
|
|
53
|
+
super(options.message);
|
|
54
|
+
this.name = "DealCrawlError";
|
|
55
|
+
this.code = options.code;
|
|
56
|
+
this.statusCode = options.statusCode ?? 500;
|
|
57
|
+
this.details = options.details;
|
|
58
|
+
this.retryAfter = options.retryAfter;
|
|
59
|
+
if (Error.captureStackTrace) {
|
|
60
|
+
Error.captureStackTrace(this, _DealCrawlError);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Check if the error is retryable
|
|
65
|
+
* Rate limits and transient errors are retryable
|
|
66
|
+
*/
|
|
67
|
+
isRetryable() {
|
|
68
|
+
const retryableCodes = [
|
|
69
|
+
ERROR_CODES.RATE_LIMIT_EXCEEDED,
|
|
70
|
+
ERROR_CODES.SERVICE_UNAVAILABLE,
|
|
71
|
+
ERROR_CODES.REDIS_ERROR,
|
|
72
|
+
ERROR_CODES.DATABASE_ERROR
|
|
73
|
+
];
|
|
74
|
+
return retryableCodes.includes(this.code);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Check if the error is due to authentication issues
|
|
78
|
+
*/
|
|
79
|
+
isAuthError() {
|
|
80
|
+
const authCodes = [
|
|
81
|
+
ERROR_CODES.INVALID_API_KEY,
|
|
82
|
+
ERROR_CODES.MISSING_API_KEY,
|
|
83
|
+
ERROR_CODES.API_KEY_EXPIRED,
|
|
84
|
+
ERROR_CODES.ACCOUNT_SUSPENDED
|
|
85
|
+
];
|
|
86
|
+
return authCodes.includes(this.code);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Check if the error is due to rate limiting
|
|
90
|
+
*/
|
|
91
|
+
isRateLimited() {
|
|
92
|
+
return this.code === ERROR_CODES.RATE_LIMIT_EXCEEDED;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Check if the error is due to quota exceeded
|
|
96
|
+
*/
|
|
97
|
+
isQuotaExceeded() {
|
|
98
|
+
return this.code === ERROR_CODES.QUOTA_EXCEEDED;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Convert error to JSON-serializable object
|
|
102
|
+
*/
|
|
103
|
+
toJSON() {
|
|
104
|
+
return {
|
|
105
|
+
name: this.name,
|
|
106
|
+
code: this.code,
|
|
107
|
+
message: this.message,
|
|
108
|
+
statusCode: this.statusCode,
|
|
109
|
+
details: this.details,
|
|
110
|
+
retryAfter: this.retryAfter
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Create a DealCrawlError from an API error response
|
|
115
|
+
*/
|
|
116
|
+
static fromResponse(statusCode, body, retryAfter) {
|
|
117
|
+
const code = body.code || mapStatusCodeToErrorCode(statusCode);
|
|
118
|
+
const message = body.message || body.error || "An error occurred";
|
|
119
|
+
let parsedRetryAfter;
|
|
120
|
+
if (retryAfter !== null && retryAfter !== void 0) {
|
|
121
|
+
parsedRetryAfter = typeof retryAfter === "string" ? parseInt(retryAfter, 10) : retryAfter;
|
|
122
|
+
if (isNaN(parsedRetryAfter)) {
|
|
123
|
+
parsedRetryAfter = void 0;
|
|
124
|
+
}
|
|
125
|
+
} else if (body.retryAfter !== void 0) {
|
|
126
|
+
parsedRetryAfter = body.retryAfter;
|
|
127
|
+
}
|
|
128
|
+
return new _DealCrawlError({
|
|
129
|
+
code,
|
|
130
|
+
message,
|
|
131
|
+
statusCode,
|
|
132
|
+
details: body.details,
|
|
133
|
+
retryAfter: parsedRetryAfter
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
function mapStatusCodeToErrorCode(statusCode) {
|
|
138
|
+
switch (statusCode) {
|
|
139
|
+
case 400:
|
|
140
|
+
return ERROR_CODES.INVALID_REQUEST;
|
|
141
|
+
case 401:
|
|
142
|
+
return ERROR_CODES.INVALID_API_KEY;
|
|
143
|
+
case 403:
|
|
144
|
+
return ERROR_CODES.ACCOUNT_SUSPENDED;
|
|
145
|
+
case 404:
|
|
146
|
+
return ERROR_CODES.JOB_NOT_FOUND;
|
|
147
|
+
case 429:
|
|
148
|
+
return ERROR_CODES.RATE_LIMIT_EXCEEDED;
|
|
149
|
+
case 500:
|
|
150
|
+
return ERROR_CODES.INTERNAL_ERROR;
|
|
151
|
+
case 503:
|
|
152
|
+
return ERROR_CODES.SERVICE_UNAVAILABLE;
|
|
153
|
+
default:
|
|
154
|
+
return ERROR_CODES.INTERNAL_ERROR;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// src/utils/request.ts
|
|
159
|
+
function buildQueryString(params) {
|
|
160
|
+
const entries = Object.entries(params).filter(([, value]) => value !== void 0).map(
|
|
161
|
+
([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`
|
|
162
|
+
);
|
|
163
|
+
return entries.length > 0 ? `?${entries.join("&")}` : "";
|
|
164
|
+
}
|
|
165
|
+
function parseRateLimitHeaders(headers) {
|
|
166
|
+
return {
|
|
167
|
+
limit: parseInt(headers.get("X-RateLimit-Limit") || "0", 10) || 0,
|
|
168
|
+
remaining: parseInt(headers.get("X-RateLimit-Remaining") || "0", 10) || 0,
|
|
169
|
+
reset: parseInt(headers.get("X-RateLimit-Reset") || "0", 10) || 0
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
function sleep(ms) {
|
|
173
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
174
|
+
}
|
|
175
|
+
async function request(ctx, path, options = {}) {
|
|
176
|
+
const { method = "GET", body, query, timeout, signal } = options;
|
|
177
|
+
const queryString = query ? buildQueryString(query) : "";
|
|
178
|
+
const url = `${ctx.baseUrl}${path}${queryString}`;
|
|
179
|
+
const timeoutMs = timeout ?? ctx.timeout;
|
|
180
|
+
let currentController = null;
|
|
181
|
+
const onAbort = () => {
|
|
182
|
+
if (currentController) {
|
|
183
|
+
currentController.abort();
|
|
184
|
+
}
|
|
185
|
+
};
|
|
186
|
+
if (signal) {
|
|
187
|
+
signal.addEventListener("abort", onAbort);
|
|
188
|
+
}
|
|
189
|
+
const cleanupExternalSignal = () => {
|
|
190
|
+
if (signal) {
|
|
191
|
+
signal.removeEventListener("abort", onAbort);
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
let lastError = null;
|
|
195
|
+
let attempt = 0;
|
|
196
|
+
while (attempt <= ctx.maxRetries) {
|
|
197
|
+
const controller = new AbortController();
|
|
198
|
+
currentController = controller;
|
|
199
|
+
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
200
|
+
let attemptCleanupDone = false;
|
|
201
|
+
const cleanupAttempt = () => {
|
|
202
|
+
if (attemptCleanupDone) return;
|
|
203
|
+
attemptCleanupDone = true;
|
|
204
|
+
clearTimeout(timeoutId);
|
|
205
|
+
currentController = null;
|
|
206
|
+
};
|
|
207
|
+
try {
|
|
208
|
+
const response = await fetch(url, {
|
|
209
|
+
method,
|
|
210
|
+
headers: {
|
|
211
|
+
"Content-Type": "application/json",
|
|
212
|
+
Authorization: `Bearer ${ctx.apiKey}`
|
|
213
|
+
},
|
|
214
|
+
body: body ? JSON.stringify(body) : void 0,
|
|
215
|
+
signal: controller.signal
|
|
216
|
+
});
|
|
217
|
+
cleanupAttempt();
|
|
218
|
+
const rateLimit = parseRateLimitHeaders(response.headers);
|
|
219
|
+
if (response.status === 429) {
|
|
220
|
+
const retryAfter = parseInt(
|
|
221
|
+
response.headers.get("Retry-After") || "60",
|
|
222
|
+
10
|
|
223
|
+
);
|
|
224
|
+
if (ctx.onRateLimit) {
|
|
225
|
+
ctx.onRateLimit(rateLimit);
|
|
226
|
+
}
|
|
227
|
+
throw new DealCrawlError({
|
|
228
|
+
code: ERROR_CODES.RATE_LIMIT_EXCEEDED,
|
|
229
|
+
message: "Rate limit exceeded",
|
|
230
|
+
statusCode: 429,
|
|
231
|
+
retryAfter,
|
|
232
|
+
details: { rateLimit }
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
if (!response.ok) {
|
|
236
|
+
let errorBody = {};
|
|
237
|
+
try {
|
|
238
|
+
errorBody = await response.json();
|
|
239
|
+
} catch {
|
|
240
|
+
}
|
|
241
|
+
const retryAfterHeader = response.headers.get("Retry-After");
|
|
242
|
+
throw DealCrawlError.fromResponse(
|
|
243
|
+
response.status,
|
|
244
|
+
errorBody,
|
|
245
|
+
retryAfterHeader
|
|
246
|
+
);
|
|
247
|
+
}
|
|
248
|
+
let data;
|
|
249
|
+
const contentType = response.headers.get("Content-Type") || "";
|
|
250
|
+
if (contentType.includes("application/json")) {
|
|
251
|
+
data = await response.json();
|
|
252
|
+
} else if (contentType.includes("text/csv") || contentType.includes("text/plain")) {
|
|
253
|
+
data = await response.text();
|
|
254
|
+
} else {
|
|
255
|
+
data = await response.json();
|
|
256
|
+
}
|
|
257
|
+
cleanupExternalSignal();
|
|
258
|
+
return { data, rateLimit };
|
|
259
|
+
} catch (error) {
|
|
260
|
+
cleanupAttempt();
|
|
261
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
262
|
+
cleanupExternalSignal();
|
|
263
|
+
throw new DealCrawlError({
|
|
264
|
+
code: ERROR_CODES.JOB_TIMEOUT,
|
|
265
|
+
message: `Request timeout after ${timeoutMs}ms`,
|
|
266
|
+
statusCode: 408
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
if (error instanceof DealCrawlError) {
|
|
270
|
+
lastError = error;
|
|
271
|
+
if (error.isRetryable() && attempt < ctx.maxRetries) {
|
|
272
|
+
attempt++;
|
|
273
|
+
await sleep(ctx.retryDelay * attempt);
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
cleanupExternalSignal();
|
|
277
|
+
throw error;
|
|
278
|
+
}
|
|
279
|
+
cleanupExternalSignal();
|
|
280
|
+
throw new DealCrawlError({
|
|
281
|
+
code: ERROR_CODES.FETCH_FAILED,
|
|
282
|
+
message: error instanceof Error ? error.message : "Network request failed",
|
|
283
|
+
statusCode: 0
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
cleanupExternalSignal();
|
|
288
|
+
throw lastError ?? new DealCrawlError({
|
|
289
|
+
code: ERROR_CODES.INTERNAL_ERROR,
|
|
290
|
+
message: "Request failed after retries",
|
|
291
|
+
statusCode: 500
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
function get(ctx, path, query) {
|
|
295
|
+
return request(ctx, path, { method: "GET", query });
|
|
296
|
+
}
|
|
297
|
+
function post(ctx, path, body) {
|
|
298
|
+
return request(ctx, path, { method: "POST", body });
|
|
299
|
+
}
|
|
300
|
+
function patch(ctx, path, body) {
|
|
301
|
+
return request(ctx, path, { method: "PATCH", body });
|
|
302
|
+
}
|
|
303
|
+
function del(ctx, path, body) {
|
|
304
|
+
return request(ctx, path, { method: "DELETE", body });
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
// src/utils/polling.ts
|
|
308
|
+
async function waitForResult(ctx, jobId, options = {}) {
|
|
309
|
+
const {
|
|
310
|
+
pollInterval = 2e3,
|
|
311
|
+
timeout = 3e5,
|
|
312
|
+
// 5 minutes
|
|
313
|
+
onProgress,
|
|
314
|
+
onStatusChange,
|
|
315
|
+
signal
|
|
316
|
+
} = options;
|
|
317
|
+
const startTime = Date.now();
|
|
318
|
+
let lastStatus = null;
|
|
319
|
+
while (true) {
|
|
320
|
+
if (signal?.aborted) {
|
|
321
|
+
throw new DealCrawlError({
|
|
322
|
+
code: ERROR_CODES.JOB_TIMEOUT,
|
|
323
|
+
message: "Wait cancelled by user",
|
|
324
|
+
statusCode: 0
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
const elapsed = Date.now() - startTime;
|
|
328
|
+
if (elapsed >= timeout) {
|
|
329
|
+
throw new DealCrawlError({
|
|
330
|
+
code: ERROR_CODES.JOB_TIMEOUT,
|
|
331
|
+
message: `Job did not complete within ${timeout}ms`,
|
|
332
|
+
statusCode: 408,
|
|
333
|
+
details: { jobId, elapsed }
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
const { data: status } = await get(
|
|
337
|
+
ctx,
|
|
338
|
+
`/v1/status/${jobId}`
|
|
339
|
+
);
|
|
340
|
+
if (onProgress) {
|
|
341
|
+
onProgress(status);
|
|
342
|
+
}
|
|
343
|
+
if (onStatusChange && lastStatus !== null && lastStatus !== status.status) {
|
|
344
|
+
onStatusChange(status.status, lastStatus);
|
|
345
|
+
}
|
|
346
|
+
lastStatus = status.status;
|
|
347
|
+
if (status.status === "completed") {
|
|
348
|
+
return {
|
|
349
|
+
jobId,
|
|
350
|
+
status: "completed",
|
|
351
|
+
result: status.result,
|
|
352
|
+
waitTime: Date.now() - startTime
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
if (status.status === "failed") {
|
|
356
|
+
return {
|
|
357
|
+
jobId,
|
|
358
|
+
status: "failed",
|
|
359
|
+
error: status.error,
|
|
360
|
+
waitTime: Date.now() - startTime
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
await sleep2(pollInterval);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
async function waitForAll(ctx, jobIds, options = {}) {
|
|
367
|
+
const promises = jobIds.map((jobId) => waitForResult(ctx, jobId, options));
|
|
368
|
+
return Promise.all(promises);
|
|
369
|
+
}
|
|
370
|
+
async function waitForAny(ctx, jobIds, options = {}) {
|
|
371
|
+
const promises = jobIds.map((jobId) => waitForResult(ctx, jobId, options));
|
|
372
|
+
return Promise.race(promises);
|
|
373
|
+
}
|
|
374
|
+
async function pollUntil(fetchFn, conditionFn, options = {}) {
|
|
375
|
+
const { pollInterval = 2e3, timeout = 3e5, signal } = options;
|
|
376
|
+
const startTime = Date.now();
|
|
377
|
+
while (true) {
|
|
378
|
+
if (signal?.aborted) {
|
|
379
|
+
throw new Error("Polling cancelled");
|
|
380
|
+
}
|
|
381
|
+
if (Date.now() - startTime >= timeout) {
|
|
382
|
+
throw new Error(`Polling timeout after ${timeout}ms`);
|
|
383
|
+
}
|
|
384
|
+
const data = await fetchFn();
|
|
385
|
+
if (conditionFn(data)) {
|
|
386
|
+
return data;
|
|
387
|
+
}
|
|
388
|
+
await sleep2(pollInterval);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
function sleep2(ms) {
|
|
392
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// src/resources/account.ts
|
|
396
|
+
var AccountResource = class {
|
|
397
|
+
constructor(ctx) {
|
|
398
|
+
this.ctx = ctx;
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Get current account information
|
|
402
|
+
*
|
|
403
|
+
* @example
|
|
404
|
+
* ```ts
|
|
405
|
+
* const account = await client.account.get();
|
|
406
|
+
* console.log(account.name);
|
|
407
|
+
* console.log(account.tier);
|
|
408
|
+
* console.log(account.usage);
|
|
409
|
+
* ```
|
|
410
|
+
*/
|
|
411
|
+
async get() {
|
|
412
|
+
const result = await get(this.ctx, "/v1/client");
|
|
413
|
+
return result.data;
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Get detailed account metrics
|
|
417
|
+
*
|
|
418
|
+
* @example
|
|
419
|
+
* ```ts
|
|
420
|
+
* const metrics = await client.account.getMetrics();
|
|
421
|
+
* console.log(metrics.dealMetrics.totalDeals);
|
|
422
|
+
* console.log(metrics.categoryPerformance);
|
|
423
|
+
* ```
|
|
424
|
+
*/
|
|
425
|
+
async getMetrics() {
|
|
426
|
+
const result = await get(
|
|
427
|
+
this.ctx,
|
|
428
|
+
"/v1/client/metrics"
|
|
429
|
+
);
|
|
430
|
+
return result.data;
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Get DealUp sync metrics (pro/enterprise only)
|
|
434
|
+
*
|
|
435
|
+
* @example
|
|
436
|
+
* ```ts
|
|
437
|
+
* const dealup = await client.account.getDealUpMetrics();
|
|
438
|
+
* console.log(dealup.sync.totalSynced);
|
|
439
|
+
* console.log(dealup.sync.clicksGenerated);
|
|
440
|
+
* ```
|
|
441
|
+
*/
|
|
442
|
+
async getDealUpMetrics() {
|
|
443
|
+
const result = await get(
|
|
444
|
+
this.ctx,
|
|
445
|
+
"/v1/client/dealup"
|
|
446
|
+
);
|
|
447
|
+
return result.data;
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* Get crawl recommendations
|
|
451
|
+
* AI-suggested URLs to crawl based on past performance
|
|
452
|
+
*
|
|
453
|
+
* @example
|
|
454
|
+
* ```ts
|
|
455
|
+
* const recommendations = await client.account.getRecommendations();
|
|
456
|
+
* recommendations.recommendations.forEach(r => {
|
|
457
|
+
* console.log(r.url, r.reason, r.estimatedDeals);
|
|
458
|
+
* });
|
|
459
|
+
* ```
|
|
460
|
+
*/
|
|
461
|
+
async getRecommendations() {
|
|
462
|
+
const result = await get(
|
|
463
|
+
this.ctx,
|
|
464
|
+
"/v1/client/recommendations"
|
|
465
|
+
);
|
|
466
|
+
return result.data;
|
|
467
|
+
}
|
|
468
|
+
/**
|
|
469
|
+
* Get current preferences
|
|
470
|
+
*
|
|
471
|
+
* @example
|
|
472
|
+
* ```ts
|
|
473
|
+
* const prefs = await client.account.getPreferences();
|
|
474
|
+
* console.log(prefs.preferences.minDealScore);
|
|
475
|
+
* console.log(prefs.preferences.autoSync);
|
|
476
|
+
* ```
|
|
477
|
+
*/
|
|
478
|
+
async getPreferences() {
|
|
479
|
+
const result = await get(
|
|
480
|
+
this.ctx,
|
|
481
|
+
"/v1/client/preferences"
|
|
482
|
+
);
|
|
483
|
+
return result.data;
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Update account preferences
|
|
487
|
+
*
|
|
488
|
+
* @example
|
|
489
|
+
* ```ts
|
|
490
|
+
* const updated = await client.account.updatePreferences({
|
|
491
|
+
* minDealScore: 70,
|
|
492
|
+
* autoSync: true,
|
|
493
|
+
* preferredCategories: ["software", "courses"]
|
|
494
|
+
* });
|
|
495
|
+
* ```
|
|
496
|
+
*/
|
|
497
|
+
async updatePreferences(options) {
|
|
498
|
+
const result = await patch(
|
|
499
|
+
this.ctx,
|
|
500
|
+
"/v1/client/preferences",
|
|
501
|
+
{
|
|
502
|
+
preferredCategories: options.preferredCategories,
|
|
503
|
+
minDealScore: options.minDealScore,
|
|
504
|
+
autoSync: options.autoSync,
|
|
505
|
+
webhookEnabled: options.webhookEnabled
|
|
506
|
+
}
|
|
507
|
+
);
|
|
508
|
+
return result.data;
|
|
509
|
+
}
|
|
510
|
+
/**
|
|
511
|
+
* Track a DealUp click (webhook endpoint)
|
|
512
|
+
* Used internally by DealUp to track click-through
|
|
513
|
+
*
|
|
514
|
+
* @example
|
|
515
|
+
* ```ts
|
|
516
|
+
* await client.account.trackClick("deal_abc123", "homepage");
|
|
517
|
+
* ```
|
|
518
|
+
*/
|
|
519
|
+
async trackClick(dealId, source) {
|
|
520
|
+
const result = await post(
|
|
521
|
+
this.ctx,
|
|
522
|
+
"/v1/client/dealup/click",
|
|
523
|
+
{
|
|
524
|
+
dealId,
|
|
525
|
+
source
|
|
526
|
+
}
|
|
527
|
+
);
|
|
528
|
+
return result.data;
|
|
529
|
+
}
|
|
530
|
+
// ============================================
|
|
531
|
+
// CONVENIENCE METHODS
|
|
532
|
+
// ============================================
|
|
533
|
+
/**
|
|
534
|
+
* Get remaining quota for a resource
|
|
535
|
+
*
|
|
536
|
+
* @example
|
|
537
|
+
* ```ts
|
|
538
|
+
* const remaining = await client.account.getRemainingQuota("scrapes");
|
|
539
|
+
* console.log(`${remaining} scrapes left this period`);
|
|
540
|
+
* ```
|
|
541
|
+
*/
|
|
542
|
+
async getRemainingQuota(resource) {
|
|
543
|
+
const account = await this.get();
|
|
544
|
+
const usage = account.usage[resource];
|
|
545
|
+
return usage.limit - usage.used;
|
|
546
|
+
}
|
|
547
|
+
/**
|
|
548
|
+
* Check if account has quota for a resource
|
|
549
|
+
*
|
|
550
|
+
* @example
|
|
551
|
+
* ```ts
|
|
552
|
+
* if (await client.account.hasQuota("crawls", 5)) {
|
|
553
|
+
* // Safe to start 5 crawls
|
|
554
|
+
* }
|
|
555
|
+
* ```
|
|
556
|
+
*/
|
|
557
|
+
async hasQuota(resource, needed = 1) {
|
|
558
|
+
const remaining = await this.getRemainingQuota(resource);
|
|
559
|
+
return remaining >= needed;
|
|
560
|
+
}
|
|
561
|
+
/**
|
|
562
|
+
* Get account tier
|
|
563
|
+
*
|
|
564
|
+
* @example
|
|
565
|
+
* ```ts
|
|
566
|
+
* const tier = await client.account.getTier();
|
|
567
|
+
* if (tier === "enterprise") {
|
|
568
|
+
* // Enable advanced features
|
|
569
|
+
* }
|
|
570
|
+
* ```
|
|
571
|
+
*/
|
|
572
|
+
async getTier() {
|
|
573
|
+
const account = await this.get();
|
|
574
|
+
return account.tier;
|
|
575
|
+
}
|
|
576
|
+
/**
|
|
577
|
+
* Check if account is pro or enterprise tier
|
|
578
|
+
*
|
|
579
|
+
* @example
|
|
580
|
+
* ```ts
|
|
581
|
+
* if (await client.account.isPremium()) {
|
|
582
|
+
* // Show premium features
|
|
583
|
+
* }
|
|
584
|
+
* ```
|
|
585
|
+
*/
|
|
586
|
+
async isPremium() {
|
|
587
|
+
const tier = await this.getTier();
|
|
588
|
+
return tier === "pro" || tier === "enterprise";
|
|
589
|
+
}
|
|
590
|
+
};
|
|
591
|
+
|
|
592
|
+
// src/resources/crawl.ts
|
|
593
|
+
var CRAWL_TEMPLATES = {
|
|
594
|
+
ecommerce: {
|
|
595
|
+
id: "ecommerce",
|
|
596
|
+
name: "E-commerce",
|
|
597
|
+
description: "Optimized for product pages and online stores",
|
|
598
|
+
defaultOptions: {
|
|
599
|
+
maxDepth: 3,
|
|
600
|
+
maxPages: 500,
|
|
601
|
+
delayMs: 1500,
|
|
602
|
+
extractDeal: true,
|
|
603
|
+
prioritizeDealPages: true,
|
|
604
|
+
excludePatterns: [
|
|
605
|
+
"*/cart*",
|
|
606
|
+
"*/checkout*",
|
|
607
|
+
"*/account*",
|
|
608
|
+
"*/login*",
|
|
609
|
+
"*/register*",
|
|
610
|
+
"*/wishlist*",
|
|
611
|
+
"*/compare*"
|
|
612
|
+
]
|
|
613
|
+
}
|
|
614
|
+
},
|
|
615
|
+
blog: {
|
|
616
|
+
id: "blog",
|
|
617
|
+
name: "Blog",
|
|
618
|
+
description: "Optimized for blog posts and articles",
|
|
619
|
+
defaultOptions: {
|
|
620
|
+
maxDepth: 2,
|
|
621
|
+
maxPages: 200,
|
|
622
|
+
delayMs: 1e3,
|
|
623
|
+
extractDeal: false,
|
|
624
|
+
prioritizeDealPages: false,
|
|
625
|
+
excludePatterns: ["*/tag/*", "*/category/*", "*/author/*", "*/page/*"]
|
|
626
|
+
}
|
|
627
|
+
},
|
|
628
|
+
docs: {
|
|
629
|
+
id: "docs",
|
|
630
|
+
name: "Documentation",
|
|
631
|
+
description: "Optimized for documentation sites",
|
|
632
|
+
defaultOptions: {
|
|
633
|
+
maxDepth: 4,
|
|
634
|
+
maxPages: 300,
|
|
635
|
+
delayMs: 500,
|
|
636
|
+
extractDeal: false,
|
|
637
|
+
prioritizeDealPages: false
|
|
638
|
+
}
|
|
639
|
+
},
|
|
640
|
+
marketplace: {
|
|
641
|
+
id: "marketplace",
|
|
642
|
+
name: "Marketplace",
|
|
643
|
+
description: "Optimized for multi-vendor marketplaces",
|
|
644
|
+
defaultOptions: {
|
|
645
|
+
maxDepth: 3,
|
|
646
|
+
maxPages: 1e3,
|
|
647
|
+
delayMs: 2e3,
|
|
648
|
+
extractDeal: true,
|
|
649
|
+
prioritizeDealPages: true,
|
|
650
|
+
followExternalLinks: false,
|
|
651
|
+
excludePatterns: [
|
|
652
|
+
"*/seller/*",
|
|
653
|
+
"*/vendor/*",
|
|
654
|
+
"*/shop/*",
|
|
655
|
+
"*/reviews*",
|
|
656
|
+
"*/questions*"
|
|
657
|
+
]
|
|
658
|
+
}
|
|
659
|
+
},
|
|
660
|
+
custom: {
|
|
661
|
+
id: "custom",
|
|
662
|
+
name: "Custom",
|
|
663
|
+
description: "No preset - use your own settings",
|
|
664
|
+
defaultOptions: {}
|
|
665
|
+
}
|
|
666
|
+
};
|
|
667
|
+
var CrawlResource = class {
|
|
668
|
+
constructor(ctx) {
|
|
669
|
+
this.ctx = ctx;
|
|
670
|
+
}
|
|
671
|
+
/**
|
|
672
|
+
* Create a new crawl job
|
|
673
|
+
*
|
|
674
|
+
* @example
|
|
675
|
+
* ```ts
|
|
676
|
+
* const job = await client.crawl.create({
|
|
677
|
+
* url: "https://example.com",
|
|
678
|
+
* maxDepth: 3,
|
|
679
|
+
* maxPages: 100,
|
|
680
|
+
* extractDeal: true
|
|
681
|
+
* });
|
|
682
|
+
* console.log(job.jobId);
|
|
683
|
+
* ```
|
|
684
|
+
*/
|
|
685
|
+
async create(options) {
|
|
686
|
+
const body = {
|
|
687
|
+
startUrl: options.url || options.startUrl,
|
|
688
|
+
url: options.url || options.startUrl,
|
|
689
|
+
prompt: options.prompt,
|
|
690
|
+
maxDepth: options.maxDepth,
|
|
691
|
+
maxPages: options.maxPages,
|
|
692
|
+
delayMs: options.delayMs,
|
|
693
|
+
detectSignals: options.detectSignals,
|
|
694
|
+
extractWithAI: options.extractWithAI,
|
|
695
|
+
extractDeal: options.extractDeal,
|
|
696
|
+
minDealScore: options.minDealScore,
|
|
697
|
+
prioritizeDealPages: options.prioritizeDealPages,
|
|
698
|
+
followExternalLinks: options.followExternalLinks,
|
|
699
|
+
allowedDomains: options.allowedDomains,
|
|
700
|
+
excludePatterns: options.excludePatterns
|
|
701
|
+
};
|
|
702
|
+
const result = await post(this.ctx, "/v1/crawl", body);
|
|
703
|
+
return result.data;
|
|
704
|
+
}
|
|
705
|
+
/**
|
|
706
|
+
* Create a crawl job using a template
|
|
707
|
+
*
|
|
708
|
+
* @example
|
|
709
|
+
* ```ts
|
|
710
|
+
* const job = await client.crawl.withTemplate("ecommerce", {
|
|
711
|
+
* url: "https://shop.example.com",
|
|
712
|
+
* maxPages: 200 // Override template default
|
|
713
|
+
* });
|
|
714
|
+
* ```
|
|
715
|
+
*/
|
|
716
|
+
async withTemplate(templateId, options) {
|
|
717
|
+
const template = CRAWL_TEMPLATES[templateId];
|
|
718
|
+
if (!template) {
|
|
719
|
+
throw new Error(
|
|
720
|
+
`Invalid crawl templateId: ${templateId}. Available templates: ${Object.keys(CRAWL_TEMPLATES).join(", ")}`
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
return this.create({
|
|
724
|
+
...template.defaultOptions,
|
|
725
|
+
...options
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
/**
|
|
729
|
+
* List available crawl templates
|
|
730
|
+
*
|
|
731
|
+
* @example
|
|
732
|
+
* ```ts
|
|
733
|
+
* const templates = client.crawl.listTemplates();
|
|
734
|
+
* templates.forEach(t => console.log(t.name, t.description));
|
|
735
|
+
* ```
|
|
736
|
+
*/
|
|
737
|
+
listTemplates() {
|
|
738
|
+
return Object.values(CRAWL_TEMPLATES);
|
|
739
|
+
}
|
|
740
|
+
/**
|
|
741
|
+
* Get a specific template by ID
|
|
742
|
+
*
|
|
743
|
+
* @example
|
|
744
|
+
* ```ts
|
|
745
|
+
* const template = client.crawl.getTemplate("ecommerce");
|
|
746
|
+
* console.log(template.defaultOptions);
|
|
747
|
+
* ```
|
|
748
|
+
*/
|
|
749
|
+
getTemplate(templateId) {
|
|
750
|
+
return CRAWL_TEMPLATES[templateId];
|
|
751
|
+
}
|
|
752
|
+
/**
|
|
753
|
+
* Analyze a URL before crawling
|
|
754
|
+
* Returns recommended settings based on site structure
|
|
755
|
+
*
|
|
756
|
+
* @example
|
|
757
|
+
* ```ts
|
|
758
|
+
* const analysis = await client.crawl.analyze("https://shop.example.com");
|
|
759
|
+
* console.log(analysis.recommendedTemplate);
|
|
760
|
+
* console.log(analysis.estimatedPages);
|
|
761
|
+
* ```
|
|
762
|
+
*/
|
|
763
|
+
async analyze(url) {
|
|
764
|
+
const result = await get(
|
|
765
|
+
this.ctx,
|
|
766
|
+
"/v1/crawl/analyze",
|
|
767
|
+
{
|
|
768
|
+
url
|
|
769
|
+
}
|
|
770
|
+
);
|
|
771
|
+
return result.data;
|
|
772
|
+
}
|
|
773
|
+
/**
|
|
774
|
+
* Crawl a URL with deal extraction enabled
|
|
775
|
+
* Convenience method for e-commerce crawling
|
|
776
|
+
*
|
|
777
|
+
* @example
|
|
778
|
+
* ```ts
|
|
779
|
+
* const job = await client.crawl.forDeals("https://shop.example.com", {
|
|
780
|
+
* minDealScore: 70
|
|
781
|
+
* });
|
|
782
|
+
* ```
|
|
783
|
+
*/
|
|
784
|
+
async forDeals(url, options) {
|
|
785
|
+
return this.withTemplate("ecommerce", {
|
|
786
|
+
url,
|
|
787
|
+
extractDeal: true,
|
|
788
|
+
prioritizeDealPages: true,
|
|
789
|
+
...options
|
|
790
|
+
});
|
|
791
|
+
}
|
|
792
|
+
};
|
|
793
|
+
|
|
794
|
+
// src/resources/data.ts
|
|
795
|
+
var DataResource = class {
|
|
796
|
+
constructor(ctx) {
|
|
797
|
+
this.ctx = ctx;
|
|
798
|
+
}
|
|
799
|
+
// ============================================
|
|
800
|
+
// JOBS
|
|
801
|
+
// ============================================
|
|
802
|
+
/**
|
|
803
|
+
* List all jobs
|
|
804
|
+
*
|
|
805
|
+
* @example
|
|
806
|
+
* ```ts
|
|
807
|
+
* const jobs = await client.data.listJobs({
|
|
808
|
+
* status: "completed",
|
|
809
|
+
* type: "crawl",
|
|
810
|
+
* page: 1,
|
|
811
|
+
* limit: 20
|
|
812
|
+
* });
|
|
813
|
+
* console.log(jobs.data);
|
|
814
|
+
* console.log(jobs.pagination.total);
|
|
815
|
+
* ```
|
|
816
|
+
*/
|
|
817
|
+
async listJobs(options = {}) {
|
|
818
|
+
const result = await get(this.ctx, "/v1/data/jobs", {
|
|
819
|
+
page: options.page,
|
|
820
|
+
limit: options.limit,
|
|
821
|
+
status: options.status,
|
|
822
|
+
type: options.type,
|
|
823
|
+
sortBy: options.sortBy,
|
|
824
|
+
sortOrder: options.sortOrder,
|
|
825
|
+
fromDate: options.fromDate,
|
|
826
|
+
toDate: options.toDate
|
|
827
|
+
});
|
|
828
|
+
return result.data;
|
|
829
|
+
}
|
|
830
|
+
/**
|
|
831
|
+
* Get jobs by status
|
|
832
|
+
* Convenience method for filtering by status
|
|
833
|
+
*
|
|
834
|
+
* @example
|
|
835
|
+
* ```ts
|
|
836
|
+
* const activeJobs = await client.data.getJobsByStatus("active");
|
|
837
|
+
* ```
|
|
838
|
+
*/
|
|
839
|
+
async getJobsByStatus(status, options) {
|
|
840
|
+
return this.listJobs({ status, ...options });
|
|
841
|
+
}
|
|
842
|
+
/**
|
|
843
|
+
* Get jobs by type
|
|
844
|
+
* Convenience method for filtering by type
|
|
845
|
+
*
|
|
846
|
+
* @example
|
|
847
|
+
* ```ts
|
|
848
|
+
* const crawlJobs = await client.data.getJobsByType("crawl");
|
|
849
|
+
* ```
|
|
850
|
+
*/
|
|
851
|
+
async getJobsByType(type, options) {
|
|
852
|
+
return this.listJobs({ type, ...options });
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* Get recent jobs
|
|
856
|
+
* Convenience method for getting latest jobs
|
|
857
|
+
*
|
|
858
|
+
* @example
|
|
859
|
+
* ```ts
|
|
860
|
+
* const recentJobs = await client.data.getRecentJobs(10);
|
|
861
|
+
* ```
|
|
862
|
+
*/
|
|
863
|
+
async getRecentJobs(limit = 10) {
|
|
864
|
+
return this.listJobs({
|
|
865
|
+
limit,
|
|
866
|
+
sortBy: "created_at",
|
|
867
|
+
sortOrder: "desc"
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
// ============================================
|
|
871
|
+
// DEALS
|
|
872
|
+
// ============================================
|
|
873
|
+
/**
|
|
874
|
+
* List all deals
|
|
875
|
+
*
|
|
876
|
+
* @example
|
|
877
|
+
* ```ts
|
|
878
|
+
* const deals = await client.data.listDeals({
|
|
879
|
+
* minScore: 70,
|
|
880
|
+
* category: "electronics",
|
|
881
|
+
* sortBy: "deal_score",
|
|
882
|
+
* sortOrder: "desc"
|
|
883
|
+
* });
|
|
884
|
+
* ```
|
|
885
|
+
*/
|
|
886
|
+
async listDeals(options = {}) {
|
|
887
|
+
const result = await get(this.ctx, "/v1/data/deals", {
|
|
888
|
+
page: options.page,
|
|
889
|
+
limit: options.limit,
|
|
890
|
+
minScore: options.minScore,
|
|
891
|
+
maxPrice: options.maxPrice,
|
|
892
|
+
category: options.category,
|
|
893
|
+
merchant: options.merchant,
|
|
894
|
+
synced: options.synced?.toString(),
|
|
895
|
+
sortBy: options.sortBy,
|
|
896
|
+
sortOrder: options.sortOrder
|
|
897
|
+
});
|
|
898
|
+
return result.data;
|
|
899
|
+
}
|
|
900
|
+
/**
|
|
901
|
+
* Get a single deal by ID
|
|
902
|
+
*
|
|
903
|
+
* @example
|
|
904
|
+
* ```ts
|
|
905
|
+
* const deal = await client.data.getDeal("deal_abc123");
|
|
906
|
+
* console.log(deal.product.name);
|
|
907
|
+
* console.log(deal.pricing.discountPercent);
|
|
908
|
+
* ```
|
|
909
|
+
*/
|
|
910
|
+
async getDeal(dealId) {
|
|
911
|
+
if (!dealId || !dealId.trim()) {
|
|
912
|
+
throw new Error("dealId is required and cannot be empty");
|
|
913
|
+
}
|
|
914
|
+
const result = await get(this.ctx, `/v1/data/deals/${dealId}`);
|
|
915
|
+
return result.data;
|
|
916
|
+
}
|
|
917
|
+
/**
|
|
918
|
+
* Get top deals by score
|
|
919
|
+
* Convenience method for finding best deals
|
|
920
|
+
*
|
|
921
|
+
* @example
|
|
922
|
+
* ```ts
|
|
923
|
+
* const topDeals = await client.data.getTopDeals(20, 80);
|
|
924
|
+
* ```
|
|
925
|
+
*/
|
|
926
|
+
async getTopDeals(limit = 20, minScore = 70) {
|
|
927
|
+
return this.listDeals({
|
|
928
|
+
limit,
|
|
929
|
+
minScore,
|
|
930
|
+
sortBy: "deal_score",
|
|
931
|
+
sortOrder: "desc"
|
|
932
|
+
});
|
|
933
|
+
}
|
|
934
|
+
/**
|
|
935
|
+
* Get deals by category
|
|
936
|
+
* Convenience method for filtering by category
|
|
937
|
+
*
|
|
938
|
+
* @example
|
|
939
|
+
* ```ts
|
|
940
|
+
* const electronicsDeals = await client.data.getDealsByCategory("electronics");
|
|
941
|
+
async getDealsByCategory(
|
|
942
|
+
category: string,
|
|
943
|
+
options?: Omit<ListDealsOptions, "category">
|
|
944
|
+
): Promise<ListDealsResponse> {
|
|
945
|
+
if (!category || !category.trim()) {
|
|
946
|
+
throw new Error("category is required and cannot be empty");
|
|
947
|
+
}
|
|
948
|
+
return this.listDeals({ category, ...options });
|
|
949
|
+
} return this.listDeals({ category, ...options });
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* Get unsynced deals (not yet sent to DealUp)
|
|
954
|
+
*
|
|
955
|
+
* @example
|
|
956
|
+
* ```ts
|
|
957
|
+
* const unsyncedDeals = await client.data.getUnsyncedDeals();
|
|
958
|
+
* ```
|
|
959
|
+
*/
|
|
960
|
+
async getUnsyncedDeals(options) {
|
|
961
|
+
return this.listDeals({ synced: false, ...options });
|
|
962
|
+
}
|
|
963
|
+
// ============================================
|
|
964
|
+
// EXPORT
|
|
965
|
+
// ============================================
|
|
966
|
+
/**
|
|
967
|
+
* Export jobs data
|
|
968
|
+
*
|
|
969
|
+
* @example
|
|
970
|
+
* ```ts
|
|
971
|
+
* // Export as JSON
|
|
972
|
+
* const jsonData = await client.data.exportJobs({ format: "json" });
|
|
973
|
+
*
|
|
974
|
+
* // Export as CSV
|
|
975
|
+
* const csvData = await client.data.exportJobs({ format: "csv" });
|
|
976
|
+
* ```
|
|
977
|
+
*/
|
|
978
|
+
async exportJobs(options = {}) {
|
|
979
|
+
const result = await get(
|
|
980
|
+
this.ctx,
|
|
981
|
+
"/v1/data/export/jobs",
|
|
982
|
+
{
|
|
983
|
+
format: options.format || "json",
|
|
984
|
+
status: options.status,
|
|
985
|
+
type: options.type,
|
|
986
|
+
fromDate: options.fromDate,
|
|
987
|
+
toDate: options.toDate
|
|
988
|
+
}
|
|
989
|
+
);
|
|
990
|
+
return result.data;
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* Export deals data
|
|
994
|
+
*
|
|
995
|
+
* @example
|
|
996
|
+
* ```ts
|
|
997
|
+
* // Export as JSON
|
|
998
|
+
* const jsonData = await client.data.exportDeals({ format: "json" });
|
|
999
|
+
*
|
|
1000
|
+
* // Export as CSV with filters
|
|
1001
|
+
* const csvData = await client.data.exportDeals({
|
|
1002
|
+
* format: "csv",
|
|
1003
|
+
* minScore: 70,
|
|
1004
|
+
* category: "software"
|
|
1005
|
+
* });
|
|
1006
|
+
* ```
|
|
1007
|
+
*/
|
|
1008
|
+
async exportDeals(options = {}) {
|
|
1009
|
+
const result = await get(
|
|
1010
|
+
this.ctx,
|
|
1011
|
+
"/v1/data/export/deals",
|
|
1012
|
+
{
|
|
1013
|
+
format: options.format || "json",
|
|
1014
|
+
minScore: options.minScore,
|
|
1015
|
+
maxPrice: options.maxPrice,
|
|
1016
|
+
category: options.category,
|
|
1017
|
+
includeRawSignals: options.includeRawSignals
|
|
1018
|
+
}
|
|
1019
|
+
);
|
|
1020
|
+
return result.data;
|
|
1021
|
+
}
|
|
1022
|
+
// ============================================
|
|
1023
|
+
// STATS
|
|
1024
|
+
// ============================================
|
|
1025
|
+
/**
|
|
1026
|
+
* Get client statistics
|
|
1027
|
+
*
|
|
1028
|
+
* @example
|
|
1029
|
+
* ```ts
|
|
1030
|
+
* const stats = await client.data.getStats();
|
|
1031
|
+
* console.log(stats.totals.deals);
|
|
1032
|
+
* console.log(stats.performance.avgDealScore);
|
|
1033
|
+
* ```
|
|
1034
|
+
*/
|
|
1035
|
+
async getStats() {
|
|
1036
|
+
const result = await get(this.ctx, "/v1/data/stats");
|
|
1037
|
+
return result.data;
|
|
1038
|
+
}
|
|
1039
|
+
};
|
|
1040
|
+
|
|
1041
|
+
// src/resources/dork.ts
|
|
1042
|
+
var DorkResource = class {
|
|
1043
|
+
constructor(ctx) {
|
|
1044
|
+
this.ctx = ctx;
|
|
1045
|
+
}
|
|
1046
|
+
/**
|
|
1047
|
+
* Create a new dork search job
|
|
1048
|
+
*
|
|
1049
|
+
* @example
|
|
1050
|
+
* ```ts
|
|
1051
|
+
* const job = await client.dork.create({
|
|
1052
|
+
* query: "discount coupon",
|
|
1053
|
+
* site: "amazon.com",
|
|
1054
|
+
* maxResults: 20
|
|
1055
|
+
* });
|
|
1056
|
+
* console.log(job.jobId);
|
|
1057
|
+
* ```
|
|
1058
|
+
*/
|
|
1059
|
+
async create(options) {
|
|
1060
|
+
const body = {
|
|
1061
|
+
query: this.buildQuery(options),
|
|
1062
|
+
maxResults: options.maxResults,
|
|
1063
|
+
region: options.region
|
|
1064
|
+
};
|
|
1065
|
+
const result = await post(this.ctx, "/v1/dork", body);
|
|
1066
|
+
return result.data;
|
|
1067
|
+
}
|
|
1068
|
+
/**
|
|
1069
|
+
* Search for deals on a specific site
|
|
1070
|
+
* Convenience method for finding discount pages
|
|
1071
|
+
*
|
|
1072
|
+
* @example
|
|
1073
|
+
* ```ts
|
|
1074
|
+
* const job = await client.dork.findDeals("amazon.com", {
|
|
1075
|
+
* maxResults: 50
|
|
1076
|
+
* });
|
|
1077
|
+
* ```
|
|
1078
|
+
*/
|
|
1079
|
+
async findDeals(site, options) {
|
|
1080
|
+
return this.create({
|
|
1081
|
+
query: "discount OR coupon OR sale OR promo OR deal",
|
|
1082
|
+
site,
|
|
1083
|
+
...options
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
/**
|
|
1087
|
+
* Search for product pages on a site
|
|
1088
|
+
* Looks for common product URL patterns
|
|
1089
|
+
*
|
|
1090
|
+
* @example
|
|
1091
|
+
* ```ts
|
|
1092
|
+
* const job = await client.dork.findProducts("shop.example.com", {
|
|
1093
|
+
* maxResults: 100
|
|
1094
|
+
* });
|
|
1095
|
+
* ```
|
|
1096
|
+
*/
|
|
1097
|
+
async findProducts(site, options) {
|
|
1098
|
+
return this.create({
|
|
1099
|
+
query: "product OR item OR buy",
|
|
1100
|
+
site,
|
|
1101
|
+
inUrl: "product",
|
|
1102
|
+
...options
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1105
|
+
/**
|
|
1106
|
+
* Search for PDFs on a site
|
|
1107
|
+
* Useful for finding manuals, guides, datasheets
|
|
1108
|
+
*
|
|
1109
|
+
* @example
|
|
1110
|
+
* ```ts
|
|
1111
|
+
* const job = await client.dork.findPDFs("docs.example.com", "user guide");
|
|
1112
|
+
* ```
|
|
1113
|
+
*/
|
|
1114
|
+
async findPDFs(site, query, options) {
|
|
1115
|
+
return this.create({
|
|
1116
|
+
query: query || "*",
|
|
1117
|
+
site,
|
|
1118
|
+
fileType: "pdf",
|
|
1119
|
+
...options
|
|
1120
|
+
});
|
|
1121
|
+
}
|
|
1122
|
+
/**
|
|
1123
|
+
* Search with custom title filter
|
|
1124
|
+
* Find pages with specific terms in title
|
|
1125
|
+
*
|
|
1126
|
+
* @example
|
|
1127
|
+
* ```ts
|
|
1128
|
+
* const job = await client.dork.inTitle("Black Friday", {
|
|
1129
|
+
* site: "bestbuy.com",
|
|
1130
|
+
* maxResults: 30
|
|
1131
|
+
* });
|
|
1132
|
+
* ```
|
|
1133
|
+
*/
|
|
1134
|
+
async inTitle(titleQuery, options) {
|
|
1135
|
+
return this.create({
|
|
1136
|
+
query: options?.query || titleQuery,
|
|
1137
|
+
inTitle: titleQuery,
|
|
1138
|
+
...options
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
1141
|
+
/**
|
|
1142
|
+
* Search with custom URL filter
|
|
1143
|
+
* Find pages with specific terms in URL
|
|
1144
|
+
*
|
|
1145
|
+
* @example
|
|
1146
|
+
* ```ts
|
|
1147
|
+
* const job = await client.dork.inUrl("clearance", {
|
|
1148
|
+
* site: "walmart.com"
|
|
1149
|
+
* });
|
|
1150
|
+
* ```
|
|
1151
|
+
*/
|
|
1152
|
+
async inUrl(urlQuery, options) {
|
|
1153
|
+
return this.create({
|
|
1154
|
+
query: options?.query || urlQuery,
|
|
1155
|
+
inUrl: urlQuery,
|
|
1156
|
+
...options
|
|
1157
|
+
});
|
|
1158
|
+
}
|
|
1159
|
+
/**
|
|
1160
|
+
* Build a raw Google dork query string
|
|
1161
|
+
* Useful for preview or debugging
|
|
1162
|
+
*
|
|
1163
|
+
* @example
|
|
1164
|
+
* ```ts
|
|
1165
|
+
* const query = client.dork.buildQuery({
|
|
1166
|
+
* query: "laptop deals",
|
|
1167
|
+
* site: "amazon.com",
|
|
1168
|
+
* inTitle: "discount"
|
|
1169
|
+
* });
|
|
1170
|
+
* // Returns: "laptop deals site:amazon.com intitle:discount"
|
|
1171
|
+
* ```
|
|
1172
|
+
*/
|
|
1173
|
+
buildQuery(options) {
|
|
1174
|
+
const parts = [];
|
|
1175
|
+
if (typeof options.query === "string" && options.query.trim() !== "") {
|
|
1176
|
+
parts.push(options.query);
|
|
1177
|
+
}
|
|
1178
|
+
if (options.site) {
|
|
1179
|
+
parts.push(`site:${options.site}`);
|
|
1180
|
+
}
|
|
1181
|
+
if (options.fileType) {
|
|
1182
|
+
parts.push(`filetype:${options.fileType}`);
|
|
1183
|
+
}
|
|
1184
|
+
if (options.inUrl) {
|
|
1185
|
+
parts.push(`inurl:${options.inUrl}`);
|
|
1186
|
+
}
|
|
1187
|
+
if (options.inTitle) {
|
|
1188
|
+
parts.push(`intitle:${options.inTitle}`);
|
|
1189
|
+
}
|
|
1190
|
+
return parts.join(" ");
|
|
1191
|
+
}
|
|
1192
|
+
};
|
|
1193
|
+
|
|
1194
|
+
// src/resources/extract.ts
|
|
1195
|
+
var ExtractResource = class {
|
|
1196
|
+
constructor(ctx) {
|
|
1197
|
+
this.ctx = ctx;
|
|
1198
|
+
}
|
|
1199
|
+
/**
|
|
1200
|
+
* Create a new extraction job
|
|
1201
|
+
* Either schema or prompt must be provided
|
|
1202
|
+
*
|
|
1203
|
+
* @example
|
|
1204
|
+
* ```ts
|
|
1205
|
+
* // Schema-based extraction
|
|
1206
|
+
* const job = await client.extract.create({
|
|
1207
|
+
* url: "https://example.com/product",
|
|
1208
|
+
* schema: {
|
|
1209
|
+
* type: "object",
|
|
1210
|
+
* properties: {
|
|
1211
|
+
* name: { type: "string" },
|
|
1212
|
+
* price: { type: "number" },
|
|
1213
|
+
* features: { type: "array", items: { type: "string" } }
|
|
1214
|
+
* }
|
|
1215
|
+
* }
|
|
1216
|
+
* });
|
|
1217
|
+
*
|
|
1218
|
+
* // Prompt-based extraction
|
|
1219
|
+
* const job = await client.extract.create({
|
|
1220
|
+
* url: "https://example.com/article",
|
|
1221
|
+
* prompt: "Extract the main points and author name"
|
|
1222
|
+
* });
|
|
1223
|
+
* ```
|
|
1224
|
+
*/
|
|
1225
|
+
async create(options) {
|
|
1226
|
+
if (!options.schema && !options.prompt) {
|
|
1227
|
+
throw new Error("Either 'schema' or 'prompt' must be provided");
|
|
1228
|
+
}
|
|
1229
|
+
const body = {
|
|
1230
|
+
url: options.url,
|
|
1231
|
+
schema: options.schema,
|
|
1232
|
+
prompt: options.prompt,
|
|
1233
|
+
onlyMainContent: options.onlyMainContent,
|
|
1234
|
+
excludeTags: options.excludeTags,
|
|
1235
|
+
excludeSelectors: options.excludeSelectors,
|
|
1236
|
+
model: options.model,
|
|
1237
|
+
temperature: options.temperature,
|
|
1238
|
+
webhook: options.webhook,
|
|
1239
|
+
headers: options.headers,
|
|
1240
|
+
timeout: options.timeout
|
|
1241
|
+
};
|
|
1242
|
+
const result = await post(
|
|
1243
|
+
this.ctx,
|
|
1244
|
+
"/v1/extract",
|
|
1245
|
+
body
|
|
1246
|
+
);
|
|
1247
|
+
return result.data;
|
|
1248
|
+
}
|
|
1249
|
+
/**
|
|
1250
|
+
* Extract data using a JSON Schema
|
|
1251
|
+
* Convenience method for schema-based extraction
|
|
1252
|
+
*
|
|
1253
|
+
* @example
|
|
1254
|
+
* ```ts
|
|
1255
|
+
* const job = await client.extract.withSchema(
|
|
1256
|
+
* "https://example.com/product",
|
|
1257
|
+
* {
|
|
1258
|
+
* type: "object",
|
|
1259
|
+
* properties: {
|
|
1260
|
+
* name: { type: "string" },
|
|
1261
|
+
* price: { type: "number" }
|
|
1262
|
+
* }
|
|
1263
|
+
* }
|
|
1264
|
+
* );
|
|
1265
|
+
* ```
|
|
1266
|
+
*/
|
|
1267
|
+
async withSchema(url, schema, options) {
|
|
1268
|
+
return this.create({
|
|
1269
|
+
url,
|
|
1270
|
+
schema,
|
|
1271
|
+
...options
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
1274
|
+
/**
|
|
1275
|
+
* Extract data using a natural language prompt
|
|
1276
|
+
* Convenience method for prompt-based extraction
|
|
1277
|
+
*
|
|
1278
|
+
* @example
|
|
1279
|
+
* ```ts
|
|
1280
|
+
* const job = await client.extract.withPrompt(
|
|
1281
|
+
* "https://example.com/article",
|
|
1282
|
+
* "Extract the article title, author, publication date, and a brief summary"
|
|
1283
|
+
* );
|
|
1284
|
+
* ```
|
|
1285
|
+
*/
|
|
1286
|
+
async withPrompt(url, prompt, options) {
|
|
1287
|
+
return this.create({
|
|
1288
|
+
url,
|
|
1289
|
+
prompt,
|
|
1290
|
+
...options
|
|
1291
|
+
});
|
|
1292
|
+
}
|
|
1293
|
+
/**
|
|
1294
|
+
* Extract product information from a page
|
|
1295
|
+
* Pre-built schema for common e-commerce use case
|
|
1296
|
+
*
|
|
1297
|
+
* @example
|
|
1298
|
+
* ```ts
|
|
1299
|
+
* const job = await client.extract.product("https://shop.example.com/item");
|
|
1300
|
+
* ```
|
|
1301
|
+
*/
|
|
1302
|
+
async product(url, options) {
|
|
1303
|
+
const productSchema = {
|
|
1304
|
+
type: "object",
|
|
1305
|
+
properties: {
|
|
1306
|
+
name: { type: "string", description: "Product name" },
|
|
1307
|
+
brand: { type: "string", description: "Brand name" },
|
|
1308
|
+
price: { type: "number", description: "Current price" },
|
|
1309
|
+
originalPrice: {
|
|
1310
|
+
type: "number",
|
|
1311
|
+
description: "Original price before discount"
|
|
1312
|
+
},
|
|
1313
|
+
currency: {
|
|
1314
|
+
type: "string",
|
|
1315
|
+
description: "Currency code (USD, EUR, etc.)"
|
|
1316
|
+
},
|
|
1317
|
+
description: { type: "string", description: "Product description" },
|
|
1318
|
+
features: {
|
|
1319
|
+
type: "array",
|
|
1320
|
+
items: { type: "string" },
|
|
1321
|
+
description: "List of product features"
|
|
1322
|
+
},
|
|
1323
|
+
images: {
|
|
1324
|
+
type: "array",
|
|
1325
|
+
items: { type: "string" },
|
|
1326
|
+
description: "Product image URLs"
|
|
1327
|
+
},
|
|
1328
|
+
availability: {
|
|
1329
|
+
type: "string",
|
|
1330
|
+
description: "In stock, out of stock, etc."
|
|
1331
|
+
},
|
|
1332
|
+
rating: { type: "number", description: "Average rating" },
|
|
1333
|
+
reviewCount: { type: "number", description: "Number of reviews" }
|
|
1334
|
+
},
|
|
1335
|
+
required: ["name", "price"]
|
|
1336
|
+
};
|
|
1337
|
+
return this.create({
|
|
1338
|
+
url,
|
|
1339
|
+
schema: productSchema,
|
|
1340
|
+
model: "gpt-4o-mini",
|
|
1341
|
+
...options
|
|
1342
|
+
});
|
|
1343
|
+
}
|
|
1344
|
+
/**
|
|
1345
|
+
* Extract article/blog post information
|
|
1346
|
+
* Pre-built schema for content extraction
|
|
1347
|
+
*
|
|
1348
|
+
* @example
|
|
1349
|
+
* ```ts
|
|
1350
|
+
* const job = await client.extract.article("https://blog.example.com/post");
|
|
1351
|
+
* ```
|
|
1352
|
+
*/
|
|
1353
|
+
async article(url, options) {
|
|
1354
|
+
const articleSchema = {
|
|
1355
|
+
type: "object",
|
|
1356
|
+
properties: {
|
|
1357
|
+
title: { type: "string", description: "Article title" },
|
|
1358
|
+
author: { type: "string", description: "Author name" },
|
|
1359
|
+
publishedAt: { type: "string", description: "Publication date" },
|
|
1360
|
+
summary: {
|
|
1361
|
+
type: "string",
|
|
1362
|
+
description: "Brief summary of the article"
|
|
1363
|
+
},
|
|
1364
|
+
content: { type: "string", description: "Main article content" },
|
|
1365
|
+
tags: {
|
|
1366
|
+
type: "array",
|
|
1367
|
+
items: { type: "string" },
|
|
1368
|
+
description: "Article tags or categories"
|
|
1369
|
+
},
|
|
1370
|
+
readingTime: {
|
|
1371
|
+
type: "number",
|
|
1372
|
+
description: "Estimated reading time in minutes"
|
|
1373
|
+
}
|
|
1374
|
+
},
|
|
1375
|
+
required: ["title", "content"]
|
|
1376
|
+
};
|
|
1377
|
+
return this.create({
|
|
1378
|
+
url,
|
|
1379
|
+
schema: articleSchema,
|
|
1380
|
+
onlyMainContent: true,
|
|
1381
|
+
...options
|
|
1382
|
+
});
|
|
1383
|
+
}
|
|
1384
|
+
/**
|
|
1385
|
+
* Extract contact information from a page
|
|
1386
|
+
* Pre-built schema for contact page scraping
|
|
1387
|
+
*
|
|
1388
|
+
* @example
|
|
1389
|
+
* ```ts
|
|
1390
|
+
* const job = await client.extract.contact("https://example.com/contact");
|
|
1391
|
+
* ```
|
|
1392
|
+
*/
|
|
1393
|
+
async contact(url, options) {
|
|
1394
|
+
const contactSchema = {
|
|
1395
|
+
type: "object",
|
|
1396
|
+
properties: {
|
|
1397
|
+
companyName: {
|
|
1398
|
+
type: "string",
|
|
1399
|
+
description: "Company or organization name"
|
|
1400
|
+
},
|
|
1401
|
+
email: { type: "string", description: "Contact email address" },
|
|
1402
|
+
phone: { type: "string", description: "Phone number" },
|
|
1403
|
+
address: {
|
|
1404
|
+
type: "object",
|
|
1405
|
+
properties: {
|
|
1406
|
+
street: { type: "string" },
|
|
1407
|
+
city: { type: "string" },
|
|
1408
|
+
state: { type: "string" },
|
|
1409
|
+
postalCode: { type: "string" },
|
|
1410
|
+
country: { type: "string" }
|
|
1411
|
+
},
|
|
1412
|
+
description: "Physical address"
|
|
1413
|
+
},
|
|
1414
|
+
socialMedia: {
|
|
1415
|
+
type: "object",
|
|
1416
|
+
properties: {
|
|
1417
|
+
twitter: { type: "string" },
|
|
1418
|
+
linkedin: { type: "string" },
|
|
1419
|
+
facebook: { type: "string" },
|
|
1420
|
+
instagram: { type: "string" }
|
|
1421
|
+
},
|
|
1422
|
+
description: "Social media links"
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
};
|
|
1426
|
+
return this.create({
|
|
1427
|
+
url,
|
|
1428
|
+
schema: contactSchema,
|
|
1429
|
+
onlyMainContent: false,
|
|
1430
|
+
// Contact info might be in footer
|
|
1431
|
+
...options
|
|
1432
|
+
});
|
|
1433
|
+
}
|
|
1434
|
+
};
|
|
1435
|
+
|
|
1436
|
+
// src/resources/keys.ts
|
|
1437
|
+
var KeysResource = class {
|
|
1438
|
+
constructor(ctx) {
|
|
1439
|
+
this.ctx = ctx;
|
|
1440
|
+
}
|
|
1441
|
+
/**
|
|
1442
|
+
* List all API keys
|
|
1443
|
+
*
|
|
1444
|
+
* @example
|
|
1445
|
+
* ```ts
|
|
1446
|
+
* const keys = await client.keys.list();
|
|
1447
|
+
* keys.keys.forEach(k => {
|
|
1448
|
+
* console.log(k.name, k.prefix, k.isActive);
|
|
1449
|
+
* });
|
|
1450
|
+
* ```
|
|
1451
|
+
*/
|
|
1452
|
+
async list(options = {}) {
|
|
1453
|
+
const result = await get(this.ctx, "/v1/keys", {
|
|
1454
|
+
includeRevoked: options.includeRevoked
|
|
1455
|
+
});
|
|
1456
|
+
return result.data;
|
|
1457
|
+
}
|
|
1458
|
+
/**
|
|
1459
|
+
* Create a new API key
|
|
1460
|
+
*
|
|
1461
|
+
* @example
|
|
1462
|
+
* ```ts
|
|
1463
|
+
* const newKey = await client.keys.create({
|
|
1464
|
+
* name: "Production Key",
|
|
1465
|
+
* scopes: ["scrape", "crawl", "status", "data:read"],
|
|
1466
|
+
* expiresInDays: 365
|
|
1467
|
+
* });
|
|
1468
|
+
*
|
|
1469
|
+
* // IMPORTANT: Save this key immediately - it won't be shown again!
|
|
1470
|
+
* console.log(newKey.key);
|
|
1471
|
+
* ```
|
|
1472
|
+
*/
|
|
1473
|
+
async create(options) {
|
|
1474
|
+
const result = await post(this.ctx, "/v1/keys", {
|
|
1475
|
+
name: options.name,
|
|
1476
|
+
scopes: options.scopes,
|
|
1477
|
+
expiresInDays: options.expiresInDays,
|
|
1478
|
+
ipAllowlist: options.ipAllowlist
|
|
1479
|
+
});
|
|
1480
|
+
return result.data;
|
|
1481
|
+
}
|
|
1482
|
+
/**
|
|
1483
|
+
* Get details of a specific key
|
|
1484
|
+
*
|
|
1485
|
+
* @example
|
|
1486
|
+
* ```ts
|
|
1487
|
+
* const key = await client.keys.get("key_abc123");
|
|
1488
|
+
* console.log(key.scopes);
|
|
1489
|
+
* console.log(key.lastUsedAt);
|
|
1490
|
+
* ```
|
|
1491
|
+
*/
|
|
1492
|
+
async get(keyId) {
|
|
1493
|
+
const result = await get(this.ctx, `/v1/keys/${keyId}`);
|
|
1494
|
+
return result.data;
|
|
1495
|
+
}
|
|
1496
|
+
/**
|
|
1497
|
+
* Get usage statistics for a key
|
|
1498
|
+
*
|
|
1499
|
+
* @example
|
|
1500
|
+
* ```ts
|
|
1501
|
+
* const stats = await client.keys.getStats("key_abc123", { days: 30 });
|
|
1502
|
+
* console.log(stats.stats.totalRequests);
|
|
1503
|
+
* console.log(stats.stats.byEndpoint);
|
|
1504
|
+
* ```
|
|
1505
|
+
*/
|
|
1506
|
+
async getStats(keyId, options = {}) {
|
|
1507
|
+
const result = await get(
|
|
1508
|
+
this.ctx,
|
|
1509
|
+
`/v1/keys/${keyId}/stats`,
|
|
1510
|
+
{
|
|
1511
|
+
days: options.days
|
|
1512
|
+
}
|
|
1513
|
+
);
|
|
1514
|
+
return result.data;
|
|
1515
|
+
}
|
|
1516
|
+
/**
|
|
1517
|
+
* Rotate an API key (revoke old, create new)
|
|
1518
|
+
*
|
|
1519
|
+
* @example
|
|
1520
|
+
* ```ts
|
|
1521
|
+
* const rotated = await client.keys.rotate("key_abc123", {
|
|
1522
|
+
* newName: "Production Key v2"
|
|
1523
|
+
* });
|
|
1524
|
+
*
|
|
1525
|
+
* // Old key is now invalid
|
|
1526
|
+
* console.log(rotated.oldKeyRevoked); // true
|
|
1527
|
+
*
|
|
1528
|
+
* // Save new key immediately!
|
|
1529
|
+
* console.log(rotated.newKey.key);
|
|
1530
|
+
* ```
|
|
1531
|
+
*/
|
|
1532
|
+
async rotate(keyId, options = {}) {
|
|
1533
|
+
const result = await post(
|
|
1534
|
+
this.ctx,
|
|
1535
|
+
`/v1/keys/${keyId}/rotate`,
|
|
1536
|
+
{
|
|
1537
|
+
newName: options.newName
|
|
1538
|
+
}
|
|
1539
|
+
);
|
|
1540
|
+
return result.data;
|
|
1541
|
+
}
|
|
1542
|
+
/**
|
|
1543
|
+
* Revoke (delete) an API key
|
|
1544
|
+
*
|
|
1545
|
+
* @example
|
|
1546
|
+
* ```ts
|
|
1547
|
+
* await client.keys.revoke("key_abc123");
|
|
1548
|
+
* ```
|
|
1549
|
+
*/
|
|
1550
|
+
async revoke(keyId) {
|
|
1551
|
+
const result = await del(this.ctx, `/v1/keys/${keyId}`);
|
|
1552
|
+
return result.data;
|
|
1553
|
+
}
|
|
1554
|
+
/**
|
|
1555
|
+
* Revoke all API keys (except the current one)
|
|
1556
|
+
*
|
|
1557
|
+
* @example
|
|
1558
|
+
* ```ts
|
|
1559
|
+
* // Use with caution!
|
|
1560
|
+
* const result = await client.keys.revokeAll();
|
|
1561
|
+
* console.log(`Revoked ${result.count} keys`);
|
|
1562
|
+
* ```
|
|
1563
|
+
*/
|
|
1564
|
+
async revokeAll() {
|
|
1565
|
+
const result = await del(
|
|
1566
|
+
this.ctx,
|
|
1567
|
+
"/v1/keys/all"
|
|
1568
|
+
);
|
|
1569
|
+
return result.data;
|
|
1570
|
+
}
|
|
1571
|
+
/**
|
|
1572
|
+
* Get active keys only
|
|
1573
|
+
*
|
|
1574
|
+
* @example
|
|
1575
|
+
* ```ts
|
|
1576
|
+
* const activeKeys = await client.keys.getActive();
|
|
1577
|
+
* ```
|
|
1578
|
+
*/
|
|
1579
|
+
async getActive() {
|
|
1580
|
+
const all = await this.list({ includeRevoked: false });
|
|
1581
|
+
return all.keys.filter((k) => k.isActive);
|
|
1582
|
+
}
|
|
1583
|
+
/**
|
|
1584
|
+
* Check if a key is valid (active and not expired)
|
|
1585
|
+
*
|
|
1586
|
+
* @example
|
|
1587
|
+
* ```ts
|
|
1588
|
+
* const isValid = await client.keys.isValid("key_abc123");
|
|
1589
|
+
* ```
|
|
1590
|
+
*/
|
|
1591
|
+
async isValid(keyId) {
|
|
1592
|
+
try {
|
|
1593
|
+
const key = await this.get(keyId);
|
|
1594
|
+
if (!key.isActive) return false;
|
|
1595
|
+
if (key.expiresAt && new Date(key.expiresAt) < /* @__PURE__ */ new Date()) return false;
|
|
1596
|
+
return true;
|
|
1597
|
+
} catch {
|
|
1598
|
+
return false;
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
};
|
|
1602
|
+
|
|
1603
|
+
// src/resources/scrape.ts
|
|
1604
|
+
var ScrapeResource = class {
|
|
1605
|
+
constructor(ctx) {
|
|
1606
|
+
this.ctx = ctx;
|
|
1607
|
+
}
|
|
1608
|
+
/**
|
|
1609
|
+
* Create a new scrape job
|
|
1610
|
+
*
|
|
1611
|
+
* @example
|
|
1612
|
+
* ```ts
|
|
1613
|
+
* const job = await client.scrape.create({
|
|
1614
|
+
* url: "https://example.com/product",
|
|
1615
|
+
* extractDeal: true,
|
|
1616
|
+
* screenshot: { enabled: true, format: "webp" }
|
|
1617
|
+
* });
|
|
1618
|
+
* console.log(job.jobId);
|
|
1619
|
+
* ```
|
|
1620
|
+
*/
|
|
1621
|
+
async create(options) {
|
|
1622
|
+
const body = {
|
|
1623
|
+
url: options.url,
|
|
1624
|
+
detectSignals: options.detectSignals ?? true,
|
|
1625
|
+
extractWithAI: options.extractWithAI,
|
|
1626
|
+
extractDeal: options.extractDeal,
|
|
1627
|
+
useAdvancedModel: options.useAdvancedModel,
|
|
1628
|
+
minDealScore: options.minDealScore,
|
|
1629
|
+
screenshot: options.screenshot,
|
|
1630
|
+
excludeTags: options.excludeTags,
|
|
1631
|
+
excludeSelectors: options.excludeSelectors,
|
|
1632
|
+
onlyMainContent: options.onlyMainContent,
|
|
1633
|
+
headers: options.headers,
|
|
1634
|
+
timeout: options.timeout
|
|
1635
|
+
};
|
|
1636
|
+
const result = await post(this.ctx, "/v1/scrape", body);
|
|
1637
|
+
return result.data;
|
|
1638
|
+
}
|
|
1639
|
+
/**
|
|
1640
|
+
* Scrape a URL with deal extraction enabled
|
|
1641
|
+
* Convenience method for common use case
|
|
1642
|
+
*
|
|
1643
|
+
* @example
|
|
1644
|
+
* ```ts
|
|
1645
|
+
* const job = await client.scrape.extractDeal("https://example.com/sale");
|
|
1646
|
+
* ```
|
|
1647
|
+
*/
|
|
1648
|
+
async extractDeal(url, options) {
|
|
1649
|
+
return this.create({
|
|
1650
|
+
url,
|
|
1651
|
+
extractDeal: true,
|
|
1652
|
+
...options
|
|
1653
|
+
});
|
|
1654
|
+
}
|
|
1655
|
+
/**
|
|
1656
|
+
* Scrape a URL with screenshot capture
|
|
1657
|
+
* Convenience method for screenshot capture
|
|
1658
|
+
*
|
|
1659
|
+
* @example
|
|
1660
|
+
* ```ts
|
|
1661
|
+
* const job = await client.scrape.withScreenshot("https://example.com", {
|
|
1662
|
+
* format: "webp",
|
|
1663
|
+
* fullPage: true
|
|
1664
|
+
* });
|
|
1665
|
+
* ```
|
|
1666
|
+
*/
|
|
1667
|
+
async withScreenshot(url, screenshotOptions, options) {
|
|
1668
|
+
return this.create({
|
|
1669
|
+
url,
|
|
1670
|
+
screenshot: {
|
|
1671
|
+
enabled: true,
|
|
1672
|
+
...screenshotOptions
|
|
1673
|
+
},
|
|
1674
|
+
...options
|
|
1675
|
+
});
|
|
1676
|
+
}
|
|
1677
|
+
};
|
|
1678
|
+
|
|
1679
|
+
// src/resources/status.ts
|
|
1680
|
+
var StatusResource = class {
|
|
1681
|
+
constructor(ctx) {
|
|
1682
|
+
this.ctx = ctx;
|
|
1683
|
+
}
|
|
1684
|
+
/**
|
|
1685
|
+
* Get the status of a job
|
|
1686
|
+
*
|
|
1687
|
+
* @example
|
|
1688
|
+
* ```ts
|
|
1689
|
+
* const status = await client.status.get("job_abc123");
|
|
1690
|
+
* console.log(status.status); // "completed"
|
|
1691
|
+
* console.log(status.result);
|
|
1692
|
+
* ```
|
|
1693
|
+
*/
|
|
1694
|
+
async get(jobId) {
|
|
1695
|
+
const result = await get(
|
|
1696
|
+
this.ctx,
|
|
1697
|
+
`/v1/status/${jobId}`
|
|
1698
|
+
);
|
|
1699
|
+
return result.data;
|
|
1700
|
+
}
|
|
1701
|
+
/**
|
|
1702
|
+
* Get deals found by a job
|
|
1703
|
+
*
|
|
1704
|
+
* @example
|
|
1705
|
+
* ```ts
|
|
1706
|
+
* const deals = await client.status.getDeals("job_abc123", {
|
|
1707
|
+
* minScore: 70,
|
|
1708
|
+
* limit: 20
|
|
1709
|
+
* });
|
|
1710
|
+
* console.log(deals.deals);
|
|
1711
|
+
* ```
|
|
1712
|
+
*/
|
|
1713
|
+
async getDeals(jobId, options) {
|
|
1714
|
+
const result = await get(
|
|
1715
|
+
this.ctx,
|
|
1716
|
+
`/v1/status/${jobId}/deals`,
|
|
1717
|
+
{
|
|
1718
|
+
minScore: options?.minScore,
|
|
1719
|
+
limit: options?.limit
|
|
1720
|
+
}
|
|
1721
|
+
);
|
|
1722
|
+
return result.data;
|
|
1723
|
+
}
|
|
1724
|
+
/**
|
|
1725
|
+
* Resume a paused or failed job from checkpoint
|
|
1726
|
+
*
|
|
1727
|
+
* @example
|
|
1728
|
+
* ```ts
|
|
1729
|
+
* const resumed = await client.status.resume("job_abc123");
|
|
1730
|
+
* console.log(resumed.newJobId);
|
|
1731
|
+
* ```
|
|
1732
|
+
*/
|
|
1733
|
+
async resume(jobId) {
|
|
1734
|
+
const result = await post(
|
|
1735
|
+
this.ctx,
|
|
1736
|
+
`/v1/status/${jobId}/resume`
|
|
1737
|
+
);
|
|
1738
|
+
return result.data;
|
|
1739
|
+
}
|
|
1740
|
+
/**
|
|
1741
|
+
* Get detailed metrics for a job
|
|
1742
|
+
*
|
|
1743
|
+
* @example
|
|
1744
|
+
* ```ts
|
|
1745
|
+
* const metrics = await client.status.getMetrics("job_abc123");
|
|
1746
|
+
* console.log(metrics.metrics.successRate);
|
|
1747
|
+
* ```
|
|
1748
|
+
*/
|
|
1749
|
+
async getMetrics(jobId) {
|
|
1750
|
+
const result = await get(
|
|
1751
|
+
this.ctx,
|
|
1752
|
+
`/v1/status/${jobId}/metrics`
|
|
1753
|
+
);
|
|
1754
|
+
return result.data;
|
|
1755
|
+
}
|
|
1756
|
+
/**
|
|
1757
|
+
* Cancel a pending or active job
|
|
1758
|
+
*
|
|
1759
|
+
* @example
|
|
1760
|
+
* ```ts
|
|
1761
|
+
* const cancelled = await client.status.cancel("job_abc123");
|
|
1762
|
+
* console.log(cancelled.success);
|
|
1763
|
+
* ```
|
|
1764
|
+
*/
|
|
1765
|
+
async cancel(jobId) {
|
|
1766
|
+
const result = await del(
|
|
1767
|
+
this.ctx,
|
|
1768
|
+
`/v1/status/${jobId}`
|
|
1769
|
+
);
|
|
1770
|
+
return result.data;
|
|
1771
|
+
}
|
|
1772
|
+
/**
|
|
1773
|
+
* Check if a job is complete
|
|
1774
|
+
* Convenience method for polling
|
|
1775
|
+
*
|
|
1776
|
+
* @example
|
|
1777
|
+
* ```ts
|
|
1778
|
+
* const isComplete = await client.status.isComplete("job_abc123");
|
|
1779
|
+
* ```
|
|
1780
|
+
*/
|
|
1781
|
+
async isComplete(jobId) {
|
|
1782
|
+
const status = await this.get(jobId);
|
|
1783
|
+
return status.status === "completed" || status.status === "failed";
|
|
1784
|
+
}
|
|
1785
|
+
/**
|
|
1786
|
+
* Check if a job succeeded
|
|
1787
|
+
* Convenience method for result checking
|
|
1788
|
+
*
|
|
1789
|
+
* @example
|
|
1790
|
+
* ```ts
|
|
1791
|
+
* const succeeded = await client.status.succeeded("job_abc123");
|
|
1792
|
+
* ```
|
|
1793
|
+
*/
|
|
1794
|
+
async succeeded(jobId) {
|
|
1795
|
+
const status = await this.get(jobId);
|
|
1796
|
+
return status.status === "completed";
|
|
1797
|
+
}
|
|
1798
|
+
/**
|
|
1799
|
+
* Get the result of a completed job
|
|
1800
|
+
* Throws if job is not complete
|
|
1801
|
+
*
|
|
1802
|
+
* @example
|
|
1803
|
+
* ```ts
|
|
1804
|
+
* const result = await client.status.getResult("job_abc123");
|
|
1805
|
+
* ```
|
|
1806
|
+
*/
|
|
1807
|
+
async getResult(jobId) {
|
|
1808
|
+
const status = await this.get(jobId);
|
|
1809
|
+
if (status.status === "failed") {
|
|
1810
|
+
throw new Error(`Job failed: ${status.error || "Unknown error"}`);
|
|
1811
|
+
}
|
|
1812
|
+
if (status.status !== "completed") {
|
|
1813
|
+
throw new Error(`Job not complete. Status: ${status.status}`);
|
|
1814
|
+
}
|
|
1815
|
+
return status.result;
|
|
1816
|
+
}
|
|
1817
|
+
};
|
|
1818
|
+
|
|
1819
|
+
// src/resources/webhooks.ts
|
|
1820
|
+
var WebhooksResource = class {
|
|
1821
|
+
constructor(ctx) {
|
|
1822
|
+
this.ctx = ctx;
|
|
1823
|
+
}
|
|
1824
|
+
/**
|
|
1825
|
+
* Create a new webhook
|
|
1826
|
+
*
|
|
1827
|
+
* @example
|
|
1828
|
+
* ```ts
|
|
1829
|
+
* const webhook = await client.webhooks.create({
|
|
1830
|
+
* event: "deal.found",
|
|
1831
|
+
* url: "https://my-server.com/webhooks/deals",
|
|
1832
|
+
* secret: "my-webhook-secret",
|
|
1833
|
+
* minDealScore: 70
|
|
1834
|
+
* });
|
|
1835
|
+
* console.log(webhook.webhookId);
|
|
1836
|
+
* ```
|
|
1837
|
+
*/
|
|
1838
|
+
async create(options) {
|
|
1839
|
+
const result = await post(this.ctx, "/v1/webhooks", {
|
|
1840
|
+
event: options.event,
|
|
1841
|
+
url: options.url,
|
|
1842
|
+
secret: options.secret,
|
|
1843
|
+
minDealScore: options.minDealScore,
|
|
1844
|
+
categories: options.categories,
|
|
1845
|
+
active: options.active
|
|
1846
|
+
});
|
|
1847
|
+
return result.data;
|
|
1848
|
+
}
|
|
1849
|
+
/**
|
|
1850
|
+
* List all webhooks
|
|
1851
|
+
*
|
|
1852
|
+
* @example
|
|
1853
|
+
* ```ts
|
|
1854
|
+
* const webhooks = await client.webhooks.list();
|
|
1855
|
+
* webhooks.webhooks.forEach(w => {
|
|
1856
|
+
* console.log(w.event, w.url, w.active);
|
|
1857
|
+
* });
|
|
1858
|
+
* ```
|
|
1859
|
+
*/
|
|
1860
|
+
async list() {
|
|
1861
|
+
const result = await get(this.ctx, "/v1/webhooks");
|
|
1862
|
+
return result.data;
|
|
1863
|
+
}
|
|
1864
|
+
/**
|
|
1865
|
+
* Get a webhook by ID
|
|
1866
|
+
*
|
|
1867
|
+
* @example
|
|
1868
|
+
* ```ts
|
|
1869
|
+
* const webhook = await client.webhooks.get("webhook_abc123");
|
|
1870
|
+
* console.log(webhook.event);
|
|
1871
|
+
* ```
|
|
1872
|
+
*/
|
|
1873
|
+
async get(webhookId) {
|
|
1874
|
+
const result = await get(
|
|
1875
|
+
this.ctx,
|
|
1876
|
+
`/v1/webhooks/${webhookId}`
|
|
1877
|
+
);
|
|
1878
|
+
return result.data;
|
|
1879
|
+
}
|
|
1880
|
+
/**
|
|
1881
|
+
* Update a webhook
|
|
1882
|
+
*
|
|
1883
|
+
* @example
|
|
1884
|
+
* ```ts
|
|
1885
|
+
* const updated = await client.webhooks.update("webhook_abc123", {
|
|
1886
|
+
* minDealScore: 80,
|
|
1887
|
+
* active: false
|
|
1888
|
+
* });
|
|
1889
|
+
* ```
|
|
1890
|
+
*/
|
|
1891
|
+
async update(webhookId, options) {
|
|
1892
|
+
const result = await patch(
|
|
1893
|
+
this.ctx,
|
|
1894
|
+
`/v1/webhooks/${webhookId}`,
|
|
1895
|
+
{
|
|
1896
|
+
url: options.url,
|
|
1897
|
+
secret: options.secret,
|
|
1898
|
+
minDealScore: options.minDealScore,
|
|
1899
|
+
categories: options.categories,
|
|
1900
|
+
active: options.active
|
|
1901
|
+
}
|
|
1902
|
+
);
|
|
1903
|
+
return result.data;
|
|
1904
|
+
}
|
|
1905
|
+
/**
|
|
1906
|
+
* Delete a webhook
|
|
1907
|
+
*
|
|
1908
|
+
* @example
|
|
1909
|
+
* ```ts
|
|
1910
|
+
* await client.webhooks.delete("webhook_abc123");
|
|
1911
|
+
* ```
|
|
1912
|
+
*/
|
|
1913
|
+
async delete(webhookId) {
|
|
1914
|
+
const result = await del(
|
|
1915
|
+
this.ctx,
|
|
1916
|
+
`/v1/webhooks/${webhookId}`
|
|
1917
|
+
);
|
|
1918
|
+
return result.data;
|
|
1919
|
+
}
|
|
1920
|
+
/**
|
|
1921
|
+
* Test a webhook by sending a test payload
|
|
1922
|
+
*
|
|
1923
|
+
* @example
|
|
1924
|
+
* ```ts
|
|
1925
|
+
* const result = await client.webhooks.test("webhook_abc123");
|
|
1926
|
+
* if (result.delivered) {
|
|
1927
|
+
* console.log(`Delivered in ${result.responseTime}ms`);
|
|
1928
|
+
* } else {
|
|
1929
|
+
* console.log(`Failed: ${result.error}`);
|
|
1930
|
+
* }
|
|
1931
|
+
* ```
|
|
1932
|
+
*/
|
|
1933
|
+
async test(webhookId) {
|
|
1934
|
+
const result = await post(
|
|
1935
|
+
this.ctx,
|
|
1936
|
+
`/v1/webhooks/${webhookId}/test`
|
|
1937
|
+
);
|
|
1938
|
+
return result.data;
|
|
1939
|
+
}
|
|
1940
|
+
/**
|
|
1941
|
+
* Enable a webhook
|
|
1942
|
+
* Convenience method for activating a webhook
|
|
1943
|
+
*
|
|
1944
|
+
* @example
|
|
1945
|
+
* ```ts
|
|
1946
|
+
* await client.webhooks.enable("webhook_abc123");
|
|
1947
|
+
* ```
|
|
1948
|
+
*/
|
|
1949
|
+
async enable(webhookId) {
|
|
1950
|
+
return this.update(webhookId, { active: true });
|
|
1951
|
+
}
|
|
1952
|
+
/**
|
|
1953
|
+
* Disable a webhook
|
|
1954
|
+
* Convenience method for deactivating a webhook
|
|
1955
|
+
*
|
|
1956
|
+
* @example
|
|
1957
|
+
* ```ts
|
|
1958
|
+
* await client.webhooks.disable("webhook_abc123");
|
|
1959
|
+
* ```
|
|
1960
|
+
*/
|
|
1961
|
+
async disable(webhookId) {
|
|
1962
|
+
return this.update(webhookId, { active: false });
|
|
1963
|
+
}
|
|
1964
|
+
/**
|
|
1965
|
+
* Get active webhooks only
|
|
1966
|
+
*
|
|
1967
|
+
* @example
|
|
1968
|
+
* ```ts
|
|
1969
|
+
* const active = await client.webhooks.getActive();
|
|
1970
|
+
* ```
|
|
1971
|
+
*/
|
|
1972
|
+
async getActive() {
|
|
1973
|
+
const all = await this.list();
|
|
1974
|
+
return all.webhooks.filter((w) => w.active);
|
|
1975
|
+
}
|
|
1976
|
+
/**
|
|
1977
|
+
* Get webhooks by event type
|
|
1978
|
+
*
|
|
1979
|
+
* @example
|
|
1980
|
+
* ```ts
|
|
1981
|
+
* const dealWebhooks = await client.webhooks.getByEvent("deal.found");
|
|
1982
|
+
* ```
|
|
1983
|
+
*/
|
|
1984
|
+
async getByEvent(event) {
|
|
1985
|
+
const all = await this.list();
|
|
1986
|
+
return all.webhooks.filter((w) => w.event === event);
|
|
1987
|
+
}
|
|
1988
|
+
};
|
|
1989
|
+
|
|
1990
|
+
// src/client.ts
|
|
1991
|
+
var DealCrawl = class {
|
|
1992
|
+
/** Internal request context */
|
|
1993
|
+
ctx;
|
|
1994
|
+
// ============================================
|
|
1995
|
+
// RESOURCES
|
|
1996
|
+
// ============================================
|
|
1997
|
+
/**
|
|
1998
|
+
* Scrape resource - Single page scraping
|
|
1999
|
+
*
|
|
2000
|
+
* @example
|
|
2001
|
+
* ```ts
|
|
2002
|
+
* const job = await client.scrape.create({
|
|
2003
|
+
* url: "https://example.com",
|
|
2004
|
+
* extractDeal: true
|
|
2005
|
+
* });
|
|
2006
|
+
* ```
|
|
2007
|
+
*/
|
|
2008
|
+
scrape;
|
|
2009
|
+
/**
|
|
2010
|
+
* Crawl resource - Website crawling
|
|
2011
|
+
*
|
|
2012
|
+
* @example
|
|
2013
|
+
* ```ts
|
|
2014
|
+
* const job = await client.crawl.create({
|
|
2015
|
+
* url: "https://shop.example.com",
|
|
2016
|
+
* maxPages: 100
|
|
2017
|
+
* });
|
|
2018
|
+
*
|
|
2019
|
+
* // With template
|
|
2020
|
+
* const job = await client.crawl.withTemplate("ecommerce", {
|
|
2021
|
+
* url: "https://shop.example.com"
|
|
2022
|
+
* });
|
|
2023
|
+
* ```
|
|
2024
|
+
*/
|
|
2025
|
+
crawl;
|
|
2026
|
+
/**
|
|
2027
|
+
* Extract resource - LLM-based structured data extraction
|
|
2028
|
+
*
|
|
2029
|
+
* @example
|
|
2030
|
+
* ```ts
|
|
2031
|
+
* const job = await client.extract.create({
|
|
2032
|
+
* url: "https://example.com/product",
|
|
2033
|
+
* schema: { type: "object", properties: {...} }
|
|
2034
|
+
* });
|
|
2035
|
+
* ```
|
|
2036
|
+
*/
|
|
2037
|
+
extract;
|
|
2038
|
+
/**
|
|
2039
|
+
* Dork resource - Google Dork searches
|
|
2040
|
+
*
|
|
2041
|
+
* @example
|
|
2042
|
+
* ```ts
|
|
2043
|
+
* const job = await client.dork.create({
|
|
2044
|
+
* query: "discount coupon",
|
|
2045
|
+
* site: "amazon.com"
|
|
2046
|
+
* });
|
|
2047
|
+
* ```
|
|
2048
|
+
*/
|
|
2049
|
+
dork;
|
|
2050
|
+
/**
|
|
2051
|
+
* Status resource - Job status management
|
|
2052
|
+
*
|
|
2053
|
+
* @example
|
|
2054
|
+
* ```ts
|
|
2055
|
+
* const status = await client.status.get(jobId);
|
|
2056
|
+
* const deals = await client.status.getDeals(jobId);
|
|
2057
|
+
* await client.status.cancel(jobId);
|
|
2058
|
+
* ```
|
|
2059
|
+
*/
|
|
2060
|
+
status;
|
|
2061
|
+
/**
|
|
2062
|
+
* Data resource - Jobs and deals data access
|
|
2063
|
+
*
|
|
2064
|
+
* @example
|
|
2065
|
+
* ```ts
|
|
2066
|
+
* const jobs = await client.data.listJobs();
|
|
2067
|
+
* const deals = await client.data.listDeals({ minScore: 70 });
|
|
2068
|
+
* const stats = await client.data.getStats();
|
|
2069
|
+
* ```
|
|
2070
|
+
*/
|
|
2071
|
+
data;
|
|
2072
|
+
/**
|
|
2073
|
+
* Webhooks resource - Webhook management
|
|
2074
|
+
*
|
|
2075
|
+
* @example
|
|
2076
|
+
* ```ts
|
|
2077
|
+
* await client.webhooks.create({
|
|
2078
|
+
* event: "deal.found",
|
|
2079
|
+
* url: "https://..."
|
|
2080
|
+
* });
|
|
2081
|
+
* ```
|
|
2082
|
+
*/
|
|
2083
|
+
webhooks;
|
|
2084
|
+
/**
|
|
2085
|
+
* Keys resource - API key management
|
|
2086
|
+
*
|
|
2087
|
+
* @example
|
|
2088
|
+
* ```ts
|
|
2089
|
+
* const newKey = await client.keys.create({
|
|
2090
|
+
* name: "Production",
|
|
2091
|
+
* scopes: ["scrape", "status"]
|
|
2092
|
+
* });
|
|
2093
|
+
* ```
|
|
2094
|
+
*/
|
|
2095
|
+
keys;
|
|
2096
|
+
/**
|
|
2097
|
+
* Account resource - Account info and preferences
|
|
2098
|
+
*
|
|
2099
|
+
* @example
|
|
2100
|
+
* ```ts
|
|
2101
|
+
* const account = await client.account.get();
|
|
2102
|
+
* await client.account.updatePreferences({
|
|
2103
|
+
* minDealScore: 70
|
|
2104
|
+
* });
|
|
2105
|
+
* ```
|
|
2106
|
+
*/
|
|
2107
|
+
account;
|
|
2108
|
+
// ============================================
|
|
2109
|
+
// CONSTRUCTOR
|
|
2110
|
+
// ============================================
|
|
2111
|
+
/**
|
|
2112
|
+
* Create a new DealCrawl client
|
|
2113
|
+
*
|
|
2114
|
+
* @param config - Client configuration
|
|
2115
|
+
*
|
|
2116
|
+
* @example
|
|
2117
|
+
* ```ts
|
|
2118
|
+
* // Minimal config
|
|
2119
|
+
* const client = new DealCrawl({ apiKey: "sk_xxx" });
|
|
2120
|
+
*
|
|
2121
|
+
* // Full config
|
|
2122
|
+
* const client = new DealCrawl({
|
|
2123
|
+
* apiKey: "sk_xxx",
|
|
2124
|
+
* baseUrl: "https://api.dealcrawl.dev",
|
|
2125
|
+
* timeout: 30000,
|
|
2126
|
+
* maxRetries: 3,
|
|
2127
|
+
* retryDelay: 1000,
|
|
2128
|
+
* onRateLimit: (info) => console.log("Rate limited!", info)
|
|
2129
|
+
* });
|
|
2130
|
+
* ```
|
|
2131
|
+
*/
|
|
2132
|
+
constructor(config) {
|
|
2133
|
+
if (!config.apiKey || !config.apiKey.trim()) {
|
|
2134
|
+
throw new Error("API key is required");
|
|
2135
|
+
}
|
|
2136
|
+
this.ctx = {
|
|
2137
|
+
apiKey: config.apiKey,
|
|
2138
|
+
baseUrl: config.baseUrl ?? DEFAULT_CONFIG.baseUrl,
|
|
2139
|
+
timeout: config.timeout ?? DEFAULT_CONFIG.timeout,
|
|
2140
|
+
maxRetries: config.maxRetries ?? DEFAULT_CONFIG.maxRetries,
|
|
2141
|
+
retryDelay: config.retryDelay ?? DEFAULT_CONFIG.retryDelay,
|
|
2142
|
+
onRateLimit: config.onRateLimit
|
|
2143
|
+
};
|
|
2144
|
+
this.scrape = new ScrapeResource(this.ctx);
|
|
2145
|
+
this.crawl = new CrawlResource(this.ctx);
|
|
2146
|
+
this.extract = new ExtractResource(this.ctx);
|
|
2147
|
+
this.dork = new DorkResource(this.ctx);
|
|
2148
|
+
this.status = new StatusResource(this.ctx);
|
|
2149
|
+
this.data = new DataResource(this.ctx);
|
|
2150
|
+
this.webhooks = new WebhooksResource(this.ctx);
|
|
2151
|
+
this.keys = new KeysResource(this.ctx);
|
|
2152
|
+
this.account = new AccountResource(this.ctx);
|
|
2153
|
+
}
|
|
2154
|
+
// ============================================
|
|
2155
|
+
// POLLING METHODS
|
|
2156
|
+
// ============================================
|
|
2157
|
+
/**
|
|
2158
|
+
* Wait for a job to complete
|
|
2159
|
+
* Polls the status endpoint until the job finishes
|
|
2160
|
+
*
|
|
2161
|
+
* @example
|
|
2162
|
+
* ```ts
|
|
2163
|
+
* const job = await client.scrape.create({ url: "..." });
|
|
2164
|
+
*
|
|
2165
|
+
* // Simple wait
|
|
2166
|
+
* const result = await client.waitForResult(job.jobId);
|
|
2167
|
+
*
|
|
2168
|
+
* // With options
|
|
2169
|
+
* const result = await client.waitForResult(job.jobId, {
|
|
2170
|
+
* pollInterval: 1000,
|
|
2171
|
+
* timeout: 60000,
|
|
2172
|
+
* onProgress: (status) => console.log(status.progress)
|
|
2173
|
+
* });
|
|
2174
|
+
* ```
|
|
2175
|
+
*/
|
|
2176
|
+
async waitForResult(jobId, options) {
|
|
2177
|
+
return waitForResult(this.ctx, jobId, options);
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* Wait for multiple jobs to complete
|
|
2181
|
+
* Returns when all jobs are done
|
|
2182
|
+
*
|
|
2183
|
+
* @example
|
|
2184
|
+
* ```ts
|
|
2185
|
+
* const jobs = await Promise.all([
|
|
2186
|
+
* client.scrape.create({ url: "url1" }),
|
|
2187
|
+
* client.scrape.create({ url: "url2" }),
|
|
2188
|
+
* ]);
|
|
2189
|
+
*
|
|
2190
|
+
* const results = await client.waitForAll(jobs.map(j => j.jobId));
|
|
2191
|
+
* ```
|
|
2192
|
+
*/
|
|
2193
|
+
async waitForAll(jobIds, options) {
|
|
2194
|
+
return waitForAll(this.ctx, jobIds, options);
|
|
2195
|
+
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Wait for any job to complete
|
|
2198
|
+
* Returns as soon as one job finishes
|
|
2199
|
+
*
|
|
2200
|
+
* @example
|
|
2201
|
+
* ```ts
|
|
2202
|
+
* const jobIds = ["job1", "job2", "job3"];
|
|
2203
|
+
* const firstResult = await client.waitForAny(jobIds);
|
|
2204
|
+
* ```
|
|
2205
|
+
*/
|
|
2206
|
+
async waitForAny(jobIds, options) {
|
|
2207
|
+
return waitForAny(this.ctx, jobIds, options);
|
|
2208
|
+
}
|
|
2209
|
+
// ============================================
|
|
2210
|
+
// CONVENIENCE METHODS
|
|
2211
|
+
// ============================================
|
|
2212
|
+
/**
|
|
2213
|
+
* Scrape a URL and wait for result
|
|
2214
|
+
* Combines create and waitForResult
|
|
2215
|
+
*
|
|
2216
|
+
* @example
|
|
2217
|
+
* ```ts
|
|
2218
|
+
* const result = await client.scrapeAndWait({
|
|
2219
|
+
* url: "https://example.com",
|
|
2220
|
+
* extractDeal: true
|
|
2221
|
+
* });
|
|
2222
|
+
* ```
|
|
2223
|
+
*/
|
|
2224
|
+
async scrapeAndWait(options, waitOptions) {
|
|
2225
|
+
const job = await this.scrape.create(options);
|
|
2226
|
+
return this.waitForResult(job.jobId, waitOptions);
|
|
2227
|
+
}
|
|
2228
|
+
/**
|
|
2229
|
+
* Crawl a URL and wait for result
|
|
2230
|
+
* Combines create and waitForResult
|
|
2231
|
+
*
|
|
2232
|
+
* @example
|
|
2233
|
+
* ```ts
|
|
2234
|
+
* const result = await client.crawlAndWait({
|
|
2235
|
+
* url: "https://shop.example.com",
|
|
2236
|
+
* maxPages: 50
|
|
2237
|
+
* });
|
|
2238
|
+
* ```
|
|
2239
|
+
*/
|
|
2240
|
+
async crawlAndWait(options, waitOptions) {
|
|
2241
|
+
const job = await this.crawl.create(options);
|
|
2242
|
+
return this.waitForResult(job.jobId, waitOptions);
|
|
2243
|
+
}
|
|
2244
|
+
/**
|
|
2245
|
+
* Extract data and wait for result
|
|
2246
|
+
* Combines create and waitForResult
|
|
2247
|
+
*
|
|
2248
|
+
* @example
|
|
2249
|
+
* ```ts
|
|
2250
|
+
* const result = await client.extractAndWait({
|
|
2251
|
+
* url: "https://example.com/product",
|
|
2252
|
+
* schema: { type: "object", properties: {...} }
|
|
2253
|
+
* });
|
|
2254
|
+
* ```
|
|
2255
|
+
*/
|
|
2256
|
+
async extractAndWait(options, waitOptions) {
|
|
2257
|
+
const job = await this.extract.create(options);
|
|
2258
|
+
return this.waitForResult(job.jobId, waitOptions);
|
|
2259
|
+
}
|
|
2260
|
+
};
|
|
2261
|
+
|
|
2262
|
+
exports.AccountResource = AccountResource;
|
|
2263
|
+
exports.CrawlResource = CrawlResource;
|
|
2264
|
+
exports.DEFAULT_CONFIG = DEFAULT_CONFIG;
|
|
2265
|
+
exports.DataResource = DataResource;
|
|
2266
|
+
exports.DealCrawl = DealCrawl;
|
|
2267
|
+
exports.DealCrawlError = DealCrawlError;
|
|
2268
|
+
exports.DorkResource = DorkResource;
|
|
2269
|
+
exports.ERROR_CODES = ERROR_CODES;
|
|
2270
|
+
exports.ExtractResource = ExtractResource;
|
|
2271
|
+
exports.KeysResource = KeysResource;
|
|
2272
|
+
exports.ScrapeResource = ScrapeResource;
|
|
2273
|
+
exports.StatusResource = StatusResource;
|
|
2274
|
+
exports.WebhooksResource = WebhooksResource;
|
|
2275
|
+
exports.default = DealCrawl;
|
|
2276
|
+
exports.pollUntil = pollUntil;
|
|
2277
|
+
exports.waitForAll = waitForAll;
|
|
2278
|
+
exports.waitForAny = waitForAny;
|
|
2279
|
+
exports.waitForResult = waitForResult;
|
|
2280
|
+
//# sourceMappingURL=index.js.map
|
|
65
2281
|
//# sourceMappingURL=index.js.map
|