@dealcrawl/sdk 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +540 -0
- package/dist/sdk/src/client.d.ts +285 -0
- package/dist/sdk/src/client.d.ts.map +1 -0
- package/dist/sdk/src/client.js +336 -0
- package/dist/sdk/src/client.js.map +1 -0
- package/dist/sdk/src/error.d.ts +54 -0
- package/dist/sdk/src/error.d.ts.map +1 -0
- package/dist/sdk/src/error.js +115 -0
- package/dist/sdk/src/error.js.map +1 -0
- package/dist/sdk/src/index.d.ts +58 -0
- package/dist/sdk/src/index.d.ts.map +1 -0
- package/dist/sdk/src/index.js +65 -0
- package/dist/sdk/src/index.js.map +1 -0
- package/dist/sdk/src/resources/account.d.ts +143 -0
- package/dist/sdk/src/resources/account.d.ts.map +1 -0
- package/dist/sdk/src/resources/account.js +186 -0
- package/dist/sdk/src/resources/account.js.map +1 -0
- package/dist/sdk/src/resources/crawl.d.ts +87 -0
- package/dist/sdk/src/resources/crawl.d.ts.map +1 -0
- package/dist/sdk/src/resources/crawl.js +205 -0
- package/dist/sdk/src/resources/crawl.js.map +1 -0
- package/dist/sdk/src/resources/data.d.ts +157 -0
- package/dist/sdk/src/resources/data.d.ts.map +1 -0
- package/dist/sdk/src/resources/data.js +239 -0
- package/dist/sdk/src/resources/data.js.map +1 -0
- package/dist/sdk/src/resources/dork.d.ts +104 -0
- package/dist/sdk/src/resources/dork.d.ts.map +1 -0
- package/dist/sdk/src/resources/dork.js +163 -0
- package/dist/sdk/src/resources/dork.js.map +1 -0
- package/dist/sdk/src/resources/extract.d.ts +105 -0
- package/dist/sdk/src/resources/extract.d.ts.map +1 -0
- package/dist/sdk/src/resources/extract.js +246 -0
- package/dist/sdk/src/resources/extract.js.map +1 -0
- package/dist/sdk/src/resources/index.d.ts +14 -0
- package/dist/sdk/src/resources/index.d.ts.map +1 -0
- package/dist/sdk/src/resources/index.js +14 -0
- package/dist/sdk/src/resources/index.js.map +1 -0
- package/dist/sdk/src/resources/keys.d.ts +124 -0
- package/dist/sdk/src/resources/keys.d.ts.map +1 -0
- package/dist/sdk/src/resources/keys.js +168 -0
- package/dist/sdk/src/resources/keys.js.map +1 -0
- package/dist/sdk/src/resources/scrape.d.ts +53 -0
- package/dist/sdk/src/resources/scrape.d.ts.map +1 -0
- package/dist/sdk/src/resources/scrape.js +85 -0
- package/dist/sdk/src/resources/scrape.js.map +1 -0
- package/dist/sdk/src/resources/status.d.ts +100 -0
- package/dist/sdk/src/resources/status.d.ts.map +1 -0
- package/dist/sdk/src/resources/status.js +133 -0
- package/dist/sdk/src/resources/status.js.map +1 -0
- package/dist/sdk/src/resources/webhooks.d.ts +126 -0
- package/dist/sdk/src/resources/webhooks.d.ts.map +1 -0
- package/dist/sdk/src/resources/webhooks.js +167 -0
- package/dist/sdk/src/resources/webhooks.js.map +1 -0
- package/dist/sdk/src/types/config.d.ts +45 -0
- package/dist/sdk/src/types/config.d.ts.map +1 -0
- package/dist/sdk/src/types/config.js +10 -0
- package/dist/sdk/src/types/config.js.map +1 -0
- package/dist/sdk/src/types/index.d.ts +8 -0
- package/dist/sdk/src/types/index.d.ts.map +1 -0
- package/dist/sdk/src/types/index.js +8 -0
- package/dist/sdk/src/types/index.js.map +1 -0
- package/dist/sdk/src/types/options.d.ts +286 -0
- package/dist/sdk/src/types/options.d.ts.map +1 -0
- package/dist/sdk/src/types/options.js +6 -0
- package/dist/sdk/src/types/options.js.map +1 -0
- package/dist/sdk/src/types/responses.d.ts +385 -0
- package/dist/sdk/src/types/responses.d.ts.map +1 -0
- package/dist/sdk/src/types/responses.js +6 -0
- package/dist/sdk/src/types/responses.js.map +1 -0
- package/dist/sdk/src/utils/polling.d.ts +57 -0
- package/dist/sdk/src/utils/polling.d.ts.map +1 -0
- package/dist/sdk/src/utils/polling.js +110 -0
- package/dist/sdk/src/utils/polling.js.map +1 -0
- package/dist/sdk/src/utils/request.d.ts +47 -0
- package/dist/sdk/src/utils/request.d.ts.map +1 -0
- package/dist/sdk/src/utils/request.js +161 -0
- package/dist/sdk/src/utils/request.js.map +1 -0
- package/dist/shared/src/constants/errors.d.ts +26 -0
- package/dist/shared/src/constants/errors.d.ts.map +1 -0
- package/dist/shared/src/constants/errors.js +39 -0
- package/dist/shared/src/constants/errors.js.map +1 -0
- package/dist/shared/src/constants/http.d.ts +26 -0
- package/dist/shared/src/constants/http.d.ts.map +1 -0
- package/dist/shared/src/constants/http.js +26 -0
- package/dist/shared/src/constants/http.js.map +1 -0
- package/dist/shared/src/constants/index.d.ts +4 -0
- package/dist/shared/src/constants/index.d.ts.map +1 -0
- package/dist/shared/src/constants/index.js +5 -0
- package/dist/shared/src/constants/index.js.map +1 -0
- package/dist/shared/src/constants/limits.d.ts +52 -0
- package/dist/shared/src/constants/limits.d.ts.map +1 -0
- package/dist/shared/src/constants/limits.js +43 -0
- package/dist/shared/src/constants/limits.js.map +1 -0
- package/dist/shared/src/index.d.ts +5 -0
- package/dist/shared/src/index.d.ts.map +1 -0
- package/dist/shared/src/index.js +11 -0
- package/dist/shared/src/index.js.map +1 -0
- package/dist/shared/src/lib/index.d.ts +2 -0
- package/dist/shared/src/lib/index.d.ts.map +1 -0
- package/dist/shared/src/lib/index.js +2 -0
- package/dist/shared/src/lib/index.js.map +1 -0
- package/dist/shared/src/lib/redis.d.ts +14 -0
- package/dist/shared/src/lib/redis.d.ts.map +1 -0
- package/dist/shared/src/lib/redis.js +60 -0
- package/dist/shared/src/lib/redis.js.map +1 -0
- package/dist/shared/src/types/api-key.types.d.ts +94 -0
- package/dist/shared/src/types/api-key.types.d.ts.map +1 -0
- package/dist/shared/src/types/api-key.types.js +30 -0
- package/dist/shared/src/types/api-key.types.js.map +1 -0
- package/dist/shared/src/types/api.types.d.ts +38 -0
- package/dist/shared/src/types/api.types.d.ts.map +1 -0
- package/dist/shared/src/types/api.types.js +2 -0
- package/dist/shared/src/types/api.types.js.map +1 -0
- package/dist/shared/src/types/client.types.d.ts +73 -0
- package/dist/shared/src/types/client.types.d.ts.map +1 -0
- package/dist/shared/src/types/client.types.js +9 -0
- package/dist/shared/src/types/client.types.js.map +1 -0
- package/dist/shared/src/types/crawl.types.d.ts +65 -0
- package/dist/shared/src/types/crawl.types.d.ts.map +1 -0
- package/dist/shared/src/types/crawl.types.js +2 -0
- package/dist/shared/src/types/crawl.types.js.map +1 -0
- package/dist/shared/src/types/deal.types.d.ts +210 -0
- package/dist/shared/src/types/deal.types.d.ts.map +1 -0
- package/dist/shared/src/types/deal.types.js +6 -0
- package/dist/shared/src/types/deal.types.js.map +1 -0
- package/dist/shared/src/types/dork.types.d.ts +29 -0
- package/dist/shared/src/types/dork.types.d.ts.map +1 -0
- package/dist/shared/src/types/dork.types.js +2 -0
- package/dist/shared/src/types/dork.types.js.map +1 -0
- package/dist/shared/src/types/index.d.ts +8 -0
- package/dist/shared/src/types/index.d.ts.map +1 -0
- package/dist/shared/src/types/index.js +9 -0
- package/dist/shared/src/types/index.js.map +1 -0
- package/dist/shared/src/types/scrape.types.d.ts +151 -0
- package/dist/shared/src/types/scrape.types.d.ts.map +1 -0
- package/dist/shared/src/types/scrape.types.js +2 -0
- package/dist/shared/src/types/scrape.types.js.map +1 -0
- package/dist/shared/src/utils/date.d.ts +7 -0
- package/dist/shared/src/utils/date.d.ts.map +1 -0
- package/dist/shared/src/utils/date.js +25 -0
- package/dist/shared/src/utils/date.js.map +1 -0
- package/dist/shared/src/utils/hash.d.ts +4 -0
- package/dist/shared/src/utils/hash.d.ts.map +1 -0
- package/dist/shared/src/utils/hash.js +21 -0
- package/dist/shared/src/utils/hash.js.map +1 -0
- package/dist/shared/src/utils/index.d.ts +7 -0
- package/dist/shared/src/utils/index.d.ts.map +1 -0
- package/dist/shared/src/utils/index.js +8 -0
- package/dist/shared/src/utils/index.js.map +1 -0
- package/dist/shared/src/utils/logger.d.ts +66 -0
- package/dist/shared/src/utils/logger.d.ts.map +1 -0
- package/dist/shared/src/utils/logger.js +268 -0
- package/dist/shared/src/utils/logger.js.map +1 -0
- package/dist/shared/src/utils/retry.d.ts +11 -0
- package/dist/shared/src/utils/retry.d.ts.map +1 -0
- package/dist/shared/src/utils/retry.js +36 -0
- package/dist/shared/src/utils/retry.js.map +1 -0
- package/dist/shared/src/utils/url-validator.d.ts +37 -0
- package/dist/shared/src/utils/url-validator.d.ts.map +1 -0
- package/dist/shared/src/utils/url-validator.js +179 -0
- package/dist/shared/src/utils/url-validator.js.map +1 -0
- package/dist/shared/src/utils/url.d.ts +6 -0
- package/dist/shared/src/utils/url.d.ts.map +1 -0
- package/dist/shared/src/utils/url.js +56 -0
- package/dist/shared/src/utils/url.js.map +1 -0
- package/package.json +49 -0
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { ERROR_CODES } from "@dealcrawl/shared";
|
|
2
|
+
/**
|
|
3
|
+
* Custom error class for DealCrawl SDK
|
|
4
|
+
* Provides structured error handling with error codes
|
|
5
|
+
*/
|
|
6
|
+
export class DealCrawlError extends Error {
|
|
7
|
+
/** Error code from ERROR_CODES */
|
|
8
|
+
code;
|
|
9
|
+
/** HTTP status code from the API response */
|
|
10
|
+
statusCode;
|
|
11
|
+
/** Additional error details */
|
|
12
|
+
details;
|
|
13
|
+
/** Retry-After header value in seconds (for rate limiting) */
|
|
14
|
+
retryAfter;
|
|
15
|
+
constructor(options) {
|
|
16
|
+
super(options.message);
|
|
17
|
+
this.name = "DealCrawlError";
|
|
18
|
+
this.code = options.code;
|
|
19
|
+
this.statusCode = options.statusCode ?? 500;
|
|
20
|
+
this.details = options.details;
|
|
21
|
+
this.retryAfter = options.retryAfter;
|
|
22
|
+
// Maintains proper stack trace for where our error was thrown
|
|
23
|
+
if (Error.captureStackTrace) {
|
|
24
|
+
Error.captureStackTrace(this, DealCrawlError);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Check if the error is retryable
|
|
29
|
+
* Rate limits and transient errors are retryable
|
|
30
|
+
*/
|
|
31
|
+
isRetryable() {
|
|
32
|
+
const retryableCodes = [
|
|
33
|
+
ERROR_CODES.RATE_LIMIT_EXCEEDED,
|
|
34
|
+
ERROR_CODES.SERVICE_UNAVAILABLE,
|
|
35
|
+
ERROR_CODES.REDIS_ERROR,
|
|
36
|
+
ERROR_CODES.DATABASE_ERROR,
|
|
37
|
+
];
|
|
38
|
+
return retryableCodes.includes(this.code);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Check if the error is due to authentication issues
|
|
42
|
+
*/
|
|
43
|
+
isAuthError() {
|
|
44
|
+
const authCodes = [
|
|
45
|
+
ERROR_CODES.INVALID_API_KEY,
|
|
46
|
+
ERROR_CODES.MISSING_API_KEY,
|
|
47
|
+
ERROR_CODES.API_KEY_EXPIRED,
|
|
48
|
+
ERROR_CODES.ACCOUNT_SUSPENDED,
|
|
49
|
+
];
|
|
50
|
+
return authCodes.includes(this.code);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Check if the error is due to rate limiting
|
|
54
|
+
*/
|
|
55
|
+
isRateLimited() {
|
|
56
|
+
return this.code === ERROR_CODES.RATE_LIMIT_EXCEEDED;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Check if the error is due to quota exceeded
|
|
60
|
+
*/
|
|
61
|
+
isQuotaExceeded() {
|
|
62
|
+
return this.code === ERROR_CODES.QUOTA_EXCEEDED;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Convert error to JSON-serializable object
|
|
66
|
+
*/
|
|
67
|
+
toJSON() {
|
|
68
|
+
return {
|
|
69
|
+
name: this.name,
|
|
70
|
+
code: this.code,
|
|
71
|
+
message: this.message,
|
|
72
|
+
statusCode: this.statusCode,
|
|
73
|
+
details: this.details,
|
|
74
|
+
retryAfter: this.retryAfter,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Create a DealCrawlError from an API error response
|
|
79
|
+
*/
|
|
80
|
+
static fromResponse(statusCode, body) {
|
|
81
|
+
const code = body.code || mapStatusCodeToErrorCode(statusCode);
|
|
82
|
+
const message = body.message || body.error || "An error occurred";
|
|
83
|
+
return new DealCrawlError({
|
|
84
|
+
code,
|
|
85
|
+
message,
|
|
86
|
+
statusCode,
|
|
87
|
+
details: body.details,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Map HTTP status codes to error codes
|
|
93
|
+
*/
|
|
94
|
+
function mapStatusCodeToErrorCode(statusCode) {
|
|
95
|
+
switch (statusCode) {
|
|
96
|
+
case 400:
|
|
97
|
+
return ERROR_CODES.INVALID_REQUEST;
|
|
98
|
+
case 401:
|
|
99
|
+
return ERROR_CODES.INVALID_API_KEY;
|
|
100
|
+
case 403:
|
|
101
|
+
return ERROR_CODES.ACCOUNT_SUSPENDED;
|
|
102
|
+
case 404:
|
|
103
|
+
return ERROR_CODES.JOB_NOT_FOUND;
|
|
104
|
+
case 429:
|
|
105
|
+
return ERROR_CODES.RATE_LIMIT_EXCEEDED;
|
|
106
|
+
case 500:
|
|
107
|
+
return ERROR_CODES.INTERNAL_ERROR;
|
|
108
|
+
case 503:
|
|
109
|
+
return ERROR_CODES.SERVICE_UNAVAILABLE;
|
|
110
|
+
default:
|
|
111
|
+
return ERROR_CODES.INTERNAL_ERROR;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
export { ERROR_CODES };
|
|
115
|
+
//# sourceMappingURL=error.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"error.js","sourceRoot":"","sources":["../../../src/error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAkB,MAAM,mBAAmB,CAAC;AAEhE;;;GAGG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,kCAAkC;IACzB,IAAI,CAAY;IAEzB,6CAA6C;IACpC,UAAU,CAAS;IAE5B,+BAA+B;IACtB,OAAO,CAA2B;IAE3C,8DAA8D;IACrD,UAAU,CAAU;IAE7B,YAAY,OAMX;QACC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,GAAG,CAAC;QAC5C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QAErC,8DAA8D;QAC9D,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,MAAM,cAAc,GAAgB;YAClC,WAAW,CAAC,mBAAmB;YAC/B,WAAW,CAAC,mBAAmB;YAC/B,WAAW,CAAC,WAAW;YACvB,WAAW,CAAC,cAAc;SAC3B,CAAC;QACF,OAAO,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,WAAW;QACT,MAAM,SAAS,GAAgB;YAC7B,WAAW,CAAC,eAAe;YAC3B,WAAW,CAAC,eAAe;YAC3B,WAAW,CAAC,eAAe;YAC3B,WAAW,CAAC,iBAAiB;SAC9B,CAAC;QACF,OAAO,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,mBAAmB,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,cAAc,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,YAAY,CACjB,UAAkB,EAClB,IAA4F;QAE5F,MAAM,IAAI,GAAI,IAAI,CAAC,IAAkB,IAAI,wBAAwB,CAAC,UAAU,CAAC,CAAC;QAC9E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,IAAI,mBAAmB,CAAC;QAElE,OAAO,IAAI,cAAc,CAAC;YACxB,IAAI;YACJ,OAAO;YACP,UAAU;YACV,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;CACF;AAED;;GAEG;AACH,SAAS,wBAAwB,CAAC,UAAkB;IAClD,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,GAAG;YACN,OAAO,WAAW,CAAC,eAAe,CAAC;QACrC,KAAK,GAAG;YACN,OAAO,WAAW,CAAC,eAAe,CAAC;QACrC,KAAK,GAAG;YACN,OAAO,WAAW,CAAC,iBAAiB,CAAC;QACvC,KAAK,GAAG;YACN,OAAO,WAAW,CAAC,aAAa,CAAC;QACnC,KAAK,GAAG;YACN,OAAO,WAAW,CAAC,mBAAmB,CAAC;QACzC,KAAK,GAAG;YACN,OAAO,WAAW,CAAC,cAAc,CAAC;QACpC,KAAK,GAAG;YACN,OAAO,WAAW,CAAC,mBAAmB,CAAC;QACzC;YACE,OAAO,WAAW,CAAC,cAAc,CAAC;IACtC,CAAC;AACH,CAAC;AAED,OAAO,EAAE,WAAW,EAAkB,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DealCrawl SDK
|
|
3
|
+
*
|
|
4
|
+
* Official SDK for the DealCrawl web scraping and crawling API.
|
|
5
|
+
*
|
|
6
|
+
* @packageDocumentation
|
|
7
|
+
*
|
|
8
|
+
* @example Basic usage
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { DealCrawl } from "@dealcrawl/sdk";
|
|
11
|
+
*
|
|
12
|
+
* const client = new DealCrawl({ apiKey: "sk_xxx" });
|
|
13
|
+
*
|
|
14
|
+
* // Scrape a single page
|
|
15
|
+
* const job = await client.scrape.create({
|
|
16
|
+
* url: "https://example.com/product",
|
|
17
|
+
* extractDeal: true
|
|
18
|
+
* });
|
|
19
|
+
*
|
|
20
|
+
* // Wait for result
|
|
21
|
+
* const result = await client.waitForResult(job.jobId);
|
|
22
|
+
* console.log(result);
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* @example Crawl with template
|
|
26
|
+
* ```ts
|
|
27
|
+
* const job = await client.crawl.withTemplate("ecommerce", {
|
|
28
|
+
* url: "https://shop.example.com",
|
|
29
|
+
* maxPages: 100
|
|
30
|
+
* });
|
|
31
|
+
* ```
|
|
32
|
+
*
|
|
33
|
+
* @example Extract structured data
|
|
34
|
+
* ```ts
|
|
35
|
+
* const job = await client.extract.withSchema(
|
|
36
|
+
* "https://example.com/product",
|
|
37
|
+
* {
|
|
38
|
+
* type: "object",
|
|
39
|
+
* properties: {
|
|
40
|
+
* name: { type: "string" },
|
|
41
|
+
* price: { type: "number" }
|
|
42
|
+
* }
|
|
43
|
+
* }
|
|
44
|
+
* );
|
|
45
|
+
* ```
|
|
46
|
+
*/
|
|
47
|
+
export { DealCrawl } from "./client";
|
|
48
|
+
export { DealCrawlError, ERROR_CODES, type ErrorCode } from "./error";
|
|
49
|
+
export { ScrapeResource, CrawlResource, ExtractResource, DorkResource, StatusResource, DataResource, WebhooksResource, KeysResource, AccountResource, } from "./resources";
|
|
50
|
+
export type { DealCrawlConfig, RequestContext } from "./types/config";
|
|
51
|
+
export { DEFAULT_CONFIG } from "./types/config";
|
|
52
|
+
export type { ScreenshotOptions, ScrapeOptions, CrawlOptions, CrawlTemplate, CrawlTemplateId, ExtractOptions, ExtractModel, DorkOptions, GetDealsOptions, JobStatusFilter, JobTypeFilter, SortOrder, ListJobsOptions, ListDealsOptions, ExportFormat, ExportJobsOptions, ExportDealsOptions, WebhookEvent, CreateWebhookOptions, UpdateWebhookOptions, ApiKeyScope, CreateApiKeyOptions, RotateApiKeyOptions, ListApiKeysOptions, GetApiKeyStatsOptions, ProductCategory, UpdatePreferencesOptions, } from "./types/options";
|
|
53
|
+
export type { JobResponse, PaginationInfo, ScrapeJobResponse, CrawlJobResponse, CrawlAnalysisResponse, ExtractJobResponse, DorkJobResponse, DealMetrics, CheckpointInfo, JobStatusResponse, JobDealsResponse, DealSummary, ResumeJobResponse, JobMetricsResponse, CancelJobResponse, JobSummary, ListJobsResponse, ProductInfo, PricingInfo, DealItem, ListDealsResponse, DealDetails, ClientStatsResponse, WebhookItem, CreateWebhookResponse, ListWebhooksResponse, UpdateWebhookResponse, DeleteWebhookResponse, TestWebhookResponse, ApiKeyInfo, CreatedApiKey, ListKeysResponse, CreateKeyResponse, RotateKeyResponse, DeleteKeyResponse, KeyStatsResponse, UsageStats, ClientPreferences, AccountInfoResponse, AccountMetricsResponse, DealUpMetrics, DealUpMetricsResponse, CrawlRecommendation, RecommendationsResponse, PreferencesResponse, UpdatePreferencesResponse, } from "./types/responses";
|
|
54
|
+
export type { WaitOptions, WaitResult } from "./utils/polling";
|
|
55
|
+
export { waitForResult, waitForAll, waitForAny, pollUntil, } from "./utils/polling";
|
|
56
|
+
export type { JobStatus, RateLimitInfo, PaginatedResponse, ApiResponse, ApiError, ScrapeResult, CrawlResult, DorkResult, ExtractedDeal, DealScoreSummary, Signal, PriceSignal, DiscountSignal, ParsedPage, } from "@dealcrawl/shared";
|
|
57
|
+
export { DealCrawl as default } from "./client";
|
|
58
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAMH,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAMrC,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,KAAK,SAAS,EAAE,MAAM,SAAS,CAAC;AAMtE,OAAO,EACL,cAAc,EACd,aAAa,EACb,eAAe,EACf,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AAMrB,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAMhD,YAAY,EAEV,iBAAiB,EACjB,aAAa,EAEb,YAAY,EACZ,aAAa,EACb,eAAe,EAEf,cAAc,EACd,YAAY,EAEZ,WAAW,EAEX,eAAe,EAEf,eAAe,EACf,aAAa,EACb,SAAS,EACT,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,EAElB,YAAY,EACZ,oBAAoB,EACpB,oBAAoB,EAEpB,WAAW,EACX,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,qBAAqB,EAErB,eAAe,EACf,wBAAwB,GACzB,MAAM,iBAAiB,CAAC;AAMzB,YAAY,EAEV,WAAW,EACX,cAAc,EAEd,iBAAiB,EAEjB,gBAAgB,EAChB,qBAAqB,EAErB,kBAAkB,EAElB,eAAe,EAEf,WAAW,EACX,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,WAAW,EACX,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EAEjB,UAAU,EACV,gBAAgB,EAChB,WAAW,EACX,WAAW,EACX,QAAQ,EACR,iBAAiB,EACjB,WAAW,EACX,mBAAmB,EAEnB,WAAW,EACX,qBAAqB,EACrB,oBAAoB,EACpB,qBAAqB,EACrB,qBAAqB,EACrB,mBAAmB,EAEnB,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAEhB,UAAU,EACV,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,EACtB,aAAa,EACb,qBAAqB,EACrB,mBAAmB,EACnB,uBAAuB,EACvB,mBAAmB,EACnB,yBAAyB,GAC1B,MAAM,mBAAmB,CAAC;AAM3B,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EACL,aAAa,EACb,UAAU,EACV,UAAU,EACV,SAAS,GACV,MAAM,iBAAiB,CAAC;AAOzB,YAAY,EAEV,SAAS,EACT,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,QAAQ,EAER,YAAY,EACZ,WAAW,EACX,UAAU,EAEV,aAAa,EACb,gBAAgB,EAEhB,MAAM,EACN,WAAW,EACX,cAAc,EAEd,UAAU,GACX,MAAM,mBAAmB,CAAC;AAM3B,OAAO,EAAE,SAAS,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DealCrawl SDK
|
|
3
|
+
*
|
|
4
|
+
* Official SDK for the DealCrawl web scraping and crawling API.
|
|
5
|
+
*
|
|
6
|
+
* @packageDocumentation
|
|
7
|
+
*
|
|
8
|
+
* @example Basic usage
|
|
9
|
+
* ```ts
|
|
10
|
+
* import { DealCrawl } from "@dealcrawl/sdk";
|
|
11
|
+
*
|
|
12
|
+
* const client = new DealCrawl({ apiKey: "sk_xxx" });
|
|
13
|
+
*
|
|
14
|
+
* // Scrape a single page
|
|
15
|
+
* const job = await client.scrape.create({
|
|
16
|
+
* url: "https://example.com/product",
|
|
17
|
+
* extractDeal: true
|
|
18
|
+
* });
|
|
19
|
+
*
|
|
20
|
+
* // Wait for result
|
|
21
|
+
* const result = await client.waitForResult(job.jobId);
|
|
22
|
+
* console.log(result);
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* @example Crawl with template
|
|
26
|
+
* ```ts
|
|
27
|
+
* const job = await client.crawl.withTemplate("ecommerce", {
|
|
28
|
+
* url: "https://shop.example.com",
|
|
29
|
+
* maxPages: 100
|
|
30
|
+
* });
|
|
31
|
+
* ```
|
|
32
|
+
*
|
|
33
|
+
* @example Extract structured data
|
|
34
|
+
* ```ts
|
|
35
|
+
* const job = await client.extract.withSchema(
|
|
36
|
+
* "https://example.com/product",
|
|
37
|
+
* {
|
|
38
|
+
* type: "object",
|
|
39
|
+
* properties: {
|
|
40
|
+
* name: { type: "string" },
|
|
41
|
+
* price: { type: "number" }
|
|
42
|
+
* }
|
|
43
|
+
* }
|
|
44
|
+
* );
|
|
45
|
+
* ```
|
|
46
|
+
*/
|
|
47
|
+
// ============================================
|
|
48
|
+
// MAIN CLIENT
|
|
49
|
+
// ============================================
|
|
50
|
+
export { DealCrawl } from "./client";
|
|
51
|
+
// ============================================
|
|
52
|
+
// ERROR HANDLING
|
|
53
|
+
// ============================================
|
|
54
|
+
export { DealCrawlError, ERROR_CODES } from "./error";
|
|
55
|
+
// ============================================
|
|
56
|
+
// RESOURCE CLASSES
|
|
57
|
+
// ============================================
|
|
58
|
+
export { ScrapeResource, CrawlResource, ExtractResource, DorkResource, StatusResource, DataResource, WebhooksResource, KeysResource, AccountResource, } from "./resources";
|
|
59
|
+
export { DEFAULT_CONFIG } from "./types/config";
|
|
60
|
+
export { waitForResult, waitForAll, waitForAny, pollUntil, } from "./utils/polling";
|
|
61
|
+
// ============================================
|
|
62
|
+
// DEFAULT EXPORT
|
|
63
|
+
// ============================================
|
|
64
|
+
export { DealCrawl as default } from "./client";
|
|
65
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,+CAA+C;AAC/C,cAAc;AACd,+CAA+C;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C,OAAO,EAAE,cAAc,EAAE,WAAW,EAAkB,MAAM,SAAS,CAAC;AAEtE,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C,OAAO,EACL,cAAc,EACd,aAAa,EACb,eAAe,EACf,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACZ,eAAe,GAChB,MAAM,aAAa,CAAC;AAOrB,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAiHhD,OAAO,EACL,aAAa,EACb,UAAU,EACV,UAAU,EACV,SAAS,GACV,MAAM,iBAAiB,CAAC;AA6BzB,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C,OAAO,EAAE,SAAS,IAAI,OAAO,EAAE,MAAM,UAAU,CAAC"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Account Resource
|
|
3
|
+
* Handles account information and preferences
|
|
4
|
+
*/
|
|
5
|
+
import type { RequestContext } from "../types/config";
|
|
6
|
+
import type { UpdatePreferencesOptions } from "../types/options";
|
|
7
|
+
import type { AccountInfoResponse, AccountMetricsResponse, DealUpMetricsResponse, RecommendationsResponse, PreferencesResponse, UpdatePreferencesResponse } from "../types/responses";
|
|
8
|
+
/**
|
|
9
|
+
* Account resource class
|
|
10
|
+
* Provides methods for account information and preferences
|
|
11
|
+
*/
|
|
12
|
+
export declare class AccountResource {
|
|
13
|
+
private ctx;
|
|
14
|
+
constructor(ctx: RequestContext);
|
|
15
|
+
/**
|
|
16
|
+
* Get current account information
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```ts
|
|
20
|
+
* const account = await client.account.get();
|
|
21
|
+
* console.log(account.name);
|
|
22
|
+
* console.log(account.tier);
|
|
23
|
+
* console.log(account.usage);
|
|
24
|
+
* ```
|
|
25
|
+
*/
|
|
26
|
+
get(): Promise<AccountInfoResponse>;
|
|
27
|
+
/**
|
|
28
|
+
* Get detailed account metrics
|
|
29
|
+
*
|
|
30
|
+
* @example
|
|
31
|
+
* ```ts
|
|
32
|
+
* const metrics = await client.account.getMetrics();
|
|
33
|
+
* console.log(metrics.dealMetrics.totalDeals);
|
|
34
|
+
* console.log(metrics.categoryPerformance);
|
|
35
|
+
* ```
|
|
36
|
+
*/
|
|
37
|
+
getMetrics(): Promise<AccountMetricsResponse>;
|
|
38
|
+
/**
|
|
39
|
+
* Get DealUp sync metrics (pro/enterprise only)
|
|
40
|
+
*
|
|
41
|
+
* @example
|
|
42
|
+
* ```ts
|
|
43
|
+
* const dealup = await client.account.getDealUpMetrics();
|
|
44
|
+
* console.log(dealup.sync.totalSynced);
|
|
45
|
+
* console.log(dealup.sync.clicksGenerated);
|
|
46
|
+
* ```
|
|
47
|
+
*/
|
|
48
|
+
getDealUpMetrics(): Promise<DealUpMetricsResponse>;
|
|
49
|
+
/**
|
|
50
|
+
* Get crawl recommendations
|
|
51
|
+
* AI-suggested URLs to crawl based on past performance
|
|
52
|
+
*
|
|
53
|
+
* @example
|
|
54
|
+
* ```ts
|
|
55
|
+
* const recommendations = await client.account.getRecommendations();
|
|
56
|
+
* recommendations.recommendations.forEach(r => {
|
|
57
|
+
* console.log(r.url, r.reason, r.estimatedDeals);
|
|
58
|
+
* });
|
|
59
|
+
* ```
|
|
60
|
+
*/
|
|
61
|
+
getRecommendations(): Promise<RecommendationsResponse>;
|
|
62
|
+
/**
|
|
63
|
+
* Get current preferences
|
|
64
|
+
*
|
|
65
|
+
* @example
|
|
66
|
+
* ```ts
|
|
67
|
+
* const prefs = await client.account.getPreferences();
|
|
68
|
+
* console.log(prefs.preferences.minDealScore);
|
|
69
|
+
* console.log(prefs.preferences.autoSync);
|
|
70
|
+
* ```
|
|
71
|
+
*/
|
|
72
|
+
getPreferences(): Promise<PreferencesResponse>;
|
|
73
|
+
/**
|
|
74
|
+
* Update account preferences
|
|
75
|
+
*
|
|
76
|
+
* @example
|
|
77
|
+
* ```ts
|
|
78
|
+
* const updated = await client.account.updatePreferences({
|
|
79
|
+
* minDealScore: 70,
|
|
80
|
+
* autoSync: true,
|
|
81
|
+
* preferredCategories: ["software", "courses"]
|
|
82
|
+
* });
|
|
83
|
+
* ```
|
|
84
|
+
*/
|
|
85
|
+
updatePreferences(options: UpdatePreferencesOptions): Promise<UpdatePreferencesResponse>;
|
|
86
|
+
/**
|
|
87
|
+
* Track a DealUp click (webhook endpoint)
|
|
88
|
+
* Used internally by DealUp to track click-through
|
|
89
|
+
*
|
|
90
|
+
* @example
|
|
91
|
+
* ```ts
|
|
92
|
+
* await client.account.trackClick("deal_abc123", "homepage");
|
|
93
|
+
* ```
|
|
94
|
+
*/
|
|
95
|
+
trackClick(dealId: string, source?: string): Promise<{
|
|
96
|
+
success: boolean;
|
|
97
|
+
}>;
|
|
98
|
+
/**
|
|
99
|
+
* Get remaining quota for a resource
|
|
100
|
+
*
|
|
101
|
+
* @example
|
|
102
|
+
* ```ts
|
|
103
|
+
* const remaining = await client.account.getRemainingQuota("scrapes");
|
|
104
|
+
* console.log(`${remaining} scrapes left this period`);
|
|
105
|
+
* ```
|
|
106
|
+
*/
|
|
107
|
+
getRemainingQuota(resource: "scrapes" | "crawls" | "dorks"): Promise<number>;
|
|
108
|
+
/**
|
|
109
|
+
* Check if account has quota for a resource
|
|
110
|
+
*
|
|
111
|
+
* @example
|
|
112
|
+
* ```ts
|
|
113
|
+
* if (await client.account.hasQuota("crawls", 5)) {
|
|
114
|
+
* // Safe to start 5 crawls
|
|
115
|
+
* }
|
|
116
|
+
* ```
|
|
117
|
+
*/
|
|
118
|
+
hasQuota(resource: "scrapes" | "crawls" | "dorks", needed?: number): Promise<boolean>;
|
|
119
|
+
/**
|
|
120
|
+
* Get account tier
|
|
121
|
+
*
|
|
122
|
+
* @example
|
|
123
|
+
* ```ts
|
|
124
|
+
* const tier = await client.account.getTier();
|
|
125
|
+
* if (tier === "enterprise") {
|
|
126
|
+
* // Enable advanced features
|
|
127
|
+
* }
|
|
128
|
+
* ```
|
|
129
|
+
*/
|
|
130
|
+
getTier(): Promise<"free" | "pro" | "enterprise">;
|
|
131
|
+
/**
|
|
132
|
+
* Check if account is pro or enterprise tier
|
|
133
|
+
*
|
|
134
|
+
* @example
|
|
135
|
+
* ```ts
|
|
136
|
+
* if (await client.account.isPremium()) {
|
|
137
|
+
* // Show premium features
|
|
138
|
+
* }
|
|
139
|
+
* ```
|
|
140
|
+
*/
|
|
141
|
+
isPremium(): Promise<boolean>;
|
|
142
|
+
}
|
|
143
|
+
//# sourceMappingURL=account.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"account.d.ts","sourceRoot":"","sources":["../../../../src/resources/account.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAC;AACjE,OAAO,KAAK,EACV,mBAAmB,EACnB,sBAAsB,EACtB,qBAAqB,EACrB,uBAAuB,EACvB,mBAAmB,EACnB,yBAAyB,EAC1B,MAAM,oBAAoB,CAAC;AAG5B;;;GAGG;AACH,qBAAa,eAAe;IACd,OAAO,CAAC,GAAG;gBAAH,GAAG,EAAE,cAAc;IAEvC;;;;;;;;;;OAUG;IACG,GAAG,IAAI,OAAO,CAAC,mBAAmB,CAAC;IAKzC;;;;;;;;;OASG;IACG,UAAU,IAAI,OAAO,CAAC,sBAAsB,CAAC;IAQnD;;;;;;;;;OASG;IACG,gBAAgB,IAAI,OAAO,CAAC,qBAAqB,CAAC;IAQxD;;;;;;;;;;;OAWG;IACG,kBAAkB,IAAI,OAAO,CAAC,uBAAuB,CAAC;IAQ5D;;;;;;;;;OASG;IACG,cAAc,IAAI,OAAO,CAAC,mBAAmB,CAAC;IAQpD;;;;;;;;;;;OAWG;IACG,iBAAiB,CACrB,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,yBAAyB,CAAC;IAcrC;;;;;;;;OAQG;IACG,UAAU,CACd,MAAM,EAAE,MAAM,EACd,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;IAgBhC;;;;;;;;OAQG;IACG,iBAAiB,CACrB,QAAQ,EAAE,SAAS,GAAG,QAAQ,GAAG,OAAO,GACvC,OAAO,CAAC,MAAM,CAAC;IAMlB;;;;;;;;;OASG;IACG,QAAQ,CACZ,QAAQ,EAAE,SAAS,GAAG,QAAQ,GAAG,OAAO,EACxC,MAAM,GAAE,MAAU,GACjB,OAAO,CAAC,OAAO,CAAC;IAKnB;;;;;;;;;;OAUG;IACG,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,KAAK,GAAG,YAAY,CAAC;IAKvD;;;;;;;;;OASG;IACG,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;CAIpC"}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Account Resource
|
|
3
|
+
* Handles account information and preferences
|
|
4
|
+
*/
|
|
5
|
+
import { get, patch, post } from "../utils/request";
|
|
6
|
+
/**
|
|
7
|
+
* Account resource class
|
|
8
|
+
* Provides methods for account information and preferences
|
|
9
|
+
*/
|
|
10
|
+
export class AccountResource {
|
|
11
|
+
ctx;
|
|
12
|
+
constructor(ctx) {
|
|
13
|
+
this.ctx = ctx;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Get current account information
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```ts
|
|
20
|
+
* const account = await client.account.get();
|
|
21
|
+
* console.log(account.name);
|
|
22
|
+
* console.log(account.tier);
|
|
23
|
+
* console.log(account.usage);
|
|
24
|
+
* ```
|
|
25
|
+
*/
|
|
26
|
+
async get() {
|
|
27
|
+
const result = await get(this.ctx, "/v1/client");
|
|
28
|
+
return result.data;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Get detailed account metrics
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* ```ts
|
|
35
|
+
* const metrics = await client.account.getMetrics();
|
|
36
|
+
* console.log(metrics.dealMetrics.totalDeals);
|
|
37
|
+
* console.log(metrics.categoryPerformance);
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
async getMetrics() {
|
|
41
|
+
const result = await get(this.ctx, "/v1/client/metrics");
|
|
42
|
+
return result.data;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Get DealUp sync metrics (pro/enterprise only)
|
|
46
|
+
*
|
|
47
|
+
* @example
|
|
48
|
+
* ```ts
|
|
49
|
+
* const dealup = await client.account.getDealUpMetrics();
|
|
50
|
+
* console.log(dealup.sync.totalSynced);
|
|
51
|
+
* console.log(dealup.sync.clicksGenerated);
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
async getDealUpMetrics() {
|
|
55
|
+
const result = await get(this.ctx, "/v1/client/dealup");
|
|
56
|
+
return result.data;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Get crawl recommendations
|
|
60
|
+
* AI-suggested URLs to crawl based on past performance
|
|
61
|
+
*
|
|
62
|
+
* @example
|
|
63
|
+
* ```ts
|
|
64
|
+
* const recommendations = await client.account.getRecommendations();
|
|
65
|
+
* recommendations.recommendations.forEach(r => {
|
|
66
|
+
* console.log(r.url, r.reason, r.estimatedDeals);
|
|
67
|
+
* });
|
|
68
|
+
* ```
|
|
69
|
+
*/
|
|
70
|
+
async getRecommendations() {
|
|
71
|
+
const result = await get(this.ctx, "/v1/client/recommendations");
|
|
72
|
+
return result.data;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Get current preferences
|
|
76
|
+
*
|
|
77
|
+
* @example
|
|
78
|
+
* ```ts
|
|
79
|
+
* const prefs = await client.account.getPreferences();
|
|
80
|
+
* console.log(prefs.preferences.minDealScore);
|
|
81
|
+
* console.log(prefs.preferences.autoSync);
|
|
82
|
+
* ```
|
|
83
|
+
*/
|
|
84
|
+
async getPreferences() {
|
|
85
|
+
const result = await get(this.ctx, "/v1/client/preferences");
|
|
86
|
+
return result.data;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Update account preferences
|
|
90
|
+
*
|
|
91
|
+
* @example
|
|
92
|
+
* ```ts
|
|
93
|
+
* const updated = await client.account.updatePreferences({
|
|
94
|
+
* minDealScore: 70,
|
|
95
|
+
* autoSync: true,
|
|
96
|
+
* preferredCategories: ["software", "courses"]
|
|
97
|
+
* });
|
|
98
|
+
* ```
|
|
99
|
+
*/
|
|
100
|
+
async updatePreferences(options) {
|
|
101
|
+
const result = await patch(this.ctx, "/v1/client/preferences", {
|
|
102
|
+
preferredCategories: options.preferredCategories,
|
|
103
|
+
minDealScore: options.minDealScore,
|
|
104
|
+
autoSync: options.autoSync,
|
|
105
|
+
webhookEnabled: options.webhookEnabled,
|
|
106
|
+
});
|
|
107
|
+
return result.data;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Track a DealUp click (webhook endpoint)
|
|
111
|
+
* Used internally by DealUp to track click-through
|
|
112
|
+
*
|
|
113
|
+
* @example
|
|
114
|
+
* ```ts
|
|
115
|
+
* await client.account.trackClick("deal_abc123", "homepage");
|
|
116
|
+
* ```
|
|
117
|
+
*/
|
|
118
|
+
async trackClick(dealId, source) {
|
|
119
|
+
const result = await post(this.ctx, "/v1/client/dealup/click", {
|
|
120
|
+
dealId,
|
|
121
|
+
source,
|
|
122
|
+
});
|
|
123
|
+
return result.data;
|
|
124
|
+
}
|
|
125
|
+
// ============================================
|
|
126
|
+
// CONVENIENCE METHODS
|
|
127
|
+
// ============================================
|
|
128
|
+
/**
|
|
129
|
+
* Get remaining quota for a resource
|
|
130
|
+
*
|
|
131
|
+
* @example
|
|
132
|
+
* ```ts
|
|
133
|
+
* const remaining = await client.account.getRemainingQuota("scrapes");
|
|
134
|
+
* console.log(`${remaining} scrapes left this period`);
|
|
135
|
+
* ```
|
|
136
|
+
*/
|
|
137
|
+
async getRemainingQuota(resource) {
|
|
138
|
+
const account = await this.get();
|
|
139
|
+
const usage = account.usage[resource];
|
|
140
|
+
return usage.limit - usage.used;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Check if account has quota for a resource
|
|
144
|
+
*
|
|
145
|
+
* @example
|
|
146
|
+
* ```ts
|
|
147
|
+
* if (await client.account.hasQuota("crawls", 5)) {
|
|
148
|
+
* // Safe to start 5 crawls
|
|
149
|
+
* }
|
|
150
|
+
* ```
|
|
151
|
+
*/
|
|
152
|
+
async hasQuota(resource, needed = 1) {
|
|
153
|
+
const remaining = await this.getRemainingQuota(resource);
|
|
154
|
+
return remaining >= needed;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Get account tier
|
|
158
|
+
*
|
|
159
|
+
* @example
|
|
160
|
+
* ```ts
|
|
161
|
+
* const tier = await client.account.getTier();
|
|
162
|
+
* if (tier === "enterprise") {
|
|
163
|
+
* // Enable advanced features
|
|
164
|
+
* }
|
|
165
|
+
* ```
|
|
166
|
+
*/
|
|
167
|
+
async getTier() {
|
|
168
|
+
const account = await this.get();
|
|
169
|
+
return account.tier;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Check if account is pro or enterprise tier
|
|
173
|
+
*
|
|
174
|
+
* @example
|
|
175
|
+
* ```ts
|
|
176
|
+
* if (await client.account.isPremium()) {
|
|
177
|
+
* // Show premium features
|
|
178
|
+
* }
|
|
179
|
+
* ```
|
|
180
|
+
*/
|
|
181
|
+
async isPremium() {
|
|
182
|
+
const tier = await this.getTier();
|
|
183
|
+
return tier === "pro" || tier === "enterprise";
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
//# sourceMappingURL=account.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"account.js","sourceRoot":"","sources":["../../../../src/resources/account.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAYH,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAEpD;;;GAGG;AACH,MAAM,OAAO,eAAe;IACN;IAApB,YAAoB,GAAmB;QAAnB,QAAG,GAAH,GAAG,CAAgB;IAAG,CAAC;IAE3C;;;;;;;;;;OAUG;IACH,KAAK,CAAC,GAAG;QACP,MAAM,MAAM,GAAG,MAAM,GAAG,CAAsB,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACtE,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,MAAM,GAAG,MAAM,GAAG,CACtB,IAAI,CAAC,GAAG,EACR,oBAAoB,CACrB,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,MAAM,GAAG,MAAM,GAAG,CACtB,IAAI,CAAC,GAAG,EACR,mBAAmB,CACpB,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,kBAAkB;QACtB,MAAM,MAAM,GAAG,MAAM,GAAG,CACtB,IAAI,CAAC,GAAG,EACR,4BAA4B,CAC7B,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,cAAc;QAClB,MAAM,MAAM,GAAG,MAAM,GAAG,CACtB,IAAI,CAAC,GAAG,EACR,wBAAwB,CACzB,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,iBAAiB,CACrB,OAAiC;QAEjC,MAAM,MAAM,GAAG,MAAM,KAAK,CACxB,IAAI,CAAC,GAAG,EACR,wBAAwB,EACxB;YACE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB;YAChD,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC,CACF,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,UAAU,CACd,MAAc,EACd,MAAe;QAEf,MAAM,MAAM,GAAG,MAAM,IAAI,CACvB,IAAI,CAAC,GAAG,EACR,yBAAyB,EACzB;YACE,MAAM;YACN,MAAM;SACP,CACF,CAAC;QACF,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED,+CAA+C;IAC/C,sBAAsB;IACtB,+CAA+C;IAE/C;;;;;;;;OAQG;IACH,KAAK,CAAC,iBAAiB,CACrB,QAAwC;QAExC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACtC,OAAO,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC;IAClC,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,QAAQ,CACZ,QAAwC,EACxC,SAAiB,CAAC;QAElB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACzD,OAAO,SAAS,IAAI,MAAM,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,OAAO,OAAO,CAAC,IAAI,CAAC;IACtB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,SAAS;QACb,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,OAAO,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,YAAY,CAAC;IACjD,CAAC;CACF"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Crawl Resource
|
|
3
|
+
* Handles website crawling operations
|
|
4
|
+
*/
|
|
5
|
+
import type { RequestContext } from "../types/config";
|
|
6
|
+
import type { CrawlOptions, CrawlTemplate, CrawlTemplateId } from "../types/options";
|
|
7
|
+
import type { CrawlJobResponse, CrawlAnalysisResponse } from "../types/responses";
|
|
8
|
+
/**
|
|
9
|
+
* Crawl resource class
|
|
10
|
+
* Provides methods for website crawling
|
|
11
|
+
*/
|
|
12
|
+
export declare class CrawlResource {
|
|
13
|
+
private ctx;
|
|
14
|
+
constructor(ctx: RequestContext);
|
|
15
|
+
/**
|
|
16
|
+
* Create a new crawl job
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```ts
|
|
20
|
+
* const job = await client.crawl.create({
|
|
21
|
+
* url: "https://example.com",
|
|
22
|
+
* maxDepth: 3,
|
|
23
|
+
* maxPages: 100,
|
|
24
|
+
* extractDeal: true
|
|
25
|
+
* });
|
|
26
|
+
* console.log(job.jobId);
|
|
27
|
+
* ```
|
|
28
|
+
*/
|
|
29
|
+
create(options: CrawlOptions): Promise<CrawlJobResponse>;
|
|
30
|
+
/**
|
|
31
|
+
* Create a crawl job using a template
|
|
32
|
+
*
|
|
33
|
+
* @example
|
|
34
|
+
* ```ts
|
|
35
|
+
* const job = await client.crawl.withTemplate("ecommerce", {
|
|
36
|
+
* url: "https://shop.example.com",
|
|
37
|
+
* maxPages: 200 // Override template default
|
|
38
|
+
* });
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
withTemplate(templateId: CrawlTemplateId, options: CrawlOptions): Promise<CrawlJobResponse>;
|
|
42
|
+
/**
|
|
43
|
+
* List available crawl templates
|
|
44
|
+
*
|
|
45
|
+
* @example
|
|
46
|
+
* ```ts
|
|
47
|
+
* const templates = client.crawl.listTemplates();
|
|
48
|
+
* templates.forEach(t => console.log(t.name, t.description));
|
|
49
|
+
* ```
|
|
50
|
+
*/
|
|
51
|
+
listTemplates(): CrawlTemplate[];
|
|
52
|
+
/**
|
|
53
|
+
* Get a specific template by ID
|
|
54
|
+
*
|
|
55
|
+
* @example
|
|
56
|
+
* ```ts
|
|
57
|
+
* const template = client.crawl.getTemplate("ecommerce");
|
|
58
|
+
* console.log(template.defaultOptions);
|
|
59
|
+
* ```
|
|
60
|
+
*/
|
|
61
|
+
getTemplate(templateId: CrawlTemplateId): CrawlTemplate | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* Analyze a URL before crawling
|
|
64
|
+
* Returns recommended settings based on site structure
|
|
65
|
+
*
|
|
66
|
+
* @example
|
|
67
|
+
* ```ts
|
|
68
|
+
* const analysis = await client.crawl.analyze("https://shop.example.com");
|
|
69
|
+
* console.log(analysis.recommendedTemplate);
|
|
70
|
+
* console.log(analysis.estimatedPages);
|
|
71
|
+
* ```
|
|
72
|
+
*/
|
|
73
|
+
analyze(url: string): Promise<CrawlAnalysisResponse>;
|
|
74
|
+
/**
|
|
75
|
+
* Crawl a URL with deal extraction enabled
|
|
76
|
+
* Convenience method for e-commerce crawling
|
|
77
|
+
*
|
|
78
|
+
* @example
|
|
79
|
+
* ```ts
|
|
80
|
+
* const job = await client.crawl.forDeals("https://shop.example.com", {
|
|
81
|
+
* minDealScore: 70
|
|
82
|
+
* });
|
|
83
|
+
* ```
|
|
84
|
+
*/
|
|
85
|
+
forDeals(url: string, options?: Omit<CrawlOptions, "url" | "extractDeal" | "prioritizeDealPages">): Promise<CrawlJobResponse>;
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=crawl.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../../../src/resources/crawl.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EACV,YAAY,EACZ,aAAa,EACb,eAAe,EAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EACV,gBAAgB,EAChB,qBAAqB,EACtB,MAAM,oBAAoB,CAAC;AAiF5B;;;GAGG;AACH,qBAAa,aAAa;IACZ,OAAO,CAAC,GAAG;gBAAH,GAAG,EAAE,cAAc;IAEvC;;;;;;;;;;;;;OAaG;IACG,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAsB9D;;;;;;;;;;OAUG;IACG,YAAY,CAChB,UAAU,EAAE,eAAe,EAC3B,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,gBAAgB,CAAC;IAQ5B;;;;;;;;OAQG;IACH,aAAa,IAAI,aAAa,EAAE;IAIhC;;;;;;;;OAQG;IACH,WAAW,CAAC,UAAU,EAAE,eAAe,GAAG,aAAa,GAAG,SAAS;IAInE;;;;;;;;;;OAUG;IACG,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAW1D;;;;;;;;;;OAUG;IACG,QAAQ,CACZ,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,GAAG,aAAa,GAAG,qBAAqB,CAAC,GAC1E,OAAO,CAAC,gBAAgB,CAAC;CAQ7B"}
|