@dealcrawl/sdk 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +540 -0
- package/dist/sdk/src/client.d.ts +285 -0
- package/dist/sdk/src/client.d.ts.map +1 -0
- package/dist/sdk/src/client.js +336 -0
- package/dist/sdk/src/client.js.map +1 -0
- package/dist/sdk/src/error.d.ts +54 -0
- package/dist/sdk/src/error.d.ts.map +1 -0
- package/dist/sdk/src/error.js +115 -0
- package/dist/sdk/src/error.js.map +1 -0
- package/dist/sdk/src/index.d.ts +58 -0
- package/dist/sdk/src/index.d.ts.map +1 -0
- package/dist/sdk/src/index.js +65 -0
- package/dist/sdk/src/index.js.map +1 -0
- package/dist/sdk/src/resources/account.d.ts +143 -0
- package/dist/sdk/src/resources/account.d.ts.map +1 -0
- package/dist/sdk/src/resources/account.js +186 -0
- package/dist/sdk/src/resources/account.js.map +1 -0
- package/dist/sdk/src/resources/crawl.d.ts +87 -0
- package/dist/sdk/src/resources/crawl.d.ts.map +1 -0
- package/dist/sdk/src/resources/crawl.js +205 -0
- package/dist/sdk/src/resources/crawl.js.map +1 -0
- package/dist/sdk/src/resources/data.d.ts +157 -0
- package/dist/sdk/src/resources/data.d.ts.map +1 -0
- package/dist/sdk/src/resources/data.js +239 -0
- package/dist/sdk/src/resources/data.js.map +1 -0
- package/dist/sdk/src/resources/dork.d.ts +104 -0
- package/dist/sdk/src/resources/dork.d.ts.map +1 -0
- package/dist/sdk/src/resources/dork.js +163 -0
- package/dist/sdk/src/resources/dork.js.map +1 -0
- package/dist/sdk/src/resources/extract.d.ts +105 -0
- package/dist/sdk/src/resources/extract.d.ts.map +1 -0
- package/dist/sdk/src/resources/extract.js +246 -0
- package/dist/sdk/src/resources/extract.js.map +1 -0
- package/dist/sdk/src/resources/index.d.ts +14 -0
- package/dist/sdk/src/resources/index.d.ts.map +1 -0
- package/dist/sdk/src/resources/index.js +14 -0
- package/dist/sdk/src/resources/index.js.map +1 -0
- package/dist/sdk/src/resources/keys.d.ts +124 -0
- package/dist/sdk/src/resources/keys.d.ts.map +1 -0
- package/dist/sdk/src/resources/keys.js +168 -0
- package/dist/sdk/src/resources/keys.js.map +1 -0
- package/dist/sdk/src/resources/scrape.d.ts +53 -0
- package/dist/sdk/src/resources/scrape.d.ts.map +1 -0
- package/dist/sdk/src/resources/scrape.js +85 -0
- package/dist/sdk/src/resources/scrape.js.map +1 -0
- package/dist/sdk/src/resources/status.d.ts +100 -0
- package/dist/sdk/src/resources/status.d.ts.map +1 -0
- package/dist/sdk/src/resources/status.js +133 -0
- package/dist/sdk/src/resources/status.js.map +1 -0
- package/dist/sdk/src/resources/webhooks.d.ts +126 -0
- package/dist/sdk/src/resources/webhooks.d.ts.map +1 -0
- package/dist/sdk/src/resources/webhooks.js +167 -0
- package/dist/sdk/src/resources/webhooks.js.map +1 -0
- package/dist/sdk/src/types/config.d.ts +45 -0
- package/dist/sdk/src/types/config.d.ts.map +1 -0
- package/dist/sdk/src/types/config.js +10 -0
- package/dist/sdk/src/types/config.js.map +1 -0
- package/dist/sdk/src/types/index.d.ts +8 -0
- package/dist/sdk/src/types/index.d.ts.map +1 -0
- package/dist/sdk/src/types/index.js +8 -0
- package/dist/sdk/src/types/index.js.map +1 -0
- package/dist/sdk/src/types/options.d.ts +286 -0
- package/dist/sdk/src/types/options.d.ts.map +1 -0
- package/dist/sdk/src/types/options.js +6 -0
- package/dist/sdk/src/types/options.js.map +1 -0
- package/dist/sdk/src/types/responses.d.ts +385 -0
- package/dist/sdk/src/types/responses.d.ts.map +1 -0
- package/dist/sdk/src/types/responses.js +6 -0
- package/dist/sdk/src/types/responses.js.map +1 -0
- package/dist/sdk/src/utils/polling.d.ts +57 -0
- package/dist/sdk/src/utils/polling.d.ts.map +1 -0
- package/dist/sdk/src/utils/polling.js +110 -0
- package/dist/sdk/src/utils/polling.js.map +1 -0
- package/dist/sdk/src/utils/request.d.ts +47 -0
- package/dist/sdk/src/utils/request.d.ts.map +1 -0
- package/dist/sdk/src/utils/request.js +161 -0
- package/dist/sdk/src/utils/request.js.map +1 -0
- package/dist/shared/src/constants/errors.d.ts +26 -0
- package/dist/shared/src/constants/errors.d.ts.map +1 -0
- package/dist/shared/src/constants/errors.js +39 -0
- package/dist/shared/src/constants/errors.js.map +1 -0
- package/dist/shared/src/constants/http.d.ts +26 -0
- package/dist/shared/src/constants/http.d.ts.map +1 -0
- package/dist/shared/src/constants/http.js +26 -0
- package/dist/shared/src/constants/http.js.map +1 -0
- package/dist/shared/src/constants/index.d.ts +4 -0
- package/dist/shared/src/constants/index.d.ts.map +1 -0
- package/dist/shared/src/constants/index.js +5 -0
- package/dist/shared/src/constants/index.js.map +1 -0
- package/dist/shared/src/constants/limits.d.ts +52 -0
- package/dist/shared/src/constants/limits.d.ts.map +1 -0
- package/dist/shared/src/constants/limits.js +43 -0
- package/dist/shared/src/constants/limits.js.map +1 -0
- package/dist/shared/src/index.d.ts +5 -0
- package/dist/shared/src/index.d.ts.map +1 -0
- package/dist/shared/src/index.js +11 -0
- package/dist/shared/src/index.js.map +1 -0
- package/dist/shared/src/lib/index.d.ts +2 -0
- package/dist/shared/src/lib/index.d.ts.map +1 -0
- package/dist/shared/src/lib/index.js +2 -0
- package/dist/shared/src/lib/index.js.map +1 -0
- package/dist/shared/src/lib/redis.d.ts +14 -0
- package/dist/shared/src/lib/redis.d.ts.map +1 -0
- package/dist/shared/src/lib/redis.js +60 -0
- package/dist/shared/src/lib/redis.js.map +1 -0
- package/dist/shared/src/types/api-key.types.d.ts +94 -0
- package/dist/shared/src/types/api-key.types.d.ts.map +1 -0
- package/dist/shared/src/types/api-key.types.js +30 -0
- package/dist/shared/src/types/api-key.types.js.map +1 -0
- package/dist/shared/src/types/api.types.d.ts +38 -0
- package/dist/shared/src/types/api.types.d.ts.map +1 -0
- package/dist/shared/src/types/api.types.js +2 -0
- package/dist/shared/src/types/api.types.js.map +1 -0
- package/dist/shared/src/types/client.types.d.ts +73 -0
- package/dist/shared/src/types/client.types.d.ts.map +1 -0
- package/dist/shared/src/types/client.types.js +9 -0
- package/dist/shared/src/types/client.types.js.map +1 -0
- package/dist/shared/src/types/crawl.types.d.ts +65 -0
- package/dist/shared/src/types/crawl.types.d.ts.map +1 -0
- package/dist/shared/src/types/crawl.types.js +2 -0
- package/dist/shared/src/types/crawl.types.js.map +1 -0
- package/dist/shared/src/types/deal.types.d.ts +210 -0
- package/dist/shared/src/types/deal.types.d.ts.map +1 -0
- package/dist/shared/src/types/deal.types.js +6 -0
- package/dist/shared/src/types/deal.types.js.map +1 -0
- package/dist/shared/src/types/dork.types.d.ts +29 -0
- package/dist/shared/src/types/dork.types.d.ts.map +1 -0
- package/dist/shared/src/types/dork.types.js +2 -0
- package/dist/shared/src/types/dork.types.js.map +1 -0
- package/dist/shared/src/types/index.d.ts +8 -0
- package/dist/shared/src/types/index.d.ts.map +1 -0
- package/dist/shared/src/types/index.js +9 -0
- package/dist/shared/src/types/index.js.map +1 -0
- package/dist/shared/src/types/scrape.types.d.ts +151 -0
- package/dist/shared/src/types/scrape.types.d.ts.map +1 -0
- package/dist/shared/src/types/scrape.types.js +2 -0
- package/dist/shared/src/types/scrape.types.js.map +1 -0
- package/dist/shared/src/utils/date.d.ts +7 -0
- package/dist/shared/src/utils/date.d.ts.map +1 -0
- package/dist/shared/src/utils/date.js +25 -0
- package/dist/shared/src/utils/date.js.map +1 -0
- package/dist/shared/src/utils/hash.d.ts +4 -0
- package/dist/shared/src/utils/hash.d.ts.map +1 -0
- package/dist/shared/src/utils/hash.js +21 -0
- package/dist/shared/src/utils/hash.js.map +1 -0
- package/dist/shared/src/utils/index.d.ts +7 -0
- package/dist/shared/src/utils/index.d.ts.map +1 -0
- package/dist/shared/src/utils/index.js +8 -0
- package/dist/shared/src/utils/index.js.map +1 -0
- package/dist/shared/src/utils/logger.d.ts +66 -0
- package/dist/shared/src/utils/logger.d.ts.map +1 -0
- package/dist/shared/src/utils/logger.js +268 -0
- package/dist/shared/src/utils/logger.js.map +1 -0
- package/dist/shared/src/utils/retry.d.ts +11 -0
- package/dist/shared/src/utils/retry.d.ts.map +1 -0
- package/dist/shared/src/utils/retry.js +36 -0
- package/dist/shared/src/utils/retry.js.map +1 -0
- package/dist/shared/src/utils/url-validator.d.ts +37 -0
- package/dist/shared/src/utils/url-validator.d.ts.map +1 -0
- package/dist/shared/src/utils/url-validator.js +179 -0
- package/dist/shared/src/utils/url-validator.js.map +1 -0
- package/dist/shared/src/utils/url.d.ts +6 -0
- package/dist/shared/src/utils/url.d.ts.map +1 -0
- package/dist/shared/src/utils/url.js +56 -0
- package/dist/shared/src/utils/url.js.map +1 -0
- package/package.json +49 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { DealCrawlError, ERROR_CODES } from "../error";
|
|
2
|
+
/**
|
|
3
|
+
* Build query string from object
|
|
4
|
+
*/
|
|
5
|
+
function buildQueryString(params) {
|
|
6
|
+
const entries = Object.entries(params)
|
|
7
|
+
.filter(([, value]) => value !== undefined)
|
|
8
|
+
.map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`);
|
|
9
|
+
return entries.length > 0 ? `?${entries.join("&")}` : "";
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Parse rate limit headers from response
|
|
13
|
+
*/
|
|
14
|
+
function parseRateLimitHeaders(headers) {
|
|
15
|
+
return {
|
|
16
|
+
limit: parseInt(headers.get("X-RateLimit-Limit") || "0", 10) || 0,
|
|
17
|
+
remaining: parseInt(headers.get("X-RateLimit-Remaining") || "0", 10) || 0,
|
|
18
|
+
reset: parseInt(headers.get("X-RateLimit-Reset") || "0", 10) || 0,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Sleep helper for retries
|
|
23
|
+
*/
|
|
24
|
+
function sleep(ms) {
|
|
25
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Make an HTTP request with authentication, error handling, and retries
|
|
29
|
+
*/
|
|
30
|
+
export async function request(ctx, path, options = {}) {
|
|
31
|
+
const { method = "GET", body, query, timeout, signal } = options;
|
|
32
|
+
// Build URL with query string
|
|
33
|
+
const queryString = query ? buildQueryString(query) : "";
|
|
34
|
+
const url = `${ctx.baseUrl}${path}${queryString}`;
|
|
35
|
+
// Create abort controller for timeout
|
|
36
|
+
const controller = new AbortController();
|
|
37
|
+
const timeoutMs = timeout ?? ctx.timeout;
|
|
38
|
+
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
39
|
+
// Combine with external signal if provided
|
|
40
|
+
if (signal) {
|
|
41
|
+
signal.addEventListener("abort", () => controller.abort());
|
|
42
|
+
}
|
|
43
|
+
let lastError = null;
|
|
44
|
+
let attempt = 0;
|
|
45
|
+
while (attempt <= ctx.maxRetries) {
|
|
46
|
+
try {
|
|
47
|
+
const response = await fetch(url, {
|
|
48
|
+
method,
|
|
49
|
+
headers: {
|
|
50
|
+
"Content-Type": "application/json",
|
|
51
|
+
Authorization: `Bearer ${ctx.apiKey}`,
|
|
52
|
+
},
|
|
53
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
54
|
+
signal: controller.signal,
|
|
55
|
+
});
|
|
56
|
+
clearTimeout(timeoutId);
|
|
57
|
+
// Parse rate limit headers
|
|
58
|
+
const rateLimit = parseRateLimitHeaders(response.headers);
|
|
59
|
+
// Handle rate limiting
|
|
60
|
+
if (response.status === 429) {
|
|
61
|
+
const retryAfter = parseInt(response.headers.get("Retry-After") || "60", 10);
|
|
62
|
+
// Notify callback
|
|
63
|
+
if (ctx.onRateLimit) {
|
|
64
|
+
ctx.onRateLimit(rateLimit);
|
|
65
|
+
}
|
|
66
|
+
throw new DealCrawlError({
|
|
67
|
+
code: ERROR_CODES.RATE_LIMIT_EXCEEDED,
|
|
68
|
+
message: "Rate limit exceeded",
|
|
69
|
+
statusCode: 429,
|
|
70
|
+
retryAfter,
|
|
71
|
+
details: { rateLimit },
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
// Handle errors
|
|
75
|
+
if (!response.ok) {
|
|
76
|
+
let errorBody = {};
|
|
77
|
+
try {
|
|
78
|
+
errorBody = (await response.json());
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
// Response might not be JSON
|
|
82
|
+
}
|
|
83
|
+
throw DealCrawlError.fromResponse(response.status, errorBody);
|
|
84
|
+
}
|
|
85
|
+
// Parse response
|
|
86
|
+
let data;
|
|
87
|
+
const contentType = response.headers.get("Content-Type") || "";
|
|
88
|
+
if (contentType.includes("application/json")) {
|
|
89
|
+
data = (await response.json());
|
|
90
|
+
}
|
|
91
|
+
else if (contentType.includes("text/csv") ||
|
|
92
|
+
contentType.includes("text/plain")) {
|
|
93
|
+
data = (await response.text());
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
data = (await response.json());
|
|
97
|
+
}
|
|
98
|
+
return { data, rateLimit };
|
|
99
|
+
}
|
|
100
|
+
catch (error) {
|
|
101
|
+
clearTimeout(timeoutId);
|
|
102
|
+
// Handle abort/timeout
|
|
103
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
104
|
+
throw new DealCrawlError({
|
|
105
|
+
code: ERROR_CODES.JOB_TIMEOUT,
|
|
106
|
+
message: `Request timeout after ${timeoutMs}ms`,
|
|
107
|
+
statusCode: 408,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
// Handle DealCrawlError
|
|
111
|
+
if (error instanceof DealCrawlError) {
|
|
112
|
+
lastError = error;
|
|
113
|
+
// Retry if error is retryable and we have attempts left
|
|
114
|
+
if (error.isRetryable() && attempt < ctx.maxRetries) {
|
|
115
|
+
attempt++;
|
|
116
|
+
await sleep(ctx.retryDelay * attempt); // Exponential backoff
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
throw error;
|
|
120
|
+
}
|
|
121
|
+
// Handle network errors
|
|
122
|
+
throw new DealCrawlError({
|
|
123
|
+
code: ERROR_CODES.FETCH_FAILED,
|
|
124
|
+
message: error instanceof Error ? error.message : "Network request failed",
|
|
125
|
+
statusCode: 0,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
// Should not reach here, but throw last error if we do
|
|
130
|
+
throw (lastError ??
|
|
131
|
+
new DealCrawlError({
|
|
132
|
+
code: ERROR_CODES.INTERNAL_ERROR,
|
|
133
|
+
message: "Request failed after retries",
|
|
134
|
+
statusCode: 500,
|
|
135
|
+
}));
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Convenience method for GET requests
|
|
139
|
+
*/
|
|
140
|
+
export function get(ctx, path, query) {
|
|
141
|
+
return request(ctx, path, { method: "GET", query });
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Convenience method for POST requests
|
|
145
|
+
*/
|
|
146
|
+
export function post(ctx, path, body) {
|
|
147
|
+
return request(ctx, path, { method: "POST", body });
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Convenience method for PATCH requests
|
|
151
|
+
*/
|
|
152
|
+
export function patch(ctx, path, body) {
|
|
153
|
+
return request(ctx, path, { method: "PATCH", body });
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Convenience method for DELETE requests
|
|
157
|
+
*/
|
|
158
|
+
export function del(ctx, path, body) {
|
|
159
|
+
return request(ctx, path, { method: "DELETE", body });
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=request.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../../../../src/utils/request.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAkCvD;;GAEG;AACH,SAAS,gBAAgB,CACvB,MAA6D;IAE7D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACnC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC1C,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,kBAAkB,CAAC,GAAG,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CACpE,CAAC;IAEJ,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,OAAgB;IAC7C,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC;QACjE,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC;QACzE,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC;KAClE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,GAAmB,EACnB,IAAY,EACZ,UAA0B,EAAE;IAE5B,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAEjE,8BAA8B;IAC9B,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,OAAO,GAAG,IAAI,GAAG,WAAW,EAAE,CAAC;IAElD,sCAAsC;IACtC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAElE,2CAA2C;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,SAAS,GAA0B,IAAI,CAAC;IAC5C,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,OAAO,OAAO,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,MAAM;gBACN,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,GAAG,CAAC,MAAM,EAAE;iBACtC;gBACD,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC7C,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,2BAA2B;YAC3B,MAAM,SAAS,GAAG,qBAAqB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE1D,uBAAuB;YACvB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,QAAQ,CACzB,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,EAC3C,EAAE,CACH,CAAC;gBAEF,kBAAkB;gBAClB,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;oBACpB,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC7B,CAAC;gBAED,MAAM,IAAI,cAAc,CAAC;oBACvB,IAAI,EAAE,WAAW,CAAC,mBAAmB;oBACrC,OAAO,EAAE,qBAAqB;oBAC9B,UAAU,EAAE,GAAG;oBACf,UAAU;oBACV,OAAO,EAAE,EAAE,SAAS,EAAE;iBACvB,CAAC,CAAC;YACL,CAAC;YAED,gBAAgB;YAChB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,SAAS,GAA4B,EAAE,CAAC;gBAC5C,IAAI,CAAC;oBACH,SAAS,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA4B,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,6BAA6B;gBAC/B,CAAC;gBAED,MAAM,cAAc,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAChE,CAAC;YAED,iBAAiB;YACjB,IAAI,IAAO,CAAC;YACZ,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE/D,IAAI,WAAW,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAC7C,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAC;YACtC,CAAC;iBAAM,IACL,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAChC,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,EAClC,CAAC;gBACD,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAiB,CAAC;YACjD,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAC;YACtC,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,uBAAuB;YACvB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,cAAc,CAAC;oBACvB,IAAI,EAAE,WAAW,CAAC,WAAW;oBAC7B,OAAO,EAAE,yBAAyB,SAAS,IAAI;oBAC/C,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;YACL,CAAC;YAED,wBAAwB;YACxB,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,SAAS,GAAG,KAAK,CAAC;gBAElB,wDAAwD;gBACxD,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,OAAO,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;oBACpD,OAAO,EAAE,CAAC;oBACV,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,sBAAsB;oBAC7D,SAAS;gBACX,CAAC;gBAED,MAAM,KAAK,CAAC;YACd,CAAC;YAED,wBAAwB;YACxB,MAAM,IAAI,cAAc,CAAC;gBACvB,IAAI,EAAE,WAAW,CAAC,YAAY;gBAC9B,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;gBACnE,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,MAAM,CACJ,SAAS;QACT,IAAI,cAAc,CAAC;YACjB,IAAI,EAAE,WAAW,CAAC,cAAc;YAChC,OAAO,EAAE,8BAA8B;YACvC,UAAU,EAAE,GAAG;SAChB,CAAC,CACH,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CACjB,GAAmB,EACnB,IAAY,EACZ,KAA6D;IAE7D,OAAO,OAAO,CAAI,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AACzD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,IAAI,CAClB,GAAmB,EACnB,IAAY,EACZ,IAAc;IAEd,OAAO,OAAO,CAAI,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AACzD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,KAAK,CACnB,GAAmB,EACnB,IAAY,EACZ,IAAc;IAEd,OAAO,OAAO,CAAI,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CACjB,GAAmB,EACnB,IAAY,EACZ,IAAc;IAEd,OAAO,OAAO,CAAI,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3D,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export declare const ERROR_CODES: {
|
|
2
|
+
readonly INVALID_API_KEY: "INVALID_API_KEY";
|
|
3
|
+
readonly MISSING_API_KEY: "MISSING_API_KEY";
|
|
4
|
+
readonly API_KEY_EXPIRED: "API_KEY_EXPIRED";
|
|
5
|
+
readonly ACCOUNT_SUSPENDED: "ACCOUNT_SUSPENDED";
|
|
6
|
+
readonly RATE_LIMIT_EXCEEDED: "RATE_LIMIT_EXCEEDED";
|
|
7
|
+
readonly QUOTA_EXCEEDED: "QUOTA_EXCEEDED";
|
|
8
|
+
readonly INVALID_URL: "INVALID_URL";
|
|
9
|
+
readonly INVALID_REQUEST: "INVALID_REQUEST";
|
|
10
|
+
readonly MISSING_REQUIRED_FIELD: "MISSING_REQUIRED_FIELD";
|
|
11
|
+
readonly JOB_NOT_FOUND: "JOB_NOT_FOUND";
|
|
12
|
+
readonly JOB_FAILED: "JOB_FAILED";
|
|
13
|
+
readonly JOB_TIMEOUT: "JOB_TIMEOUT";
|
|
14
|
+
readonly FETCH_FAILED: "FETCH_FAILED";
|
|
15
|
+
readonly PARSE_FAILED: "PARSE_FAILED";
|
|
16
|
+
readonly BLOCKED_BY_ROBOTS: "BLOCKED_BY_ROBOTS";
|
|
17
|
+
readonly CAPTCHA_DETECTED: "CAPTCHA_DETECTED";
|
|
18
|
+
readonly SITE_UNREACHABLE: "SITE_UNREACHABLE";
|
|
19
|
+
readonly INTERNAL_ERROR: "INTERNAL_ERROR";
|
|
20
|
+
readonly SERVICE_UNAVAILABLE: "SERVICE_UNAVAILABLE";
|
|
21
|
+
readonly REDIS_ERROR: "REDIS_ERROR";
|
|
22
|
+
readonly DATABASE_ERROR: "DATABASE_ERROR";
|
|
23
|
+
};
|
|
24
|
+
export type ErrorCode = (typeof ERROR_CODES)[keyof typeof ERROR_CODES];
|
|
25
|
+
export declare const ERROR_MESSAGES: Record<string, string>;
|
|
26
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../../../shared/src/constants/errors.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;CAiCd,CAAC;AAEX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,OAAO,WAAW,CAAC,CAAC;AAEvE,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQjD,CAAC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export const ERROR_CODES = {
|
|
2
|
+
// Authentication
|
|
3
|
+
INVALID_API_KEY: "INVALID_API_KEY",
|
|
4
|
+
MISSING_API_KEY: "MISSING_API_KEY",
|
|
5
|
+
API_KEY_EXPIRED: "API_KEY_EXPIRED",
|
|
6
|
+
ACCOUNT_SUSPENDED: "ACCOUNT_SUSPENDED",
|
|
7
|
+
// Rate limiting
|
|
8
|
+
RATE_LIMIT_EXCEEDED: "RATE_LIMIT_EXCEEDED",
|
|
9
|
+
QUOTA_EXCEEDED: "QUOTA_EXCEEDED",
|
|
10
|
+
// Validation
|
|
11
|
+
INVALID_URL: "INVALID_URL",
|
|
12
|
+
INVALID_REQUEST: "INVALID_REQUEST",
|
|
13
|
+
MISSING_REQUIRED_FIELD: "MISSING_REQUIRED_FIELD",
|
|
14
|
+
// Job errors
|
|
15
|
+
JOB_NOT_FOUND: "JOB_NOT_FOUND",
|
|
16
|
+
JOB_FAILED: "JOB_FAILED",
|
|
17
|
+
JOB_TIMEOUT: "JOB_TIMEOUT",
|
|
18
|
+
// Scraping errors
|
|
19
|
+
FETCH_FAILED: "FETCH_FAILED",
|
|
20
|
+
PARSE_FAILED: "PARSE_FAILED",
|
|
21
|
+
BLOCKED_BY_ROBOTS: "BLOCKED_BY_ROBOTS",
|
|
22
|
+
CAPTCHA_DETECTED: "CAPTCHA_DETECTED",
|
|
23
|
+
SITE_UNREACHABLE: "SITE_UNREACHABLE",
|
|
24
|
+
// System errors
|
|
25
|
+
INTERNAL_ERROR: "INTERNAL_ERROR",
|
|
26
|
+
SERVICE_UNAVAILABLE: "SERVICE_UNAVAILABLE",
|
|
27
|
+
REDIS_ERROR: "REDIS_ERROR",
|
|
28
|
+
DATABASE_ERROR: "DATABASE_ERROR",
|
|
29
|
+
};
|
|
30
|
+
export const ERROR_MESSAGES = {
|
|
31
|
+
[ERROR_CODES.INVALID_API_KEY]: "The provided API key is invalid",
|
|
32
|
+
[ERROR_CODES.MISSING_API_KEY]: "API key is required in Authorization header",
|
|
33
|
+
[ERROR_CODES.RATE_LIMIT_EXCEEDED]: "Rate limit exceeded. Please try again later",
|
|
34
|
+
[ERROR_CODES.QUOTA_EXCEEDED]: "Monthly quota exceeded. Upgrade your plan for more",
|
|
35
|
+
[ERROR_CODES.INVALID_URL]: "The provided URL is not valid",
|
|
36
|
+
[ERROR_CODES.JOB_NOT_FOUND]: "Job not found",
|
|
37
|
+
[ERROR_CODES.INTERNAL_ERROR]: "An internal error occurred",
|
|
38
|
+
};
|
|
39
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../../../../shared/src/constants/errors.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,iBAAiB;IACjB,eAAe,EAAE,iBAAiB;IAClC,eAAe,EAAE,iBAAiB;IAClC,eAAe,EAAE,iBAAiB;IAClC,iBAAiB,EAAE,mBAAmB;IAEtC,gBAAgB;IAChB,mBAAmB,EAAE,qBAAqB;IAC1C,cAAc,EAAE,gBAAgB;IAEhC,aAAa;IACb,WAAW,EAAE,aAAa;IAC1B,eAAe,EAAE,iBAAiB;IAClC,sBAAsB,EAAE,wBAAwB;IAEhD,aAAa;IACb,aAAa,EAAE,eAAe;IAC9B,UAAU,EAAE,YAAY;IACxB,WAAW,EAAE,aAAa;IAE1B,kBAAkB;IAClB,YAAY,EAAE,cAAc;IAC5B,YAAY,EAAE,cAAc;IAC5B,iBAAiB,EAAE,mBAAmB;IACtC,gBAAgB,EAAE,kBAAkB;IACpC,gBAAgB,EAAE,kBAAkB;IAEpC,gBAAgB;IAChB,cAAc,EAAE,gBAAgB;IAChC,mBAAmB,EAAE,qBAAqB;IAC1C,WAAW,EAAE,aAAa;IAC1B,cAAc,EAAE,gBAAgB;CACxB,CAAC;AAIX,MAAM,CAAC,MAAM,cAAc,GAA2B;IACpD,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,iCAAiC;IAChE,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,6CAA6C;IAC5E,CAAC,WAAW,CAAC,mBAAmB,CAAC,EAAE,6CAA6C;IAChF,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE,oDAAoD;IAClF,CAAC,WAAW,CAAC,WAAW,CAAC,EAAE,+BAA+B;IAC1D,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,eAAe;IAC5C,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE,4BAA4B;CAC3D,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export declare const HTTP_STATUS: {
|
|
2
|
+
readonly OK: 200;
|
|
3
|
+
readonly CREATED: 201;
|
|
4
|
+
readonly ACCEPTED: 202;
|
|
5
|
+
readonly NO_CONTENT: 204;
|
|
6
|
+
readonly BAD_REQUEST: 400;
|
|
7
|
+
readonly UNAUTHORIZED: 401;
|
|
8
|
+
readonly FORBIDDEN: 403;
|
|
9
|
+
readonly NOT_FOUND: 404;
|
|
10
|
+
readonly CONFLICT: 409;
|
|
11
|
+
readonly UNPROCESSABLE_ENTITY: 422;
|
|
12
|
+
readonly TOO_MANY_REQUESTS: 429;
|
|
13
|
+
readonly INTERNAL_SERVER_ERROR: 500;
|
|
14
|
+
readonly BAD_GATEWAY: 502;
|
|
15
|
+
readonly SERVICE_UNAVAILABLE: 503;
|
|
16
|
+
};
|
|
17
|
+
export declare const DEFAULT_HEADERS: {
|
|
18
|
+
readonly "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
|
|
19
|
+
readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
20
|
+
readonly "Accept-Language": "en-US,en;q=0.9";
|
|
21
|
+
readonly "Accept-Encoding": "gzip, deflate, br";
|
|
22
|
+
readonly Connection: "keep-alive";
|
|
23
|
+
};
|
|
24
|
+
export declare const API_VERSION = "v1";
|
|
25
|
+
export declare const API_PREFIX = "/api/v1";
|
|
26
|
+
//# sourceMappingURL=http.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../../../../shared/src/constants/http.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;CAed,CAAC;AAEX,eAAO,MAAM,eAAe;;;;;;CAOlB,CAAC;AAEX,eAAO,MAAM,WAAW,OAAO,CAAC;AAChC,eAAO,MAAM,UAAU,YAAwB,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export const HTTP_STATUS = {
|
|
2
|
+
OK: 200,
|
|
3
|
+
CREATED: 201,
|
|
4
|
+
ACCEPTED: 202,
|
|
5
|
+
NO_CONTENT: 204,
|
|
6
|
+
BAD_REQUEST: 400,
|
|
7
|
+
UNAUTHORIZED: 401,
|
|
8
|
+
FORBIDDEN: 403,
|
|
9
|
+
NOT_FOUND: 404,
|
|
10
|
+
CONFLICT: 409,
|
|
11
|
+
UNPROCESSABLE_ENTITY: 422,
|
|
12
|
+
TOO_MANY_REQUESTS: 429,
|
|
13
|
+
INTERNAL_SERVER_ERROR: 500,
|
|
14
|
+
BAD_GATEWAY: 502,
|
|
15
|
+
SERVICE_UNAVAILABLE: 503,
|
|
16
|
+
};
|
|
17
|
+
export const DEFAULT_HEADERS = {
|
|
18
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
19
|
+
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
20
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
21
|
+
"Accept-Encoding": "gzip, deflate, br",
|
|
22
|
+
Connection: "keep-alive",
|
|
23
|
+
};
|
|
24
|
+
export const API_VERSION = "v1";
|
|
25
|
+
export const API_PREFIX = `/api/${API_VERSION}`;
|
|
26
|
+
//# sourceMappingURL=http.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../../../../../shared/src/constants/http.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,EAAE,EAAE,GAAG;IACP,OAAO,EAAE,GAAG;IACZ,QAAQ,EAAE,GAAG;IACb,UAAU,EAAE,GAAG;IACf,WAAW,EAAE,GAAG;IAChB,YAAY,EAAE,GAAG;IACjB,SAAS,EAAE,GAAG;IACd,SAAS,EAAE,GAAG;IACd,QAAQ,EAAE,GAAG;IACb,oBAAoB,EAAE,GAAG;IACzB,iBAAiB,EAAE,GAAG;IACtB,qBAAqB,EAAE,GAAG;IAC1B,WAAW,EAAE,GAAG;IAChB,mBAAmB,EAAE,GAAG;CAChB,CAAC;AAEX,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,YAAY,EACV,iHAAiH;IACnH,MAAM,EAAE,iEAAiE;IACzE,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC;AAChC,MAAM,CAAC,MAAM,UAAU,GAAG,QAAQ,WAAW,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../shared/src/constants/index.ts"],"names":[],"mappings":"AACA,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../shared/src/constants/index.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
export declare const CRAWL_LIMITS: {
|
|
2
|
+
readonly MAX_DEPTH: 5;
|
|
3
|
+
readonly MAX_PAGES: 1000;
|
|
4
|
+
readonly DEFAULT_DEPTH: 3;
|
|
5
|
+
readonly DEFAULT_PAGES: 100;
|
|
6
|
+
readonly DEFAULT_DELAY_MS: 1000;
|
|
7
|
+
readonly MAX_DELAY_MS: 10000;
|
|
8
|
+
readonly MIN_DELAY_MS: 100;
|
|
9
|
+
};
|
|
10
|
+
export declare const SCRAPE_LIMITS: {
|
|
11
|
+
readonly MAX_CONTENT_LENGTH: 10000000;
|
|
12
|
+
readonly DEFAULT_TIMEOUT_MS: 30000;
|
|
13
|
+
readonly MAX_TIMEOUT_MS: 120000;
|
|
14
|
+
readonly MAX_RETRIES: 3;
|
|
15
|
+
};
|
|
16
|
+
export declare const DORK_LIMITS: {
|
|
17
|
+
readonly MAX_RESULTS: 100;
|
|
18
|
+
readonly DEFAULT_RESULTS: 10;
|
|
19
|
+
readonly MAX_QUERY_LENGTH: 500;
|
|
20
|
+
};
|
|
21
|
+
export declare const QUOTA_LIMITS: {
|
|
22
|
+
readonly free: {
|
|
23
|
+
readonly crawls: 100;
|
|
24
|
+
readonly scrapes: 1000;
|
|
25
|
+
readonly dorks: 50;
|
|
26
|
+
};
|
|
27
|
+
readonly pro: {
|
|
28
|
+
readonly crawls: 1000;
|
|
29
|
+
readonly scrapes: 10000;
|
|
30
|
+
readonly dorks: 500;
|
|
31
|
+
};
|
|
32
|
+
readonly enterprise: {
|
|
33
|
+
readonly crawls: 999999;
|
|
34
|
+
readonly scrapes: 999999;
|
|
35
|
+
readonly dorks: 999999;
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
export declare const RATE_LIMITS: {
|
|
39
|
+
readonly free: {
|
|
40
|
+
readonly requests: 100;
|
|
41
|
+
readonly windowMs: 3600000;
|
|
42
|
+
};
|
|
43
|
+
readonly pro: {
|
|
44
|
+
readonly requests: 1000;
|
|
45
|
+
readonly windowMs: 3600000;
|
|
46
|
+
};
|
|
47
|
+
readonly enterprise: {
|
|
48
|
+
readonly requests: 10000;
|
|
49
|
+
readonly windowMs: 3600000;
|
|
50
|
+
};
|
|
51
|
+
};
|
|
52
|
+
//# sourceMappingURL=limits.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"limits.d.ts","sourceRoot":"","sources":["../../../../../shared/src/constants/limits.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY;;;;;;;;CAQf,CAAC;AAEX,eAAO,MAAM,aAAa;;;;;CAKhB,CAAC;AAEX,eAAO,MAAM,WAAW;;;;CAId,CAAC;AAEX,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;CAgBf,CAAC;AAEX,eAAO,MAAM,WAAW;;;;;;;;;;;;;CAId,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export const CRAWL_LIMITS = {
|
|
2
|
+
MAX_DEPTH: 5,
|
|
3
|
+
MAX_PAGES: 1000,
|
|
4
|
+
DEFAULT_DEPTH: 3,
|
|
5
|
+
DEFAULT_PAGES: 100,
|
|
6
|
+
DEFAULT_DELAY_MS: 1000,
|
|
7
|
+
MAX_DELAY_MS: 10000,
|
|
8
|
+
MIN_DELAY_MS: 100,
|
|
9
|
+
};
|
|
10
|
+
export const SCRAPE_LIMITS = {
|
|
11
|
+
MAX_CONTENT_LENGTH: 10_000_000, // 10MB
|
|
12
|
+
DEFAULT_TIMEOUT_MS: 30_000,
|
|
13
|
+
MAX_TIMEOUT_MS: 120_000,
|
|
14
|
+
MAX_RETRIES: 3,
|
|
15
|
+
};
|
|
16
|
+
export const DORK_LIMITS = {
|
|
17
|
+
MAX_RESULTS: 100,
|
|
18
|
+
DEFAULT_RESULTS: 10,
|
|
19
|
+
MAX_QUERY_LENGTH: 500,
|
|
20
|
+
};
|
|
21
|
+
export const QUOTA_LIMITS = {
|
|
22
|
+
free: {
|
|
23
|
+
crawls: 100,
|
|
24
|
+
scrapes: 1000,
|
|
25
|
+
dorks: 50,
|
|
26
|
+
},
|
|
27
|
+
pro: {
|
|
28
|
+
crawls: 1000,
|
|
29
|
+
scrapes: 10000,
|
|
30
|
+
dorks: 500,
|
|
31
|
+
},
|
|
32
|
+
enterprise: {
|
|
33
|
+
crawls: 999999,
|
|
34
|
+
scrapes: 999999,
|
|
35
|
+
dorks: 999999,
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
export const RATE_LIMITS = {
|
|
39
|
+
free: { requests: 100, windowMs: 3600000 },
|
|
40
|
+
pro: { requests: 1000, windowMs: 3600000 },
|
|
41
|
+
enterprise: { requests: 10000, windowMs: 3600000 },
|
|
42
|
+
};
|
|
43
|
+
//# sourceMappingURL=limits.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"limits.js","sourceRoot":"","sources":["../../../../../shared/src/constants/limits.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,IAAI;IACf,aAAa,EAAE,CAAC;IAChB,aAAa,EAAE,GAAG;IAClB,gBAAgB,EAAE,IAAI;IACtB,YAAY,EAAE,KAAK;IACnB,YAAY,EAAE,GAAG;CACT,CAAC;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,kBAAkB,EAAE,UAAU,EAAE,OAAO;IACvC,kBAAkB,EAAE,MAAM;IAC1B,cAAc,EAAE,OAAO;IACvB,WAAW,EAAE,CAAC;CACN,CAAC;AAEX,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,WAAW,EAAE,GAAG;IAChB,eAAe,EAAE,EAAE;IACnB,gBAAgB,EAAE,GAAG;CACb,CAAC;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE;QACJ,MAAM,EAAE,GAAG;QACX,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,EAAE;KACV;IACD,GAAG,EAAE;QACH,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,GAAG;KACX;IACD,UAAU,EAAE;QACV,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,MAAM;KACd;CACO,CAAC;AAEX,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,IAAI,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE;IAC1C,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IAC1C,UAAU,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE;CAC1C,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../shared/src/index.ts"],"names":[],"mappings":"AAIA,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,SAAS,CAAC;AAGxB,cAAc,OAAO,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// @dealcrawl/shared - Shared types, constants, and utilities
|
|
2
|
+
// ==========================================================
|
|
3
|
+
// Types
|
|
4
|
+
export * from "./types";
|
|
5
|
+
// Constants
|
|
6
|
+
export * from "./constants";
|
|
7
|
+
// Utilities
|
|
8
|
+
export * from "./utils";
|
|
9
|
+
// Lib (Redis, etc.)
|
|
10
|
+
export * from "./lib";
|
|
11
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../shared/src/index.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,6DAA6D;AAE7D,QAAQ;AACR,cAAc,SAAS,CAAC;AAExB,YAAY;AACZ,cAAc,aAAa,CAAC;AAE5B,YAAY;AACZ,cAAc,SAAS,CAAC;AAExB,oBAAoB;AACpB,cAAc,OAAO,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../shared/src/lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../shared/src/lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import Redis from "ioredis";
|
|
2
|
+
/**
|
|
3
|
+
* Shared Redis connection configuration
|
|
4
|
+
* Used by both API and Worker packages
|
|
5
|
+
*/
|
|
6
|
+
export declare function createRedisConnection(): Redis;
|
|
7
|
+
export declare function getRedisConnection(): Redis;
|
|
8
|
+
/**
|
|
9
|
+
* Close the singleton Redis connection
|
|
10
|
+
* Should be called during graceful shutdown
|
|
11
|
+
*/
|
|
12
|
+
export declare function closeRedisConnection(): Promise<void>;
|
|
13
|
+
export type { Redis };
|
|
14
|
+
//# sourceMappingURL=redis.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.d.ts","sourceRoot":"","sources":["../../../../../shared/src/lib/redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,SAAS,CAAC;AAE5B;;;GAGG;AACH,wBAAgB,qBAAqB,IAAI,KAAK,CA0C7C;AAQD,wBAAgB,kBAAkB,IAAI,KAAK,CAK1C;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1D;AAGD,YAAY,EAAE,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import Redis from "ioredis";
|
|
2
|
+
/**
|
|
3
|
+
* Shared Redis connection configuration
|
|
4
|
+
* Used by both API and Worker packages
|
|
5
|
+
*/
|
|
6
|
+
export function createRedisConnection() {
|
|
7
|
+
// Use clean URL without embedded password (SEC-005 fix)
|
|
8
|
+
const redisUrl = process.env.REDIS_URL || "redis://localhost:6379";
|
|
9
|
+
// Derive password exclusively from environment variable
|
|
10
|
+
const redisPassword = process.env.REDIS_PASSWORD;
|
|
11
|
+
// Development fallback with explicit warning
|
|
12
|
+
const isDevelopment = process.env.NODE_ENV === "development" || !process.env.NODE_ENV;
|
|
13
|
+
const devPassword = isDevelopment ? "changeme" : undefined;
|
|
14
|
+
// Warn if using development fallback password
|
|
15
|
+
if (isDevelopment && !redisPassword && devPassword) {
|
|
16
|
+
console.warn("⚠️ [Redis] Using default development password. " +
|
|
17
|
+
"Set REDIS_PASSWORD environment variable for production!");
|
|
18
|
+
}
|
|
19
|
+
// Final password: explicit env var, or dev fallback, or none
|
|
20
|
+
const finalPassword = redisPassword || devPassword;
|
|
21
|
+
const options = {
|
|
22
|
+
maxRetriesPerRequest: null,
|
|
23
|
+
lazyConnect: false,
|
|
24
|
+
enableReadyCheck: true,
|
|
25
|
+
retryStrategy(times) {
|
|
26
|
+
if (times > 3) {
|
|
27
|
+
console.error("Redis connection failed after 3 retries");
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
return Math.min(times * 100, 2000);
|
|
31
|
+
},
|
|
32
|
+
};
|
|
33
|
+
// Only include password if available
|
|
34
|
+
if (finalPassword) {
|
|
35
|
+
return new Redis(redisUrl, { ...options, password: finalPassword });
|
|
36
|
+
}
|
|
37
|
+
return new Redis(redisUrl, options);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Singleton Redis connection for the current process
|
|
41
|
+
* Lazy-initialized on first access
|
|
42
|
+
*/
|
|
43
|
+
let _connection = null;
|
|
44
|
+
export function getRedisConnection() {
|
|
45
|
+
if (!_connection) {
|
|
46
|
+
_connection = createRedisConnection();
|
|
47
|
+
}
|
|
48
|
+
return _connection;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Close the singleton Redis connection
|
|
52
|
+
* Should be called during graceful shutdown
|
|
53
|
+
*/
|
|
54
|
+
export async function closeRedisConnection() {
|
|
55
|
+
if (_connection) {
|
|
56
|
+
await _connection.quit();
|
|
57
|
+
_connection = null;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=redis.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redis.js","sourceRoot":"","sources":["../../../../../shared/src/lib/redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,SAAS,CAAC;AAE5B;;;GAGG;AACH,MAAM,UAAU,qBAAqB;IACnC,wDAAwD;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,wBAAwB,CAAC;IAEnE,wDAAwD;IACxD,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAEjD,6CAA6C;IAC7C,MAAM,aAAa,GACjB,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClE,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IAE3D,8CAA8C;IAC9C,IAAI,aAAa,IAAI,CAAC,aAAa,IAAI,WAAW,EAAE,CAAC;QACnD,OAAO,CAAC,IAAI,CACV,kDAAkD;YAChD,yDAAyD,CAC5D,CAAC;IACJ,CAAC;IAED,6DAA6D;IAC7D,MAAM,aAAa,GAAG,aAAa,IAAI,WAAW,CAAC;IAEnD,MAAM,OAAO,GAAG;QACd,oBAAoB,EAAE,IAAI;QAC1B,WAAW,EAAE,KAAK;QAClB,gBAAgB,EAAE,IAAI;QACtB,aAAa,CAAC,KAAa;YACzB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBACzD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;QACrC,CAAC;KACF,CAAC;IAEF,qCAAqC;IACrC,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,IAAI,WAAW,GAAiB,IAAI,CAAC;AAErC,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,qBAAqB,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Available scopes for API keys
|
|
3
|
+
*/
|
|
4
|
+
export type ApiKeyScope = "scrape" | "crawl" | "dork" | "extract" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
|
|
5
|
+
/**
|
|
6
|
+
* All available scopes
|
|
7
|
+
*/
|
|
8
|
+
export declare const ALL_API_KEY_SCOPES: ApiKeyScope[];
|
|
9
|
+
/**
|
|
10
|
+
* Default scopes for new keys
|
|
11
|
+
*/
|
|
12
|
+
export declare const DEFAULT_API_KEY_SCOPES: ApiKeyScope[];
|
|
13
|
+
/**
|
|
14
|
+
* API Key information (without the actual key)
|
|
15
|
+
*/
|
|
16
|
+
export interface ApiKeyInfo {
|
|
17
|
+
id: string;
|
|
18
|
+
name: string;
|
|
19
|
+
keyPrefix: string;
|
|
20
|
+
scopes: ApiKeyScope[];
|
|
21
|
+
isActive: boolean;
|
|
22
|
+
expiresAt: string | null;
|
|
23
|
+
lastUsedAt: string | null;
|
|
24
|
+
usageCount: number;
|
|
25
|
+
createdAt: string;
|
|
26
|
+
revokedAt: string | null;
|
|
27
|
+
revokedReason: string | null;
|
|
28
|
+
ipAllowlist: string[] | null;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Newly created API key (includes raw key - only shown once!)
|
|
32
|
+
*/
|
|
33
|
+
export interface CreatedApiKey {
|
|
34
|
+
id: string;
|
|
35
|
+
rawKey: string;
|
|
36
|
+
keyPrefix: string;
|
|
37
|
+
name: string;
|
|
38
|
+
scopes: ApiKeyScope[];
|
|
39
|
+
expiresAt: string | null;
|
|
40
|
+
createdAt: string;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Request to create a new API key
|
|
44
|
+
*/
|
|
45
|
+
export interface CreateApiKeyRequest {
|
|
46
|
+
name: string;
|
|
47
|
+
scopes?: ApiKeyScope[];
|
|
48
|
+
expiresInDays?: number;
|
|
49
|
+
ipAllowlist?: string[];
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Request to rotate an API key
|
|
53
|
+
*/
|
|
54
|
+
export interface RotateApiKeyRequest {
|
|
55
|
+
newName?: string;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* API key validation result from database
|
|
59
|
+
*/
|
|
60
|
+
export interface ApiKeyValidation {
|
|
61
|
+
clientId: string;
|
|
62
|
+
clientName: string;
|
|
63
|
+
clientTier: string;
|
|
64
|
+
keyId: string;
|
|
65
|
+
keyName: string;
|
|
66
|
+
scopes: ApiKeyScope[];
|
|
67
|
+
quotas: Record<string, number>;
|
|
68
|
+
usage: Record<string, number>;
|
|
69
|
+
isValid: boolean;
|
|
70
|
+
errorMessage: string | null;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* API key usage statistics
|
|
74
|
+
*/
|
|
75
|
+
export interface ApiKeyStats {
|
|
76
|
+
totalRequests: number;
|
|
77
|
+
successfulRequests: number;
|
|
78
|
+
failedRequests: number;
|
|
79
|
+
avgDurationMs: number;
|
|
80
|
+
endpointsUsed: Record<string, number>;
|
|
81
|
+
requestsByDay: Record<string, number>;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Context from validated API key (attached to request)
|
|
85
|
+
*/
|
|
86
|
+
export interface ApiKeyContext {
|
|
87
|
+
keyId: string;
|
|
88
|
+
keyName: string;
|
|
89
|
+
scopes: ApiKeyScope[];
|
|
90
|
+
clientId: string;
|
|
91
|
+
clientName: string;
|
|
92
|
+
clientTier: string;
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=api-key.types.d.ts.map
|