@dealcrawl/sdk 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/README.md +540 -0
  2. package/dist/sdk/src/client.d.ts +285 -0
  3. package/dist/sdk/src/client.d.ts.map +1 -0
  4. package/dist/sdk/src/client.js +336 -0
  5. package/dist/sdk/src/client.js.map +1 -0
  6. package/dist/sdk/src/error.d.ts +54 -0
  7. package/dist/sdk/src/error.d.ts.map +1 -0
  8. package/dist/sdk/src/error.js +115 -0
  9. package/dist/sdk/src/error.js.map +1 -0
  10. package/dist/sdk/src/index.d.ts +58 -0
  11. package/dist/sdk/src/index.d.ts.map +1 -0
  12. package/dist/sdk/src/index.js +65 -0
  13. package/dist/sdk/src/index.js.map +1 -0
  14. package/dist/sdk/src/resources/account.d.ts +143 -0
  15. package/dist/sdk/src/resources/account.d.ts.map +1 -0
  16. package/dist/sdk/src/resources/account.js +186 -0
  17. package/dist/sdk/src/resources/account.js.map +1 -0
  18. package/dist/sdk/src/resources/crawl.d.ts +87 -0
  19. package/dist/sdk/src/resources/crawl.d.ts.map +1 -0
  20. package/dist/sdk/src/resources/crawl.js +205 -0
  21. package/dist/sdk/src/resources/crawl.js.map +1 -0
  22. package/dist/sdk/src/resources/data.d.ts +157 -0
  23. package/dist/sdk/src/resources/data.d.ts.map +1 -0
  24. package/dist/sdk/src/resources/data.js +239 -0
  25. package/dist/sdk/src/resources/data.js.map +1 -0
  26. package/dist/sdk/src/resources/dork.d.ts +104 -0
  27. package/dist/sdk/src/resources/dork.d.ts.map +1 -0
  28. package/dist/sdk/src/resources/dork.js +163 -0
  29. package/dist/sdk/src/resources/dork.js.map +1 -0
  30. package/dist/sdk/src/resources/extract.d.ts +105 -0
  31. package/dist/sdk/src/resources/extract.d.ts.map +1 -0
  32. package/dist/sdk/src/resources/extract.js +246 -0
  33. package/dist/sdk/src/resources/extract.js.map +1 -0
  34. package/dist/sdk/src/resources/index.d.ts +14 -0
  35. package/dist/sdk/src/resources/index.d.ts.map +1 -0
  36. package/dist/sdk/src/resources/index.js +14 -0
  37. package/dist/sdk/src/resources/index.js.map +1 -0
  38. package/dist/sdk/src/resources/keys.d.ts +124 -0
  39. package/dist/sdk/src/resources/keys.d.ts.map +1 -0
  40. package/dist/sdk/src/resources/keys.js +168 -0
  41. package/dist/sdk/src/resources/keys.js.map +1 -0
  42. package/dist/sdk/src/resources/scrape.d.ts +53 -0
  43. package/dist/sdk/src/resources/scrape.d.ts.map +1 -0
  44. package/dist/sdk/src/resources/scrape.js +85 -0
  45. package/dist/sdk/src/resources/scrape.js.map +1 -0
  46. package/dist/sdk/src/resources/status.d.ts +100 -0
  47. package/dist/sdk/src/resources/status.d.ts.map +1 -0
  48. package/dist/sdk/src/resources/status.js +133 -0
  49. package/dist/sdk/src/resources/status.js.map +1 -0
  50. package/dist/sdk/src/resources/webhooks.d.ts +126 -0
  51. package/dist/sdk/src/resources/webhooks.d.ts.map +1 -0
  52. package/dist/sdk/src/resources/webhooks.js +167 -0
  53. package/dist/sdk/src/resources/webhooks.js.map +1 -0
  54. package/dist/sdk/src/types/config.d.ts +45 -0
  55. package/dist/sdk/src/types/config.d.ts.map +1 -0
  56. package/dist/sdk/src/types/config.js +10 -0
  57. package/dist/sdk/src/types/config.js.map +1 -0
  58. package/dist/sdk/src/types/index.d.ts +8 -0
  59. package/dist/sdk/src/types/index.d.ts.map +1 -0
  60. package/dist/sdk/src/types/index.js +8 -0
  61. package/dist/sdk/src/types/index.js.map +1 -0
  62. package/dist/sdk/src/types/options.d.ts +286 -0
  63. package/dist/sdk/src/types/options.d.ts.map +1 -0
  64. package/dist/sdk/src/types/options.js +6 -0
  65. package/dist/sdk/src/types/options.js.map +1 -0
  66. package/dist/sdk/src/types/responses.d.ts +385 -0
  67. package/dist/sdk/src/types/responses.d.ts.map +1 -0
  68. package/dist/sdk/src/types/responses.js +6 -0
  69. package/dist/sdk/src/types/responses.js.map +1 -0
  70. package/dist/sdk/src/utils/polling.d.ts +57 -0
  71. package/dist/sdk/src/utils/polling.d.ts.map +1 -0
  72. package/dist/sdk/src/utils/polling.js +110 -0
  73. package/dist/sdk/src/utils/polling.js.map +1 -0
  74. package/dist/sdk/src/utils/request.d.ts +47 -0
  75. package/dist/sdk/src/utils/request.d.ts.map +1 -0
  76. package/dist/sdk/src/utils/request.js +161 -0
  77. package/dist/sdk/src/utils/request.js.map +1 -0
  78. package/dist/shared/src/constants/errors.d.ts +26 -0
  79. package/dist/shared/src/constants/errors.d.ts.map +1 -0
  80. package/dist/shared/src/constants/errors.js +39 -0
  81. package/dist/shared/src/constants/errors.js.map +1 -0
  82. package/dist/shared/src/constants/http.d.ts +26 -0
  83. package/dist/shared/src/constants/http.d.ts.map +1 -0
  84. package/dist/shared/src/constants/http.js +26 -0
  85. package/dist/shared/src/constants/http.js.map +1 -0
  86. package/dist/shared/src/constants/index.d.ts +4 -0
  87. package/dist/shared/src/constants/index.d.ts.map +1 -0
  88. package/dist/shared/src/constants/index.js +5 -0
  89. package/dist/shared/src/constants/index.js.map +1 -0
  90. package/dist/shared/src/constants/limits.d.ts +52 -0
  91. package/dist/shared/src/constants/limits.d.ts.map +1 -0
  92. package/dist/shared/src/constants/limits.js +43 -0
  93. package/dist/shared/src/constants/limits.js.map +1 -0
  94. package/dist/shared/src/index.d.ts +5 -0
  95. package/dist/shared/src/index.d.ts.map +1 -0
  96. package/dist/shared/src/index.js +11 -0
  97. package/dist/shared/src/index.js.map +1 -0
  98. package/dist/shared/src/lib/index.d.ts +2 -0
  99. package/dist/shared/src/lib/index.d.ts.map +1 -0
  100. package/dist/shared/src/lib/index.js +2 -0
  101. package/dist/shared/src/lib/index.js.map +1 -0
  102. package/dist/shared/src/lib/redis.d.ts +14 -0
  103. package/dist/shared/src/lib/redis.d.ts.map +1 -0
  104. package/dist/shared/src/lib/redis.js +60 -0
  105. package/dist/shared/src/lib/redis.js.map +1 -0
  106. package/dist/shared/src/types/api-key.types.d.ts +94 -0
  107. package/dist/shared/src/types/api-key.types.d.ts.map +1 -0
  108. package/dist/shared/src/types/api-key.types.js +30 -0
  109. package/dist/shared/src/types/api-key.types.js.map +1 -0
  110. package/dist/shared/src/types/api.types.d.ts +38 -0
  111. package/dist/shared/src/types/api.types.d.ts.map +1 -0
  112. package/dist/shared/src/types/api.types.js +2 -0
  113. package/dist/shared/src/types/api.types.js.map +1 -0
  114. package/dist/shared/src/types/client.types.d.ts +73 -0
  115. package/dist/shared/src/types/client.types.d.ts.map +1 -0
  116. package/dist/shared/src/types/client.types.js +9 -0
  117. package/dist/shared/src/types/client.types.js.map +1 -0
  118. package/dist/shared/src/types/crawl.types.d.ts +65 -0
  119. package/dist/shared/src/types/crawl.types.d.ts.map +1 -0
  120. package/dist/shared/src/types/crawl.types.js +2 -0
  121. package/dist/shared/src/types/crawl.types.js.map +1 -0
  122. package/dist/shared/src/types/deal.types.d.ts +210 -0
  123. package/dist/shared/src/types/deal.types.d.ts.map +1 -0
  124. package/dist/shared/src/types/deal.types.js +6 -0
  125. package/dist/shared/src/types/deal.types.js.map +1 -0
  126. package/dist/shared/src/types/dork.types.d.ts +29 -0
  127. package/dist/shared/src/types/dork.types.d.ts.map +1 -0
  128. package/dist/shared/src/types/dork.types.js +2 -0
  129. package/dist/shared/src/types/dork.types.js.map +1 -0
  130. package/dist/shared/src/types/index.d.ts +8 -0
  131. package/dist/shared/src/types/index.d.ts.map +1 -0
  132. package/dist/shared/src/types/index.js +9 -0
  133. package/dist/shared/src/types/index.js.map +1 -0
  134. package/dist/shared/src/types/scrape.types.d.ts +151 -0
  135. package/dist/shared/src/types/scrape.types.d.ts.map +1 -0
  136. package/dist/shared/src/types/scrape.types.js +2 -0
  137. package/dist/shared/src/types/scrape.types.js.map +1 -0
  138. package/dist/shared/src/utils/date.d.ts +7 -0
  139. package/dist/shared/src/utils/date.d.ts.map +1 -0
  140. package/dist/shared/src/utils/date.js +25 -0
  141. package/dist/shared/src/utils/date.js.map +1 -0
  142. package/dist/shared/src/utils/hash.d.ts +4 -0
  143. package/dist/shared/src/utils/hash.d.ts.map +1 -0
  144. package/dist/shared/src/utils/hash.js +21 -0
  145. package/dist/shared/src/utils/hash.js.map +1 -0
  146. package/dist/shared/src/utils/index.d.ts +7 -0
  147. package/dist/shared/src/utils/index.d.ts.map +1 -0
  148. package/dist/shared/src/utils/index.js +8 -0
  149. package/dist/shared/src/utils/index.js.map +1 -0
  150. package/dist/shared/src/utils/logger.d.ts +66 -0
  151. package/dist/shared/src/utils/logger.d.ts.map +1 -0
  152. package/dist/shared/src/utils/logger.js +268 -0
  153. package/dist/shared/src/utils/logger.js.map +1 -0
  154. package/dist/shared/src/utils/retry.d.ts +11 -0
  155. package/dist/shared/src/utils/retry.d.ts.map +1 -0
  156. package/dist/shared/src/utils/retry.js +36 -0
  157. package/dist/shared/src/utils/retry.js.map +1 -0
  158. package/dist/shared/src/utils/url-validator.d.ts +37 -0
  159. package/dist/shared/src/utils/url-validator.d.ts.map +1 -0
  160. package/dist/shared/src/utils/url-validator.js +179 -0
  161. package/dist/shared/src/utils/url-validator.js.map +1 -0
  162. package/dist/shared/src/utils/url.d.ts +6 -0
  163. package/dist/shared/src/utils/url.d.ts.map +1 -0
  164. package/dist/shared/src/utils/url.js +56 -0
  165. package/dist/shared/src/utils/url.js.map +1 -0
  166. package/package.json +49 -0
@@ -0,0 +1,161 @@
1
+ import { DealCrawlError, ERROR_CODES } from "../error";
2
+ /**
3
+ * Build query string from object
4
+ */
5
+ function buildQueryString(params) {
6
+ const entries = Object.entries(params)
7
+ .filter(([, value]) => value !== undefined)
8
+ .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`);
9
+ return entries.length > 0 ? `?${entries.join("&")}` : "";
10
+ }
11
+ /**
12
+ * Parse rate limit headers from response
13
+ */
14
+ function parseRateLimitHeaders(headers) {
15
+ return {
16
+ limit: parseInt(headers.get("X-RateLimit-Limit") || "0", 10) || 0,
17
+ remaining: parseInt(headers.get("X-RateLimit-Remaining") || "0", 10) || 0,
18
+ reset: parseInt(headers.get("X-RateLimit-Reset") || "0", 10) || 0,
19
+ };
20
+ }
21
+ /**
22
+ * Sleep helper for retries
23
+ */
24
+ function sleep(ms) {
25
+ return new Promise((resolve) => setTimeout(resolve, ms));
26
+ }
27
+ /**
28
+ * Make an HTTP request with authentication, error handling, and retries
29
+ */
30
+ export async function request(ctx, path, options = {}) {
31
+ const { method = "GET", body, query, timeout, signal } = options;
32
+ // Build URL with query string
33
+ const queryString = query ? buildQueryString(query) : "";
34
+ const url = `${ctx.baseUrl}${path}${queryString}`;
35
+ // Create abort controller for timeout
36
+ const controller = new AbortController();
37
+ const timeoutMs = timeout ?? ctx.timeout;
38
+ const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
39
+ // Combine with external signal if provided
40
+ if (signal) {
41
+ signal.addEventListener("abort", () => controller.abort());
42
+ }
43
+ let lastError = null;
44
+ let attempt = 0;
45
+ while (attempt <= ctx.maxRetries) {
46
+ try {
47
+ const response = await fetch(url, {
48
+ method,
49
+ headers: {
50
+ "Content-Type": "application/json",
51
+ Authorization: `Bearer ${ctx.apiKey}`,
52
+ },
53
+ body: body ? JSON.stringify(body) : undefined,
54
+ signal: controller.signal,
55
+ });
56
+ clearTimeout(timeoutId);
57
+ // Parse rate limit headers
58
+ const rateLimit = parseRateLimitHeaders(response.headers);
59
+ // Handle rate limiting
60
+ if (response.status === 429) {
61
+ const retryAfter = parseInt(response.headers.get("Retry-After") || "60", 10);
62
+ // Notify callback
63
+ if (ctx.onRateLimit) {
64
+ ctx.onRateLimit(rateLimit);
65
+ }
66
+ throw new DealCrawlError({
67
+ code: ERROR_CODES.RATE_LIMIT_EXCEEDED,
68
+ message: "Rate limit exceeded",
69
+ statusCode: 429,
70
+ retryAfter,
71
+ details: { rateLimit },
72
+ });
73
+ }
74
+ // Handle errors
75
+ if (!response.ok) {
76
+ let errorBody = {};
77
+ try {
78
+ errorBody = (await response.json());
79
+ }
80
+ catch {
81
+ // Response might not be JSON
82
+ }
83
+ throw DealCrawlError.fromResponse(response.status, errorBody);
84
+ }
85
+ // Parse response
86
+ let data;
87
+ const contentType = response.headers.get("Content-Type") || "";
88
+ if (contentType.includes("application/json")) {
89
+ data = (await response.json());
90
+ }
91
+ else if (contentType.includes("text/csv") ||
92
+ contentType.includes("text/plain")) {
93
+ data = (await response.text());
94
+ }
95
+ else {
96
+ data = (await response.json());
97
+ }
98
+ return { data, rateLimit };
99
+ }
100
+ catch (error) {
101
+ clearTimeout(timeoutId);
102
+ // Handle abort/timeout
103
+ if (error instanceof Error && error.name === "AbortError") {
104
+ throw new DealCrawlError({
105
+ code: ERROR_CODES.JOB_TIMEOUT,
106
+ message: `Request timeout after ${timeoutMs}ms`,
107
+ statusCode: 408,
108
+ });
109
+ }
110
+ // Handle DealCrawlError
111
+ if (error instanceof DealCrawlError) {
112
+ lastError = error;
113
+ // Retry if error is retryable and we have attempts left
114
+ if (error.isRetryable() && attempt < ctx.maxRetries) {
115
+ attempt++;
116
+ await sleep(ctx.retryDelay * attempt); // Exponential backoff
117
+ continue;
118
+ }
119
+ throw error;
120
+ }
121
+ // Handle network errors
122
+ throw new DealCrawlError({
123
+ code: ERROR_CODES.FETCH_FAILED,
124
+ message: error instanceof Error ? error.message : "Network request failed",
125
+ statusCode: 0,
126
+ });
127
+ }
128
+ }
129
+ // Should not reach here, but throw last error if we do
130
+ throw (lastError ??
131
+ new DealCrawlError({
132
+ code: ERROR_CODES.INTERNAL_ERROR,
133
+ message: "Request failed after retries",
134
+ statusCode: 500,
135
+ }));
136
+ }
137
+ /**
138
+ * Convenience method for GET requests
139
+ */
140
+ export function get(ctx, path, query) {
141
+ return request(ctx, path, { method: "GET", query });
142
+ }
143
+ /**
144
+ * Convenience method for POST requests
145
+ */
146
+ export function post(ctx, path, body) {
147
+ return request(ctx, path, { method: "POST", body });
148
+ }
149
+ /**
150
+ * Convenience method for PATCH requests
151
+ */
152
+ export function patch(ctx, path, body) {
153
+ return request(ctx, path, { method: "PATCH", body });
154
+ }
155
+ /**
156
+ * Convenience method for DELETE requests
157
+ */
158
+ export function del(ctx, path, body) {
159
+ return request(ctx, path, { method: "DELETE", body });
160
+ }
161
+ //# sourceMappingURL=request.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"request.js","sourceRoot":"","sources":["../../../../src/utils/request.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAkCvD;;GAEG;AACH,SAAS,gBAAgB,CACvB,MAA6D;IAE7D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACnC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC;SAC1C,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,kBAAkB,CAAC,GAAG,CAAC,IAAI,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CACpE,CAAC;IAEJ,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,OAAgB;IAC7C,OAAO;QACL,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC;QACjE,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC;QACzE,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC;KAClE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,GAAmB,EACnB,IAAY,EACZ,UAA0B,EAAE;IAE5B,MAAM,EAAE,MAAM,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAEjE,8BAA8B;IAC9B,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,OAAO,GAAG,IAAI,GAAG,WAAW,EAAE,CAAC;IAElD,sCAAsC;IACtC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAElE,2CAA2C;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,SAAS,GAA0B,IAAI,CAAC;IAC5C,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,OAAO,OAAO,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,MAAM;gBACN,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,GAAG,CAAC,MAAM,EAAE;iBACtC;gBACD,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;gBAC7C,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,2BAA2B;YAC3B,MAAM,SAAS,GAAG,qBAAqB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE1D,uBAAuB;YACvB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,QAAQ,CACzB,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,IAAI,EAC3C,EAAE,CACH,CAAC;gBAEF,kBAAkB;gBAClB,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;oBACpB,GAAG,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;gBAC7B,CAAC;gBAED,MAAM,IAAI,cAAc,CAAC;oBACvB,IAAI,EAAE,WAAW,CAAC,mBAAmB;oBACrC,OAAO,EAAE,qBAAqB;oBAC9B,UAAU,EAAE,GAAG;oBACf,UAAU;oBACV,OAAO,EAAE,EAAE,SAAS,EAAE;iBACvB,CAAC,CAAC;YACL,CAAC;YAED,gBAAgB;YAChB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,SAAS,GAA4B,EAAE,CAAC;gBAC5C,IAAI,CAAC;oBACH,SAAS,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA4B,CAAC;gBACjE,CAAC;gBAAC,MAAM,CAAC;oBACP,6BAA6B;gBAC/B,CAAC;gBAED,MAAM,cAAc,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAChE,CAAC;YAED,iBAAiB;YACjB,IAAI,IAAO,CAAC;YACZ,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE/D,IAAI,WAAW,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAC7C,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAC;YACtC,CAAC;iBAAM,IACL,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAChC,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,EAClC,CAAC;gBACD,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAiB,CAAC;YACjD,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAM,CAAC;YACtC,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,uBAAuB;YACvB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,cAAc,CAAC;oBACvB,IAAI,EAAE,WAAW,CAAC,WAAW;oBAC7B,OAAO,EAAE,yBAAyB,SAAS,IAAI;oBAC/C,UAAU,EAAE,GAAG;iBAChB,CAAC,CAAC;YACL,CAAC;YAED,wBAAwB;YACxB,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,SAAS,GAAG,KAAK,CAAC;gBAElB,wDAAwD;gBACxD,IAAI,KAAK,CAAC,WAAW,EAAE,IAAI,OAAO,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;oBACpD,OAAO,EAAE,CAAC;oBACV,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,sBAAsB;oBAC7D,SAAS;gBACX,CAAC;gBAED,MAAM,KAAK,CAAC;YACd,CAAC;YAED,wBAAwB;YACxB,MAAM,IAAI,cAAc,CAAC;gBACvB,IAAI,EAAE,WAAW,CAAC,YAAY;gBAC9B,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;gBACnE,UAAU,EAAE,CAAC;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,MAAM,CACJ,SAAS;QACT,IAAI,cAAc,CAAC;YACjB,IAAI,EAAE,WAAW,CAAC,cAAc;YAChC,OAAO,EAAE,8BAA8B;YACvC,UAAU,EAAE,GAAG;SAChB,CAAC,CACH,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CACjB,GAAmB,EACnB,IAAY,EACZ,KAA6D;IAE7D,OAAO,OAAO,CAAI,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;AACzD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,IAAI,CAClB,GAAmB,EACnB,IAAY,EACZ,IAAc;IAEd,OAAO,OAAO,CAAI,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;AACzD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,KAAK,CACnB,GAAmB,EACnB,IAAY,EACZ,IAAc;IAEd,OAAO,OAAO,CAAI,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1D,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,CACjB,GAAmB,EACnB,IAAY,EACZ,IAAc;IAEd,OAAO,OAAO,CAAI,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3D,CAAC"}
@@ -0,0 +1,26 @@
1
+ export declare const ERROR_CODES: {
2
+ readonly INVALID_API_KEY: "INVALID_API_KEY";
3
+ readonly MISSING_API_KEY: "MISSING_API_KEY";
4
+ readonly API_KEY_EXPIRED: "API_KEY_EXPIRED";
5
+ readonly ACCOUNT_SUSPENDED: "ACCOUNT_SUSPENDED";
6
+ readonly RATE_LIMIT_EXCEEDED: "RATE_LIMIT_EXCEEDED";
7
+ readonly QUOTA_EXCEEDED: "QUOTA_EXCEEDED";
8
+ readonly INVALID_URL: "INVALID_URL";
9
+ readonly INVALID_REQUEST: "INVALID_REQUEST";
10
+ readonly MISSING_REQUIRED_FIELD: "MISSING_REQUIRED_FIELD";
11
+ readonly JOB_NOT_FOUND: "JOB_NOT_FOUND";
12
+ readonly JOB_FAILED: "JOB_FAILED";
13
+ readonly JOB_TIMEOUT: "JOB_TIMEOUT";
14
+ readonly FETCH_FAILED: "FETCH_FAILED";
15
+ readonly PARSE_FAILED: "PARSE_FAILED";
16
+ readonly BLOCKED_BY_ROBOTS: "BLOCKED_BY_ROBOTS";
17
+ readonly CAPTCHA_DETECTED: "CAPTCHA_DETECTED";
18
+ readonly SITE_UNREACHABLE: "SITE_UNREACHABLE";
19
+ readonly INTERNAL_ERROR: "INTERNAL_ERROR";
20
+ readonly SERVICE_UNAVAILABLE: "SERVICE_UNAVAILABLE";
21
+ readonly REDIS_ERROR: "REDIS_ERROR";
22
+ readonly DATABASE_ERROR: "DATABASE_ERROR";
23
+ };
24
+ export type ErrorCode = (typeof ERROR_CODES)[keyof typeof ERROR_CODES];
25
+ export declare const ERROR_MESSAGES: Record<string, string>;
26
+ //# sourceMappingURL=errors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../../../shared/src/constants/errors.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;CAiCd,CAAC;AAEX,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,OAAO,WAAW,CAAC,CAAC;AAEvE,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQjD,CAAC"}
@@ -0,0 +1,39 @@
1
+ export const ERROR_CODES = {
2
+ // Authentication
3
+ INVALID_API_KEY: "INVALID_API_KEY",
4
+ MISSING_API_KEY: "MISSING_API_KEY",
5
+ API_KEY_EXPIRED: "API_KEY_EXPIRED",
6
+ ACCOUNT_SUSPENDED: "ACCOUNT_SUSPENDED",
7
+ // Rate limiting
8
+ RATE_LIMIT_EXCEEDED: "RATE_LIMIT_EXCEEDED",
9
+ QUOTA_EXCEEDED: "QUOTA_EXCEEDED",
10
+ // Validation
11
+ INVALID_URL: "INVALID_URL",
12
+ INVALID_REQUEST: "INVALID_REQUEST",
13
+ MISSING_REQUIRED_FIELD: "MISSING_REQUIRED_FIELD",
14
+ // Job errors
15
+ JOB_NOT_FOUND: "JOB_NOT_FOUND",
16
+ JOB_FAILED: "JOB_FAILED",
17
+ JOB_TIMEOUT: "JOB_TIMEOUT",
18
+ // Scraping errors
19
+ FETCH_FAILED: "FETCH_FAILED",
20
+ PARSE_FAILED: "PARSE_FAILED",
21
+ BLOCKED_BY_ROBOTS: "BLOCKED_BY_ROBOTS",
22
+ CAPTCHA_DETECTED: "CAPTCHA_DETECTED",
23
+ SITE_UNREACHABLE: "SITE_UNREACHABLE",
24
+ // System errors
25
+ INTERNAL_ERROR: "INTERNAL_ERROR",
26
+ SERVICE_UNAVAILABLE: "SERVICE_UNAVAILABLE",
27
+ REDIS_ERROR: "REDIS_ERROR",
28
+ DATABASE_ERROR: "DATABASE_ERROR",
29
+ };
30
+ export const ERROR_MESSAGES = {
31
+ [ERROR_CODES.INVALID_API_KEY]: "The provided API key is invalid",
32
+ [ERROR_CODES.MISSING_API_KEY]: "API key is required in Authorization header",
33
+ [ERROR_CODES.RATE_LIMIT_EXCEEDED]: "Rate limit exceeded. Please try again later",
34
+ [ERROR_CODES.QUOTA_EXCEEDED]: "Monthly quota exceeded. Upgrade your plan for more",
35
+ [ERROR_CODES.INVALID_URL]: "The provided URL is not valid",
36
+ [ERROR_CODES.JOB_NOT_FOUND]: "Job not found",
37
+ [ERROR_CODES.INTERNAL_ERROR]: "An internal error occurred",
38
+ };
39
+ //# sourceMappingURL=errors.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors.js","sourceRoot":"","sources":["../../../../../shared/src/constants/errors.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,iBAAiB;IACjB,eAAe,EAAE,iBAAiB;IAClC,eAAe,EAAE,iBAAiB;IAClC,eAAe,EAAE,iBAAiB;IAClC,iBAAiB,EAAE,mBAAmB;IAEtC,gBAAgB;IAChB,mBAAmB,EAAE,qBAAqB;IAC1C,cAAc,EAAE,gBAAgB;IAEhC,aAAa;IACb,WAAW,EAAE,aAAa;IAC1B,eAAe,EAAE,iBAAiB;IAClC,sBAAsB,EAAE,wBAAwB;IAEhD,aAAa;IACb,aAAa,EAAE,eAAe;IAC9B,UAAU,EAAE,YAAY;IACxB,WAAW,EAAE,aAAa;IAE1B,kBAAkB;IAClB,YAAY,EAAE,cAAc;IAC5B,YAAY,EAAE,cAAc;IAC5B,iBAAiB,EAAE,mBAAmB;IACtC,gBAAgB,EAAE,kBAAkB;IACpC,gBAAgB,EAAE,kBAAkB;IAEpC,gBAAgB;IAChB,cAAc,EAAE,gBAAgB;IAChC,mBAAmB,EAAE,qBAAqB;IAC1C,WAAW,EAAE,aAAa;IAC1B,cAAc,EAAE,gBAAgB;CACxB,CAAC;AAIX,MAAM,CAAC,MAAM,cAAc,GAA2B;IACpD,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,iCAAiC;IAChE,CAAC,WAAW,CAAC,eAAe,CAAC,EAAE,6CAA6C;IAC5E,CAAC,WAAW,CAAC,mBAAmB,CAAC,EAAE,6CAA6C;IAChF,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE,oDAAoD;IAClF,CAAC,WAAW,CAAC,WAAW,CAAC,EAAE,+BAA+B;IAC1D,CAAC,WAAW,CAAC,aAAa,CAAC,EAAE,eAAe;IAC5C,CAAC,WAAW,CAAC,cAAc,CAAC,EAAE,4BAA4B;CAC3D,CAAC"}
@@ -0,0 +1,26 @@
1
+ export declare const HTTP_STATUS: {
2
+ readonly OK: 200;
3
+ readonly CREATED: 201;
4
+ readonly ACCEPTED: 202;
5
+ readonly NO_CONTENT: 204;
6
+ readonly BAD_REQUEST: 400;
7
+ readonly UNAUTHORIZED: 401;
8
+ readonly FORBIDDEN: 403;
9
+ readonly NOT_FOUND: 404;
10
+ readonly CONFLICT: 409;
11
+ readonly UNPROCESSABLE_ENTITY: 422;
12
+ readonly TOO_MANY_REQUESTS: 429;
13
+ readonly INTERNAL_SERVER_ERROR: 500;
14
+ readonly BAD_GATEWAY: 502;
15
+ readonly SERVICE_UNAVAILABLE: 503;
16
+ };
17
+ export declare const DEFAULT_HEADERS: {
18
+ readonly "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
19
+ readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
20
+ readonly "Accept-Language": "en-US,en;q=0.9";
21
+ readonly "Accept-Encoding": "gzip, deflate, br";
22
+ readonly Connection: "keep-alive";
23
+ };
24
+ export declare const API_VERSION = "v1";
25
+ export declare const API_PREFIX = "/api/v1";
26
+ //# sourceMappingURL=http.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../../../../shared/src/constants/http.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;CAed,CAAC;AAEX,eAAO,MAAM,eAAe;;;;;;CAOlB,CAAC;AAEX,eAAO,MAAM,WAAW,OAAO,CAAC;AAChC,eAAO,MAAM,UAAU,YAAwB,CAAC"}
@@ -0,0 +1,26 @@
1
+ export const HTTP_STATUS = {
2
+ OK: 200,
3
+ CREATED: 201,
4
+ ACCEPTED: 202,
5
+ NO_CONTENT: 204,
6
+ BAD_REQUEST: 400,
7
+ UNAUTHORIZED: 401,
8
+ FORBIDDEN: 403,
9
+ NOT_FOUND: 404,
10
+ CONFLICT: 409,
11
+ UNPROCESSABLE_ENTITY: 422,
12
+ TOO_MANY_REQUESTS: 429,
13
+ INTERNAL_SERVER_ERROR: 500,
14
+ BAD_GATEWAY: 502,
15
+ SERVICE_UNAVAILABLE: 503,
16
+ };
17
+ export const DEFAULT_HEADERS = {
18
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
19
+ Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
20
+ "Accept-Language": "en-US,en;q=0.9",
21
+ "Accept-Encoding": "gzip, deflate, br",
22
+ Connection: "keep-alive",
23
+ };
24
+ export const API_VERSION = "v1";
25
+ export const API_PREFIX = `/api/${API_VERSION}`;
26
+ //# sourceMappingURL=http.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"http.js","sourceRoot":"","sources":["../../../../../shared/src/constants/http.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,EAAE,EAAE,GAAG;IACP,OAAO,EAAE,GAAG;IACZ,QAAQ,EAAE,GAAG;IACb,UAAU,EAAE,GAAG;IACf,WAAW,EAAE,GAAG;IAChB,YAAY,EAAE,GAAG;IACjB,SAAS,EAAE,GAAG;IACd,SAAS,EAAE,GAAG;IACd,QAAQ,EAAE,GAAG;IACb,oBAAoB,EAAE,GAAG;IACzB,iBAAiB,EAAE,GAAG;IACtB,qBAAqB,EAAE,GAAG;IAC1B,WAAW,EAAE,GAAG;IAChB,mBAAmB,EAAE,GAAG;CAChB,CAAC;AAEX,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,YAAY,EACV,iHAAiH;IACnH,MAAM,EAAE,iEAAiE;IACzE,iBAAiB,EAAE,gBAAgB;IACnC,iBAAiB,EAAE,mBAAmB;IACtC,UAAU,EAAE,YAAY;CAChB,CAAC;AAEX,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC;AAChC,MAAM,CAAC,MAAM,UAAU,GAAG,QAAQ,WAAW,EAAE,CAAC"}
@@ -0,0 +1,4 @@
1
+ export * from "./http";
2
+ export * from "./limits";
3
+ export * from "./errors";
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../shared/src/constants/index.ts"],"names":[],"mappings":"AACA,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC"}
@@ -0,0 +1,5 @@
1
+ // Constants exports
2
+ export * from "./http";
3
+ export * from "./limits";
4
+ export * from "./errors";
5
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../shared/src/constants/index.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB,cAAc,QAAQ,CAAC;AACvB,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC"}
@@ -0,0 +1,52 @@
1
+ export declare const CRAWL_LIMITS: {
2
+ readonly MAX_DEPTH: 5;
3
+ readonly MAX_PAGES: 1000;
4
+ readonly DEFAULT_DEPTH: 3;
5
+ readonly DEFAULT_PAGES: 100;
6
+ readonly DEFAULT_DELAY_MS: 1000;
7
+ readonly MAX_DELAY_MS: 10000;
8
+ readonly MIN_DELAY_MS: 100;
9
+ };
10
+ export declare const SCRAPE_LIMITS: {
11
+ readonly MAX_CONTENT_LENGTH: 10000000;
12
+ readonly DEFAULT_TIMEOUT_MS: 30000;
13
+ readonly MAX_TIMEOUT_MS: 120000;
14
+ readonly MAX_RETRIES: 3;
15
+ };
16
+ export declare const DORK_LIMITS: {
17
+ readonly MAX_RESULTS: 100;
18
+ readonly DEFAULT_RESULTS: 10;
19
+ readonly MAX_QUERY_LENGTH: 500;
20
+ };
21
+ export declare const QUOTA_LIMITS: {
22
+ readonly free: {
23
+ readonly crawls: 100;
24
+ readonly scrapes: 1000;
25
+ readonly dorks: 50;
26
+ };
27
+ readonly pro: {
28
+ readonly crawls: 1000;
29
+ readonly scrapes: 10000;
30
+ readonly dorks: 500;
31
+ };
32
+ readonly enterprise: {
33
+ readonly crawls: 999999;
34
+ readonly scrapes: 999999;
35
+ readonly dorks: 999999;
36
+ };
37
+ };
38
+ export declare const RATE_LIMITS: {
39
+ readonly free: {
40
+ readonly requests: 100;
41
+ readonly windowMs: 3600000;
42
+ };
43
+ readonly pro: {
44
+ readonly requests: 1000;
45
+ readonly windowMs: 3600000;
46
+ };
47
+ readonly enterprise: {
48
+ readonly requests: 10000;
49
+ readonly windowMs: 3600000;
50
+ };
51
+ };
52
+ //# sourceMappingURL=limits.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"limits.d.ts","sourceRoot":"","sources":["../../../../../shared/src/constants/limits.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY;;;;;;;;CAQf,CAAC;AAEX,eAAO,MAAM,aAAa;;;;;CAKhB,CAAC;AAEX,eAAO,MAAM,WAAW;;;;CAId,CAAC;AAEX,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;CAgBf,CAAC;AAEX,eAAO,MAAM,WAAW;;;;;;;;;;;;;CAId,CAAC"}
@@ -0,0 +1,43 @@
1
+ export const CRAWL_LIMITS = {
2
+ MAX_DEPTH: 5,
3
+ MAX_PAGES: 1000,
4
+ DEFAULT_DEPTH: 3,
5
+ DEFAULT_PAGES: 100,
6
+ DEFAULT_DELAY_MS: 1000,
7
+ MAX_DELAY_MS: 10000,
8
+ MIN_DELAY_MS: 100,
9
+ };
10
+ export const SCRAPE_LIMITS = {
11
+ MAX_CONTENT_LENGTH: 10_000_000, // 10MB
12
+ DEFAULT_TIMEOUT_MS: 30_000,
13
+ MAX_TIMEOUT_MS: 120_000,
14
+ MAX_RETRIES: 3,
15
+ };
16
+ export const DORK_LIMITS = {
17
+ MAX_RESULTS: 100,
18
+ DEFAULT_RESULTS: 10,
19
+ MAX_QUERY_LENGTH: 500,
20
+ };
21
+ export const QUOTA_LIMITS = {
22
+ free: {
23
+ crawls: 100,
24
+ scrapes: 1000,
25
+ dorks: 50,
26
+ },
27
+ pro: {
28
+ crawls: 1000,
29
+ scrapes: 10000,
30
+ dorks: 500,
31
+ },
32
+ enterprise: {
33
+ crawls: 999999,
34
+ scrapes: 999999,
35
+ dorks: 999999,
36
+ },
37
+ };
38
+ export const RATE_LIMITS = {
39
+ free: { requests: 100, windowMs: 3600000 },
40
+ pro: { requests: 1000, windowMs: 3600000 },
41
+ enterprise: { requests: 10000, windowMs: 3600000 },
42
+ };
43
+ //# sourceMappingURL=limits.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"limits.js","sourceRoot":"","sources":["../../../../../shared/src/constants/limits.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,IAAI;IACf,aAAa,EAAE,CAAC;IAChB,aAAa,EAAE,GAAG;IAClB,gBAAgB,EAAE,IAAI;IACtB,YAAY,EAAE,KAAK;IACnB,YAAY,EAAE,GAAG;CACT,CAAC;AAEX,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,kBAAkB,EAAE,UAAU,EAAE,OAAO;IACvC,kBAAkB,EAAE,MAAM;IAC1B,cAAc,EAAE,OAAO;IACvB,WAAW,EAAE,CAAC;CACN,CAAC;AAEX,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,WAAW,EAAE,GAAG;IAChB,eAAe,EAAE,EAAE;IACnB,gBAAgB,EAAE,GAAG;CACb,CAAC;AAEX,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE;QACJ,MAAM,EAAE,GAAG;QACX,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,EAAE;KACV;IACD,GAAG,EAAE;QACH,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,GAAG;KACX;IACD,UAAU,EAAE;QACV,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,MAAM;KACd;CACO,CAAC;AAEX,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,IAAI,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE;IAC1C,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IAC1C,UAAU,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE;CAC1C,CAAC"}
@@ -0,0 +1,5 @@
1
+ export * from "./types";
2
+ export * from "./constants";
3
+ export * from "./utils";
4
+ export * from "./lib";
5
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../shared/src/index.ts"],"names":[],"mappings":"AAIA,cAAc,SAAS,CAAC;AAGxB,cAAc,aAAa,CAAC;AAG5B,cAAc,SAAS,CAAC;AAGxB,cAAc,OAAO,CAAC"}
@@ -0,0 +1,11 @@
1
+ // @dealcrawl/shared - Shared types, constants, and utilities
2
+ // ==========================================================
3
+ // Types
4
+ export * from "./types";
5
+ // Constants
6
+ export * from "./constants";
7
+ // Utilities
8
+ export * from "./utils";
9
+ // Lib (Redis, etc.)
10
+ export * from "./lib";
11
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../shared/src/index.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,6DAA6D;AAE7D,QAAQ;AACR,cAAc,SAAS,CAAC;AAExB,YAAY;AACZ,cAAc,aAAa,CAAC;AAE5B,YAAY;AACZ,cAAc,SAAS,CAAC;AAExB,oBAAoB;AACpB,cAAc,OAAO,CAAC"}
@@ -0,0 +1,2 @@
1
+ export * from "./redis";
2
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../shared/src/lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC"}
@@ -0,0 +1,2 @@
1
+ export * from "./redis";
2
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../shared/src/lib/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC"}
@@ -0,0 +1,14 @@
1
+ import Redis from "ioredis";
2
+ /**
3
+ * Shared Redis connection configuration
4
+ * Used by both API and Worker packages
5
+ */
6
+ export declare function createRedisConnection(): Redis;
7
+ export declare function getRedisConnection(): Redis;
8
+ /**
9
+ * Close the singleton Redis connection
10
+ * Should be called during graceful shutdown
11
+ */
12
+ export declare function closeRedisConnection(): Promise<void>;
13
+ export type { Redis };
14
+ //# sourceMappingURL=redis.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"redis.d.ts","sourceRoot":"","sources":["../../../../../shared/src/lib/redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,SAAS,CAAC;AAE5B;;;GAGG;AACH,wBAAgB,qBAAqB,IAAI,KAAK,CA0C7C;AAQD,wBAAgB,kBAAkB,IAAI,KAAK,CAK1C;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1D;AAGD,YAAY,EAAE,KAAK,EAAE,CAAC"}
@@ -0,0 +1,60 @@
1
+ import Redis from "ioredis";
2
+ /**
3
+ * Shared Redis connection configuration
4
+ * Used by both API and Worker packages
5
+ */
6
+ export function createRedisConnection() {
7
+ // Use clean URL without embedded password (SEC-005 fix)
8
+ const redisUrl = process.env.REDIS_URL || "redis://localhost:6379";
9
+ // Derive password exclusively from environment variable
10
+ const redisPassword = process.env.REDIS_PASSWORD;
11
+ // Development fallback with explicit warning
12
+ const isDevelopment = process.env.NODE_ENV === "development" || !process.env.NODE_ENV;
13
+ const devPassword = isDevelopment ? "changeme" : undefined;
14
+ // Warn if using development fallback password
15
+ if (isDevelopment && !redisPassword && devPassword) {
16
+ console.warn("⚠️ [Redis] Using default development password. " +
17
+ "Set REDIS_PASSWORD environment variable for production!");
18
+ }
19
+ // Final password: explicit env var, or dev fallback, or none
20
+ const finalPassword = redisPassword || devPassword;
21
+ const options = {
22
+ maxRetriesPerRequest: null,
23
+ lazyConnect: false,
24
+ enableReadyCheck: true,
25
+ retryStrategy(times) {
26
+ if (times > 3) {
27
+ console.error("Redis connection failed after 3 retries");
28
+ return null;
29
+ }
30
+ return Math.min(times * 100, 2000);
31
+ },
32
+ };
33
+ // Only include password if available
34
+ if (finalPassword) {
35
+ return new Redis(redisUrl, { ...options, password: finalPassword });
36
+ }
37
+ return new Redis(redisUrl, options);
38
+ }
39
+ /**
40
+ * Singleton Redis connection for the current process
41
+ * Lazy-initialized on first access
42
+ */
43
+ let _connection = null;
44
+ export function getRedisConnection() {
45
+ if (!_connection) {
46
+ _connection = createRedisConnection();
47
+ }
48
+ return _connection;
49
+ }
50
+ /**
51
+ * Close the singleton Redis connection
52
+ * Should be called during graceful shutdown
53
+ */
54
+ export async function closeRedisConnection() {
55
+ if (_connection) {
56
+ await _connection.quit();
57
+ _connection = null;
58
+ }
59
+ }
60
+ //# sourceMappingURL=redis.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"redis.js","sourceRoot":"","sources":["../../../../../shared/src/lib/redis.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,SAAS,CAAC;AAE5B;;;GAGG;AACH,MAAM,UAAU,qBAAqB;IACnC,wDAAwD;IACxD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,wBAAwB,CAAC;IAEnE,wDAAwD;IACxD,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAEjD,6CAA6C;IAC7C,MAAM,aAAa,GACjB,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,aAAa,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClE,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IAE3D,8CAA8C;IAC9C,IAAI,aAAa,IAAI,CAAC,aAAa,IAAI,WAAW,EAAE,CAAC;QACnD,OAAO,CAAC,IAAI,CACV,kDAAkD;YAChD,yDAAyD,CAC5D,CAAC;IACJ,CAAC;IAED,6DAA6D;IAC7D,MAAM,aAAa,GAAG,aAAa,IAAI,WAAW,CAAC;IAEnD,MAAM,OAAO,GAAG;QACd,oBAAoB,EAAE,IAAI;QAC1B,WAAW,EAAE,KAAK;QAClB,gBAAgB,EAAE,IAAI;QACtB,aAAa,CAAC,KAAa;YACzB,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBACzD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;QACrC,CAAC;KACF,CAAC;IAEF,qCAAqC;IACrC,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,IAAI,WAAW,GAAiB,IAAI,CAAC;AAErC,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,qBAAqB,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;AACH,CAAC"}
@@ -0,0 +1,94 @@
1
+ /**
2
+ * Available scopes for API keys
3
+ */
4
+ export type ApiKeyScope = "scrape" | "crawl" | "dork" | "extract" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
5
+ /**
6
+ * All available scopes
7
+ */
8
+ export declare const ALL_API_KEY_SCOPES: ApiKeyScope[];
9
+ /**
10
+ * Default scopes for new keys
11
+ */
12
+ export declare const DEFAULT_API_KEY_SCOPES: ApiKeyScope[];
13
+ /**
14
+ * API Key information (without the actual key)
15
+ */
16
+ export interface ApiKeyInfo {
17
+ id: string;
18
+ name: string;
19
+ keyPrefix: string;
20
+ scopes: ApiKeyScope[];
21
+ isActive: boolean;
22
+ expiresAt: string | null;
23
+ lastUsedAt: string | null;
24
+ usageCount: number;
25
+ createdAt: string;
26
+ revokedAt: string | null;
27
+ revokedReason: string | null;
28
+ ipAllowlist: string[] | null;
29
+ }
30
+ /**
31
+ * Newly created API key (includes raw key - only shown once!)
32
+ */
33
+ export interface CreatedApiKey {
34
+ id: string;
35
+ rawKey: string;
36
+ keyPrefix: string;
37
+ name: string;
38
+ scopes: ApiKeyScope[];
39
+ expiresAt: string | null;
40
+ createdAt: string;
41
+ }
42
+ /**
43
+ * Request to create a new API key
44
+ */
45
+ export interface CreateApiKeyRequest {
46
+ name: string;
47
+ scopes?: ApiKeyScope[];
48
+ expiresInDays?: number;
49
+ ipAllowlist?: string[];
50
+ }
51
+ /**
52
+ * Request to rotate an API key
53
+ */
54
+ export interface RotateApiKeyRequest {
55
+ newName?: string;
56
+ }
57
+ /**
58
+ * API key validation result from database
59
+ */
60
+ export interface ApiKeyValidation {
61
+ clientId: string;
62
+ clientName: string;
63
+ clientTier: string;
64
+ keyId: string;
65
+ keyName: string;
66
+ scopes: ApiKeyScope[];
67
+ quotas: Record<string, number>;
68
+ usage: Record<string, number>;
69
+ isValid: boolean;
70
+ errorMessage: string | null;
71
+ }
72
+ /**
73
+ * API key usage statistics
74
+ */
75
+ export interface ApiKeyStats {
76
+ totalRequests: number;
77
+ successfulRequests: number;
78
+ failedRequests: number;
79
+ avgDurationMs: number;
80
+ endpointsUsed: Record<string, number>;
81
+ requestsByDay: Record<string, number>;
82
+ }
83
+ /**
84
+ * Context from validated API key (attached to request)
85
+ */
86
+ export interface ApiKeyContext {
87
+ keyId: string;
88
+ keyName: string;
89
+ scopes: ApiKeyScope[];
90
+ clientId: string;
91
+ clientName: string;
92
+ clientTier: string;
93
+ }
94
+ //# sourceMappingURL=api-key.types.d.ts.map