mcp-scraper 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +5 -0
  2. package/dist/bin/api-server.cjs +15730 -7780
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-stdio-server.cjs +300 -110
  6. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  7. package/dist/bin/mcp-stdio-server.js +1 -1
  8. package/dist/bin/paa-harvest.cjs +1537 -165
  9. package/dist/bin/paa-harvest.cjs.map +1 -1
  10. package/dist/bin/paa-harvest.js +1 -1
  11. package/dist/{chunk-ZBP4RHNW.js → chunk-4743MZHT.js} +298 -106
  12. package/dist/chunk-4743MZHT.js.map +1 -0
  13. package/dist/{chunk-LXZDJJXR.js → chunk-D4CJBZBY.js} +426 -29
  14. package/dist/chunk-D4CJBZBY.js.map +1 -0
  15. package/dist/chunk-HERFK7W6.js +2781 -0
  16. package/dist/chunk-HERFK7W6.js.map +1 -0
  17. package/dist/chunk-Y74EXABN.js +295 -0
  18. package/dist/chunk-Y74EXABN.js.map +1 -0
  19. package/dist/{db-IOYMX64U.js → db-YWCNHBLH.js} +36 -4
  20. package/dist/index.cjs +1660 -237
  21. package/dist/index.cjs.map +1 -1
  22. package/dist/index.d.cts +169 -2
  23. package/dist/index.d.ts +169 -2
  24. package/dist/index.js +120 -69
  25. package/dist/index.js.map +1 -1
  26. package/dist/server-N7Q6H4OR.js +11612 -0
  27. package/dist/server-N7Q6H4OR.js.map +1 -0
  28. package/dist/{worker-3ECJHPRE.js → worker-D4D2YQTA.js} +44 -9
  29. package/dist/worker-D4D2YQTA.js.map +1 -0
  30. package/package.json +17 -5
  31. package/dist/chunk-4API3ZCT.js +0 -1387
  32. package/dist/chunk-4API3ZCT.js.map +0 -1
  33. package/dist/chunk-LXZDJJXR.js.map +0 -1
  34. package/dist/chunk-ZBP4RHNW.js.map +0 -1
  35. package/dist/server-63DR2HE5.js +0 -6062
  36. package/dist/server-63DR2HE5.js.map +0 -1
  37. package/dist/worker-3ECJHPRE.js.map +0 -1
  38. /package/dist/{db-IOYMX64U.js.map → db-YWCNHBLH.js.map} +0 -0
@@ -0,0 +1,295 @@
1
+ import {
2
+ CaptchaError,
3
+ RequestAbortedError
4
+ } from "./chunk-HERFK7W6.js";
5
+ import {
6
+ finishHarvestAttempt,
7
+ startHarvestAttempt
8
+ } from "./chunk-D4CJBZBY.js";
9
+
10
+ // src/api/rates.ts
11
+ var MC_COSTS = {
12
+ serp: 100,
13
+ paa: 100,
14
+ page_scrape: 100,
15
+ url_map: 2e3,
16
+ yt_channel: 50,
17
+ yt_transcription: 200,
18
+ fb_ad: 50,
19
+ maps_place: 2e3,
20
+ maps_review: 50,
21
+ fb_search: 50,
22
+ fb_transcribe: 50
23
+ };
24
+ var MC_PER_CREDIT = 1e3;
25
+ var CREDIT_COST_CATALOG = [
26
+ {
27
+ key: "serp",
28
+ label: "SERP search",
29
+ aliases: ["search_serp", "serp", "google search", "organic results"],
30
+ credits: mcToCredits(MC_COSTS.serp),
31
+ unit: "per search",
32
+ notes: "Returns AI Overview, PAA snippet, videos, forums, and local pack."
33
+ },
34
+ {
35
+ key: "paa",
36
+ label: "PAA harvest",
37
+ aliases: ["harvest_paa", "paa", "people also ask", "questions"],
38
+ credits: mcToCredits(MC_COSTS.paa),
39
+ unit: "per extracted question",
40
+ notes: "Includes full SERP feature extraction. Billed on actual questions returned \u2014 no cap enforced."
41
+ },
42
+ {
43
+ key: "page_scrape",
44
+ label: "Page crawl / extract",
45
+ aliases: ["extract_url", "extract_site", "page scrape", "url scrape", "single page", "site crawl"],
46
+ credits: mcToCredits(MC_COSTS.page_scrape),
47
+ unit: "per page",
48
+ notes: "Applies to both single-URL extraction and per-page site crawls."
49
+ },
50
+ {
51
+ key: "url_map",
52
+ label: "Site URL mapping",
53
+ aliases: ["map_site_urls", "url map", "site map", "crawl urls"],
54
+ credits: mcToCredits(MC_COSTS.url_map),
55
+ unit: "per mapping operation",
56
+ notes: "Flat rate for the full /map-urls call regardless of URL count discovered."
57
+ },
58
+ {
59
+ key: "yt_channel",
60
+ label: "YouTube search / channel harvest",
61
+ aliases: ["youtube_harvest", "youtube search", "youtube channel", "yt_channel"],
62
+ credits: mcToCredits(MC_COSTS.yt_channel),
63
+ unit: "per call"
64
+ },
65
+ {
66
+ key: "yt_transcription",
67
+ label: "YouTube transcription",
68
+ aliases: ["youtube_transcribe", "youtube transcript", "transcription", "yt_transcription"],
69
+ credits: mcToCredits(MC_COSTS.yt_transcription),
70
+ unit: "per minute",
71
+ notes: "A 5-minute hold is taken, then reconciled to actual video duration."
72
+ },
73
+ {
74
+ key: "fb_ad",
75
+ label: "Facebook search / ad lookup",
76
+ aliases: ["facebook_page_intel", "facebook_ad_search", "facebook_ad", "facebook ads", "fb ads"],
77
+ credits: mcToCredits(MC_COSTS.fb_ad),
78
+ unit: "per call"
79
+ },
80
+ {
81
+ key: "maps_place",
82
+ label: "Maps business lookup",
83
+ aliases: ["maps_place_intel", "google maps", "maps place", "place intel"],
84
+ credits: mcToCredits(MC_COSTS.maps_place),
85
+ unit: "per business",
86
+ notes: "Base lookup. Reviews billed separately per card at maps_review rate."
87
+ },
88
+ {
89
+ key: "maps_review",
90
+ label: "Maps review",
91
+ aliases: ["maps_reviews", "google reviews", "review cards", "reviews"],
92
+ credits: mcToCredits(MC_COSTS.maps_review),
93
+ unit: "per review card",
94
+ notes: "Charged after extraction when includeReviews is true."
95
+ },
96
+ {
97
+ key: "fb_search",
98
+ label: "Facebook ad library search",
99
+ aliases: ["facebook_search", "fb_search", "fb ad search"],
100
+ credits: mcToCredits(MC_COSTS.fb_search),
101
+ unit: "per search",
102
+ notes: "Browser automation to search Facebook Ads Library by keyword."
103
+ },
104
+ {
105
+ key: "fb_transcribe",
106
+ label: "Facebook ad transcription",
107
+ aliases: ["facebook_transcribe", "fb_transcribe", "fb ad transcript"],
108
+ credits: mcToCredits(MC_COSTS.fb_transcribe),
109
+ unit: "per call",
110
+ notes: "Whisper transcription of Facebook ad video via fal.ai."
111
+ }
112
+ ];
113
+ var CONCURRENCY_PRICE_ID = "price_1Ta1NRS8aAcsk3TGwsRnYbix";
114
+ var FREE_SIGNUP_MC = 5e5;
115
+ var FREE_MONTHLY_REFRESH_MC = 25e4;
116
+ var BALANCE_PRICE_IDS = {
117
+ "price_1TZx6rS8aAcsk3TGNMc1Vgpo": 11e6,
118
+ "price_1TZx6sS8aAcsk3TGxgqB7khO": 275e5,
119
+ "price_1TZx6tS8aAcsk3TG8PnJqHlG": 605e5,
120
+ "price_1TZx6tS8aAcsk3TGNgRMpy0e": 121e6
121
+ };
122
+ var BALANCE_PACK_LABELS = {
123
+ "price_1TZx6rS8aAcsk3TGNMc1Vgpo": "$10",
124
+ "price_1TZx6sS8aAcsk3TGxgqB7khO": "$25",
125
+ "price_1TZx6tS8aAcsk3TG8PnJqHlG": "$50",
126
+ "price_1TZx6tS8aAcsk3TGNgRMpy0e": "$100"
127
+ };
128
+ function mcToCredits(mc) {
129
+ return mc / MC_PER_CREDIT;
130
+ }
131
+ function insufficientBalanceResponse(balanceMc, requiredMc) {
132
+ const topupUrl = process.env.TOPUP_URL ?? "https://mcpscraper.dev/billing";
133
+ const balanceCredits = mcToCredits(balanceMc);
134
+ const requiredCredits = mcToCredits(requiredMc);
135
+ return {
136
+ error: "insufficient_balance",
137
+ error_code: "insufficient_balance",
138
+ message: `Insufficient credits. Balance: ${balanceCredits} credits. This call requires ${requiredCredits} credits. Top up at ${topupUrl}`,
139
+ balance_credits: balanceCredits,
140
+ required_credits: requiredCredits,
141
+ topup_url: topupUrl
142
+ };
143
+ }
144
+ var LedgerOperation = {
145
+ TOPUP: "topup",
146
+ SIGNUP_GRANT: "signup_grant",
147
+ MONTHLY_REFRESH: "monthly_free_refresh",
148
+ PAA: "paa",
149
+ PAA_REFUND: "paa_refund",
150
+ SERP: "serp",
151
+ REFUND: "refund",
152
+ TRANSCRIPTION: "transcription",
153
+ TRANSCRIPTION_HOLD: "transcription_hold",
154
+ TRANSCRIPTION_REFUND: "transcription_refund",
155
+ YT_CHANNEL: "yt_channel",
156
+ FB_AD: "fb_ad",
157
+ MAPS_PLACE: "maps_place",
158
+ MAPS_REVIEW: "maps_review",
159
+ MAPS_REVIEW_REFUND: "maps_review_refund",
160
+ EXTRACT_SITE: "extract_site",
161
+ EXTRACT_SITE_REFUND: "extract_site_refund",
162
+ EXTRACT_URL: "page_scrape",
163
+ URL_MAP: "url_map",
164
+ EXTRACT_SITE_HOLD: "extract_site_hold",
165
+ YT_CHANNEL_REFUND: "yt_channel_refund",
166
+ FB_AD_REFUND: "fb_ad_refund",
167
+ URL_MAP_REFUND: "url_map_refund",
168
+ FB_SEARCH: "fb_search",
169
+ FB_TRANSCRIBE: "fb_transcribe",
170
+ FB_SEARCH_REFUND: "fb_search_refund",
171
+ FB_TRANSCRIBE_REFUND: "fb_transcribe_refund"
172
+ };
173
+
174
+ // src/api/harvest-problems.ts
175
+ function errorMessage(err) {
176
+ return err instanceof Error ? err.message : String(err);
177
+ }
178
+ function looksLikeTimeout(err, message) {
179
+ if (err instanceof DOMException && (err.name === "TimeoutError" || err.name === "AbortError")) return true;
180
+ return /timeout|timed out|Timeout \d+ms exceeded|deadline/i.test(message);
181
+ }
182
+ function looksLikeCaptcha(message) {
183
+ return /captcha|recaptcha|unusual traffic|google\.com\/sorry|blocked/i.test(message);
184
+ }
185
+ function classifyHarvestProblem(err) {
186
+ const message = errorMessage(err);
187
+ if (err instanceof RequestAbortedError) {
188
+ return {
189
+ error_code: "request_aborted",
190
+ error_type: "request_aborted",
191
+ message,
192
+ retryable: true,
193
+ httpStatus: 408,
194
+ terminalStatus: "cancelled"
195
+ };
196
+ }
197
+ if (err instanceof CaptchaError || looksLikeCaptcha(message)) {
198
+ return {
199
+ error_code: "captcha_exhausted",
200
+ error_type: "captcha",
201
+ message,
202
+ retryable: true,
203
+ httpStatus: 503,
204
+ terminalStatus: "failed"
205
+ };
206
+ }
207
+ if (looksLikeTimeout(err, message)) {
208
+ return {
209
+ error_code: "harvest_timeout",
210
+ error_type: "timeout",
211
+ message,
212
+ retryable: true,
213
+ httpStatus: 504,
214
+ terminalStatus: "failed"
215
+ };
216
+ }
217
+ return {
218
+ error_code: "extraction_failed",
219
+ error_type: "extraction",
220
+ message,
221
+ retryable: false,
222
+ httpStatus: 500,
223
+ terminalStatus: "failed"
224
+ };
225
+ }
226
+ function serializeHarvestProblem(problem) {
227
+ return JSON.stringify({
228
+ error_code: problem.error_code,
229
+ error_type: problem.error_type,
230
+ message: problem.message,
231
+ retryable: problem.retryable
232
+ });
233
+ }
234
+ function harvestProblemResponse(problem) {
235
+ return {
236
+ error: problem.message,
237
+ error_code: problem.error_code,
238
+ error_type: problem.error_type,
239
+ retryable: problem.retryable
240
+ };
241
+ }
242
+
243
+ // src/api/harvest-attempt-events.ts
244
+ function createHarvestAttemptRecorder(jobId, userId) {
245
+ return async (event) => {
246
+ if (event.type === "started") {
247
+ await startHarvestAttempt({
248
+ jobId,
249
+ userId,
250
+ attemptNumber: event.attemptNumber,
251
+ maxAttempts: event.maxAttempts,
252
+ query: event.query,
253
+ location: event.location,
254
+ maxQuestions: event.maxQuestions,
255
+ startedAt: event.startedAt
256
+ });
257
+ return;
258
+ }
259
+ await finishHarvestAttempt({
260
+ jobId,
261
+ attemptNumber: event.attemptNumber,
262
+ outcome: event.outcome,
263
+ kernelSessionId: event.kernelSessionId,
264
+ questionCount: event.questionCount,
265
+ durationMs: event.durationMs,
266
+ error: event.error,
267
+ willRetry: event.willRetry,
268
+ kernelDeleteStarted: event.cleanup.kernelDeleteStarted,
269
+ kernelDeleteSucceeded: event.cleanup.kernelDeleteSucceeded,
270
+ kernelDeleteError: event.cleanup.kernelDeleteError,
271
+ browserCloseSucceeded: event.cleanup.browserCloseSucceeded,
272
+ browserCloseError: event.cleanup.browserCloseError,
273
+ debug: event.debug,
274
+ completedAt: event.completedAt
275
+ });
276
+ };
277
+ }
278
+
279
+ export {
280
+ MC_COSTS,
281
+ MC_PER_CREDIT,
282
+ CREDIT_COST_CATALOG,
283
+ CONCURRENCY_PRICE_ID,
284
+ FREE_SIGNUP_MC,
285
+ FREE_MONTHLY_REFRESH_MC,
286
+ BALANCE_PRICE_IDS,
287
+ BALANCE_PACK_LABELS,
288
+ insufficientBalanceResponse,
289
+ LedgerOperation,
290
+ classifyHarvestProblem,
291
+ serializeHarvestProblem,
292
+ harvestProblemResponse,
293
+ createHarvestAttemptRecorder
294
+ };
295
+ //# sourceMappingURL=chunk-Y74EXABN.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/api/rates.ts","../src/api/harvest-problems.ts","../src/api/harvest-attempt-events.ts"],"sourcesContent":["export const MC_COSTS = {\n serp: 100,\n paa: 100,\n page_scrape: 100,\n url_map: 2_000,\n yt_channel: 50,\n yt_transcription: 200,\n fb_ad: 50,\n maps_place: 2_000,\n maps_review: 50,\n fb_search: 50,\n fb_transcribe: 50,\n} as const\n\nexport type McCostKey = keyof typeof MC_COSTS\n\nexport const MC_PER_CREDIT = 1_000\n\nexport const CREDIT_COST_CATALOG: Array<{\n key: McCostKey\n label: string\n aliases: string[]\n credits: number\n unit: string\n notes?: string\n}> = [\n {\n key: 'serp',\n label: 'SERP search',\n aliases: ['search_serp', 'serp', 'google search', 'organic results'],\n credits: mcToCredits(MC_COSTS.serp),\n unit: 'per search',\n notes: 'Returns AI Overview, PAA snippet, videos, forums, and local pack.',\n },\n {\n key: 'paa',\n label: 'PAA harvest',\n aliases: ['harvest_paa', 'paa', 'people also ask', 'questions'],\n credits: mcToCredits(MC_COSTS.paa),\n unit: 'per extracted question',\n notes: 'Includes full SERP feature extraction. Billed on actual questions returned — no cap enforced.',\n },\n {\n key: 'page_scrape',\n label: 'Page crawl / extract',\n aliases: ['extract_url', 'extract_site', 'page scrape', 'url scrape', 'single page', 'site crawl'],\n credits: mcToCredits(MC_COSTS.page_scrape),\n unit: 'per page',\n notes: 'Applies to both single-URL extraction and per-page site crawls.',\n },\n {\n key: 'url_map',\n label: 'Site URL mapping',\n aliases: ['map_site_urls', 'url map', 'site map', 'crawl urls'],\n credits: mcToCredits(MC_COSTS.url_map),\n unit: 'per mapping operation',\n notes: 'Flat rate for the full /map-urls call regardless of URL count discovered.',\n },\n {\n key: 'yt_channel',\n label: 'YouTube search / channel harvest',\n aliases: ['youtube_harvest', 'youtube search', 'youtube channel', 'yt_channel'],\n credits: mcToCredits(MC_COSTS.yt_channel),\n unit: 'per call',\n },\n {\n key: 'yt_transcription',\n label: 'YouTube transcription',\n aliases: ['youtube_transcribe', 'youtube transcript', 'transcription', 'yt_transcription'],\n credits: mcToCredits(MC_COSTS.yt_transcription),\n unit: 'per minute',\n notes: 'A 5-minute hold is taken, then reconciled to actual video duration.',\n },\n {\n key: 'fb_ad',\n label: 'Facebook search / ad lookup',\n aliases: ['facebook_page_intel', 'facebook_ad_search', 'facebook_ad', 'facebook ads', 'fb ads'],\n credits: mcToCredits(MC_COSTS.fb_ad),\n unit: 'per call',\n },\n {\n key: 'maps_place',\n label: 'Maps business lookup',\n aliases: ['maps_place_intel', 'google maps', 'maps place', 'place intel'],\n credits: mcToCredits(MC_COSTS.maps_place),\n unit: 'per business',\n notes: 'Base lookup. Reviews billed separately per card at maps_review rate.',\n },\n {\n key: 'maps_review',\n label: 'Maps review',\n aliases: ['maps_reviews', 'google reviews', 'review cards', 'reviews'],\n credits: mcToCredits(MC_COSTS.maps_review),\n unit: 'per review card',\n notes: 'Charged after extraction when includeReviews is true.',\n },\n {\n key: 'fb_search',\n label: 'Facebook ad library search',\n aliases: ['facebook_search', 'fb_search', 'fb ad search'],\n credits: mcToCredits(MC_COSTS.fb_search),\n unit: 'per search',\n notes: 'Browser automation to search Facebook Ads Library by keyword.',\n },\n {\n key: 'fb_transcribe',\n label: 'Facebook ad transcription',\n aliases: ['facebook_transcribe', 'fb_transcribe', 'fb ad transcript'],\n credits: mcToCredits(MC_COSTS.fb_transcribe),\n unit: 'per call',\n notes: 'Whisper transcription of Facebook ad video via fal.ai.',\n },\n]\n\nexport const CONCURRENCY_PRICE_ID = 'price_1Ta1NRS8aAcsk3TGwsRnYbix'\n\nexport const FREE_SIGNUP_MC = 500_000\nexport const FREE_MONTHLY_REFRESH_MC = 250_000\n\nexport const BALANCE_PRICE_IDS: Record<string, number> = {\n 'price_1TZx6rS8aAcsk3TGNMc1Vgpo': 11_000_000,\n 'price_1TZx6sS8aAcsk3TGxgqB7khO': 27_500_000,\n 'price_1TZx6tS8aAcsk3TG8PnJqHlG': 60_500_000,\n 'price_1TZx6tS8aAcsk3TGNgRMpy0e': 121_000_000,\n}\n\nexport const BALANCE_PACK_LABELS: Record<string, string> = {\n 'price_1TZx6rS8aAcsk3TGNMc1Vgpo': '$10',\n 'price_1TZx6sS8aAcsk3TGxgqB7khO': '$25',\n 'price_1TZx6tS8aAcsk3TG8PnJqHlG': '$50',\n 'price_1TZx6tS8aAcsk3TGNgRMpy0e': '$100',\n}\n\nexport function mcToCredits(mc: number): number {\n return mc / MC_PER_CREDIT\n}\n\nexport function insufficientBalanceResponse(balanceMc: number, requiredMc: number) {\n const topupUrl = process.env.TOPUP_URL ?? 'https://mcpscraper.dev/billing'\n const balanceCredits = mcToCredits(balanceMc)\n const requiredCredits = mcToCredits(requiredMc)\n return {\n error: 'insufficient_balance',\n error_code: 'insufficient_balance' as const,\n message: `Insufficient credits. Balance: ${balanceCredits} credits. This call requires ${requiredCredits} credits. Top up at ${topupUrl}`,\n balance_credits: balanceCredits,\n required_credits: requiredCredits,\n topup_url: topupUrl,\n }\n}\n\nexport const LedgerOperation = {\n TOPUP: 'topup',\n SIGNUP_GRANT: 'signup_grant',\n MONTHLY_REFRESH: 'monthly_free_refresh',\n PAA: 'paa',\n PAA_REFUND: 'paa_refund',\n SERP: 'serp',\n REFUND: 'refund',\n TRANSCRIPTION: 'transcription',\n TRANSCRIPTION_HOLD: 'transcription_hold',\n TRANSCRIPTION_REFUND: 'transcription_refund',\n YT_CHANNEL: 'yt_channel',\n FB_AD: 'fb_ad',\n MAPS_PLACE: 'maps_place',\n MAPS_REVIEW: 'maps_review',\n MAPS_REVIEW_REFUND: 'maps_review_refund',\n EXTRACT_SITE: 'extract_site',\n EXTRACT_SITE_REFUND: 'extract_site_refund',\n EXTRACT_URL: 'page_scrape',\n URL_MAP: 'url_map',\n EXTRACT_SITE_HOLD: 'extract_site_hold',\n YT_CHANNEL_REFUND: 'yt_channel_refund',\n FB_AD_REFUND: 'fb_ad_refund',\n URL_MAP_REFUND: 'url_map_refund',\n FB_SEARCH: 'fb_search',\n FB_TRANSCRIBE: 'fb_transcribe',\n FB_SEARCH_REFUND: 'fb_search_refund',\n FB_TRANSCRIBE_REFUND: 'fb_transcribe_refund',\n} as const\n\nexport type LedgerOperation = typeof LedgerOperation[keyof typeof LedgerOperation]\n","import { CaptchaError, RequestAbortedError } from '../errors.js'\n\nexport type HarvestProblemCode =\n | 'request_aborted'\n | 'captcha_exhausted'\n | 'harvest_timeout'\n | 'extraction_failed'\n\nexport interface HarvestProblem {\n error_code: HarvestProblemCode\n error_type: string\n message: string\n retryable: boolean\n httpStatus: number\n terminalStatus: 'cancelled' | 'failed'\n}\n\nfunction errorMessage(err: unknown): string {\n return err instanceof Error ? err.message : String(err)\n}\n\nfunction looksLikeTimeout(err: unknown, message: string): boolean {\n if (err instanceof DOMException && (err.name === 'TimeoutError' || err.name === 'AbortError')) return true\n return /timeout|timed out|Timeout \\d+ms exceeded|deadline/i.test(message)\n}\n\nfunction looksLikeCaptcha(message: string): boolean {\n return /captcha|recaptcha|unusual traffic|google\\.com\\/sorry|blocked/i.test(message)\n}\n\nexport function classifyHarvestProblem(err: unknown): HarvestProblem {\n const message = errorMessage(err)\n\n if (err instanceof RequestAbortedError) {\n return {\n error_code: 'request_aborted',\n error_type: 'request_aborted',\n message,\n retryable: true,\n httpStatus: 408,\n terminalStatus: 'cancelled',\n }\n }\n\n if (err instanceof CaptchaError || looksLikeCaptcha(message)) {\n return {\n error_code: 'captcha_exhausted',\n error_type: 'captcha',\n message,\n retryable: true,\n httpStatus: 503,\n terminalStatus: 'failed',\n }\n }\n\n if (looksLikeTimeout(err, message)) {\n return {\n error_code: 'harvest_timeout',\n error_type: 'timeout',\n message,\n retryable: true,\n httpStatus: 504,\n terminalStatus: 'failed',\n }\n }\n\n return {\n error_code: 'extraction_failed',\n error_type: 'extraction',\n message,\n retryable: false,\n httpStatus: 500,\n terminalStatus: 'failed',\n }\n}\n\nexport function serializeHarvestProblem(problem: HarvestProblem): string {\n return JSON.stringify({\n error_code: problem.error_code,\n error_type: problem.error_type,\n message: problem.message,\n retryable: problem.retryable,\n })\n}\n\nexport function harvestProblemResponse(problem: HarvestProblem): {\n error: string\n error_code: HarvestProblemCode\n error_type: string\n retryable: boolean\n} {\n return {\n error: problem.message,\n error_code: problem.error_code,\n error_type: problem.error_type,\n retryable: problem.retryable,\n }\n}\n","import type { HarvestAttemptLogEvent } from '../harvest.js'\nimport { finishHarvestAttempt, startHarvestAttempt } from './db.js'\n\nexport function createHarvestAttemptRecorder(jobId: string, userId: number | bigint) {\n return async (event: HarvestAttemptLogEvent): Promise<void> => {\n if (event.type === 'started') {\n await startHarvestAttempt({\n jobId,\n userId,\n attemptNumber: event.attemptNumber,\n maxAttempts: event.maxAttempts,\n query: event.query,\n location: event.location,\n maxQuestions: event.maxQuestions,\n startedAt: event.startedAt,\n })\n return\n }\n\n await finishHarvestAttempt({\n jobId,\n attemptNumber: event.attemptNumber,\n outcome: event.outcome,\n kernelSessionId: event.kernelSessionId,\n questionCount: event.questionCount,\n durationMs: event.durationMs,\n error: event.error,\n willRetry: event.willRetry,\n kernelDeleteStarted: event.cleanup.kernelDeleteStarted,\n kernelDeleteSucceeded: event.cleanup.kernelDeleteSucceeded,\n kernelDeleteError: event.cleanup.kernelDeleteError,\n browserCloseSucceeded: event.cleanup.browserCloseSucceeded,\n browserCloseError: event.cleanup.browserCloseError,\n debug: event.debug,\n completedAt: event.completedAt,\n })\n }\n}\n"],"mappings":";;;;;;;;;;AAAO,IAAM,WAAW;AAAA,EACtB,MAAmB;AAAA,EACnB,KAAmB;AAAA,EACnB,aAAmB;AAAA,EACnB,SAAiB;AAAA,EACjB,YAAoB;AAAA,EACpB,kBAAmB;AAAA,EACnB,OAAoB;AAAA,EACpB,YAAiB;AAAA,EACjB,aAAoB;AAAA,EACpB,WAAoB;AAAA,EACpB,eAAoB;AACtB;AAIO,IAAM,gBAAgB;AAEtB,IAAM,sBAOR;AAAA,EACH;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,eAAe,QAAQ,iBAAiB,iBAAiB;AAAA,IACnE,SAAS,YAAY,SAAS,IAAI;AAAA,IAClC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,eAAe,OAAO,mBAAmB,WAAW;AAAA,IAC9D,SAAS,YAAY,SAAS,GAAG;AAAA,IACjC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,eAAe,gBAAgB,eAAe,cAAc,eAAe,YAAY;AAAA,IACjG,SAAS,YAAY,SAAS,WAAW;AAAA,IACzC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,iBAAiB,WAAW,YAAY,YAAY;AAAA,IAC9D,SAAS,YAAY,SAAS,OAAO;AAAA,IACrC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,mBAAmB,kBAAkB,mBAAmB,YAAY;AAAA,IAC9E,SAAS,YAAY,SAAS,UAAU;AAAA,IACxC,MAAM;AAAA,EACR;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,sBAAsB,sBAAsB,iBAAiB,kBAAkB;AAAA,IACzF,SAAS,YAAY,SAAS,gBAAgB;AAAA,IAC9C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,uBAAuB,sBAAsB,eAAe,gBAAgB,QAAQ;AAAA,IAC9F,SAAS,YAAY,SAAS,KAAK;AAAA,IACnC,MAAM;AAAA,EACR;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,oBAAoB,eAAe,cAAc,aAAa;AAAA,IACxE,SAAS,YAAY,SAAS,UAAU;AAAA,IACxC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,gBAAgB,kBAAkB,gBAAgB,SAAS;AAAA,IACrE,SAAS,YAAY,SAAS,WAAW;AAAA,IACzC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,mBAAmB,aAAa,cAAc;AAAA,IACxD,SAAS,YAAY,SAAS,SAAS;AAAA,IACvC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,uBAAuB,iBAAiB,kBAAkB;AAAA,IACpE,SAAS,YAAY,SAAS,aAAa;AAAA,IAC3C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AACF;AAEO,IAAM,uBAAuB;AAE7B,IAAM,iBAAiB;AACvB,IAAM,0BAA0B;AAEhC,IAAM,oBAA4C;AAAA,EACvD,kCAAkC;AAAA,EAClC,kCAAkC;AAAA,EAClC,kCAAkC;AAAA,EAClC,kCAAkC;AACpC;AAEO,IAAM,sBAA8C;AAAA,EACzD,kCAAkC;AAAA,EAClC,kCAAkC;AAAA,EAClC,kCAAkC;AAAA,EAClC,kCAAkC;AACpC;AAEO,SAAS,YAAY,IAAoB;AAC9C,SAAO,KAAK;AACd;AAEO,SAAS,4BAA4B,WAAmB,YAAoB;AACjF,QAAM,WAAW,QAAQ,IAAI,aAAa;AAC1C,QAAM,iBAAiB,YAAY,SAAS;AAC5C,QAAM,kBAAkB,YAAY,UAAU;AAC9C,SAAO;AAAA,IACL,OAAO;AAAA,IACP,YAAY;AAAA,IACZ,SAAS,kCAAkC,cAAc,gCAAgC,eAAe,uBAAuB,QAAQ;AAAA,IACvI,iBAAiB;AAAA,IACjB,kBAAkB;AAAA,IAClB,WAAW;AAAA,EACb;AACF;AAEO,IAAM,kBAAkB;AAAA,EAC7B,OAAuB;AAAA,EACvB,cAAuB;AAAA,EACvB,iBAAuB;AAAA,EACvB,KAAuB;AAAA,EACvB,YAAuB;AAAA,EACvB,MAAuB;AAAA,EACvB,QAAuB;AAAA,EACvB,eAAuB;AAAA,EACvB,oBAAuB;AAAA,EACvB,sBAAuB;AAAA,EACvB,YAAuB;AAAA,EACvB,OAAuB;AAAA,EACvB,YAAuB;AAAA,EACvB,aAAuB;AAAA,EACvB,oBAAuB;AAAA,EACvB,cAAuB;AAAA,EACvB,qBAAuB;AAAA,EACvB,aAAuB;AAAA,EACvB,SAAuB;AAAA,EACvB,mBAAuB;AAAA,EACvB,mBAAuB;AAAA,EACvB,cAAuB;AAAA,EACvB,gBAAuB;AAAA,EACvB,WAAuB;AAAA,EACvB,eAAuB;AAAA,EACvB,kBAAuB;AAAA,EACvB,sBAAuB;AACzB;;;AClKA,SAAS,aAAa,KAAsB;AAC1C,SAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AACxD;AAEA,SAAS,iBAAiB,KAAc,SAA0B;AAChE,MAAI,eAAe,iBAAiB,IAAI,SAAS,kBAAkB,IAAI,SAAS,cAAe,QAAO;AACtG,SAAO,qDAAqD,KAAK,OAAO;AAC1E;AAEA,SAAS,iBAAiB,SAA0B;AAClD,SAAO,gEAAgE,KAAK,OAAO;AACrF;AAEO,SAAS,uBAAuB,KAA8B;AACnE,QAAM,UAAU,aAAa,GAAG;AAEhC,MAAI,eAAe,qBAAqB;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,gBAAgB,iBAAiB,OAAO,GAAG;AAC5D,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,iBAAiB,KAAK,OAAO,GAAG;AAClC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ;AAAA,IACA,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,gBAAgB;AAAA,EAClB;AACF;AAEO,SAAS,wBAAwB,SAAiC;AACvE,SAAO,KAAK,UAAU;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,SAAS,QAAQ;AAAA,IACjB,WAAW,QAAQ;AAAA,EACrB,CAAC;AACH;AAEO,SAAS,uBAAuB,SAKrC;AACA,SAAO;AAAA,IACL,OAAO,QAAQ;AAAA,IACf,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,WAAW,QAAQ;AAAA,EACrB;AACF;;;AC9FO,SAAS,6BAA6B,OAAe,QAAyB;AACnF,SAAO,OAAO,UAAiD;AAC7D,QAAI,MAAM,SAAS,WAAW;AAC5B,YAAM,oBAAoB;AAAA,QACxB;AAAA,QACA;AAAA,QACA,eAAe,MAAM;AAAA,QACrB,aAAa,MAAM;AAAA,QACnB,OAAO,MAAM;AAAA,QACb,UAAU,MAAM;AAAA,QAChB,cAAc,MAAM;AAAA,QACpB,WAAW,MAAM;AAAA,MACnB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA,eAAe,MAAM;AAAA,MACrB,SAAS,MAAM;AAAA,MACf,iBAAiB,MAAM;AAAA,MACvB,eAAe,MAAM;AAAA,MACrB,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,MACb,WAAW,MAAM;AAAA,MACjB,qBAAqB,MAAM,QAAQ;AAAA,MACnC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,OAAO,MAAM;AAAA,MACb,aAAa,MAAM;AAAA,IACrB,CAAC;AAAA,EACH;AACF;","names":[]}
@@ -1,15 +1,19 @@
1
1
  import {
2
2
  SiteAuditJobRowSchema,
3
3
  SiteAuditPhaseLogRowSchema,
4
+ cancelJob,
5
+ checkRateLimit,
6
+ claimMonthlyFreeRefresh,
4
7
  claimPendingJob,
5
8
  claimPendingKpoJob,
6
9
  completeJob,
7
10
  completeKpoJob,
11
+ consumePasswordResetToken,
8
12
  countActiveJobsForUser,
9
13
  countActiveUsers,
10
- countJobsLast7Days,
11
14
  createJob,
12
15
  createKpoJob,
16
+ createPasswordResetToken,
13
17
  createRunningJob,
14
18
  createUser,
15
19
  creditMc,
@@ -17,6 +21,7 @@ import {
17
21
  debitMc,
18
22
  failJob,
19
23
  failKpoJob,
24
+ finishHarvestAttempt,
20
25
  generateApiKey,
21
26
  getDb,
22
27
  getJob,
@@ -29,30 +34,45 @@ import {
29
34
  getUserByStripeCustomerId,
30
35
  getUserStats,
31
36
  hashPassword,
37
+ ledgerExistsForOperation,
38
+ ledgerExistsForStripePI,
39
+ listHarvestAttempts,
32
40
  listJobs,
33
41
  listKpoJobs,
42
+ listRequestEvents,
34
43
  listUsers,
35
44
  logKpoPhaseComplete,
45
+ logRequestEvent,
36
46
  migrate,
47
+ reconcileBalanceMc,
48
+ recordStripeEvent,
37
49
  revokeApiKey,
38
50
  rotateApiKey,
51
+ setConcurrencySubId,
52
+ setExtraConcurrencySlots,
39
53
  setPassword,
40
54
  setStripeCustomerId,
55
+ startHarvestAttempt,
56
+ stripeEventAlreadyProcessed,
41
57
  updateKpoJobState,
42
58
  verifyPassword
43
- } from "./chunk-LXZDJJXR.js";
59
+ } from "./chunk-D4CJBZBY.js";
44
60
  export {
45
61
  SiteAuditJobRowSchema,
46
62
  SiteAuditPhaseLogRowSchema,
63
+ cancelJob,
64
+ checkRateLimit,
65
+ claimMonthlyFreeRefresh,
47
66
  claimPendingJob,
48
67
  claimPendingKpoJob,
49
68
  completeJob,
50
69
  completeKpoJob,
70
+ consumePasswordResetToken,
51
71
  countActiveJobsForUser,
52
72
  countActiveUsers,
53
- countJobsLast7Days,
54
73
  createJob,
55
74
  createKpoJob,
75
+ createPasswordResetToken,
56
76
  createRunningJob,
57
77
  createUser,
58
78
  creditMc,
@@ -60,6 +80,7 @@ export {
60
80
  debitMc,
61
81
  failJob,
62
82
  failKpoJob,
83
+ finishHarvestAttempt,
63
84
  generateApiKey,
64
85
  getDb,
65
86
  getJob,
@@ -72,16 +93,27 @@ export {
72
93
  getUserByStripeCustomerId,
73
94
  getUserStats,
74
95
  hashPassword,
96
+ ledgerExistsForOperation,
97
+ ledgerExistsForStripePI,
98
+ listHarvestAttempts,
75
99
  listJobs,
76
100
  listKpoJobs,
101
+ listRequestEvents,
77
102
  listUsers,
78
103
  logKpoPhaseComplete,
104
+ logRequestEvent,
79
105
  migrate,
106
+ reconcileBalanceMc,
107
+ recordStripeEvent,
80
108
  revokeApiKey,
81
109
  rotateApiKey,
110
+ setConcurrencySubId,
111
+ setExtraConcurrencySlots,
82
112
  setPassword,
83
113
  setStripeCustomerId,
114
+ startHarvestAttempt,
115
+ stripeEventAlreadyProcessed,
84
116
  updateKpoJobState,
85
117
  verifyPassword
86
118
  };
87
- //# sourceMappingURL=db-IOYMX64U.js.map
119
+ //# sourceMappingURL=db-YWCNHBLH.js.map