mcp-scraper 0.88.2 → 0.89.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +36 -15
  2. package/README.md +6 -5
  3. package/THIRD_PARTY_NOTICES.html +203 -0
  4. package/dist/analytics-repository-2BMT5JNE.js +1 -0
  5. package/dist/bin/api-server.js +2 -41
  6. package/dist/bin/mcp-scraper-cli.js +39 -756
  7. package/dist/bin/mcp-scraper-core.js +1 -60
  8. package/dist/bin/mcp-scraper-install.js +2 -25
  9. package/dist/bin/mcp-stdio-server.js +1 -19
  10. package/dist/bin/paa-harvest.js +1 -41
  11. package/dist/chunk-3GP5CYZX.js +1 -0
  12. package/dist/chunk-4AI7DOS7.js +59 -0
  13. package/dist/chunk-4FROKQJN.js +1 -0
  14. package/dist/chunk-7YGVI5J4.js +21710 -0
  15. package/dist/chunk-CCYWSJNG.js +16 -0
  16. package/dist/chunk-CFI6CXIV.js +182 -0
  17. package/dist/chunk-E2WRWV3A.js +1 -0
  18. package/dist/chunk-GMWKPIYX.js +1172 -0
  19. package/dist/chunk-HDPYG3XV.js +102 -0
  20. package/dist/chunk-HE45FFBU.js +1 -0
  21. package/dist/chunk-HUV2WTRW.js +1 -0
  22. package/dist/chunk-KJQXUZ4Y.js +4 -0
  23. package/dist/chunk-L4CGLFPU.js +4 -0
  24. package/dist/chunk-M22MM4N4.js +84 -0
  25. package/dist/chunk-MASR22K4.js +73 -0
  26. package/dist/chunk-MZN4U5BL.js +1 -0
  27. package/dist/chunk-PUHFVA7P.js +1280 -0
  28. package/dist/chunk-QPWPR5XG.js +10 -0
  29. package/dist/chunk-TMB56NCA.js +1 -0
  30. package/dist/chunk-TXENITMS.js +20 -0
  31. package/dist/chunk-W2BVJ7S2.js +13 -0
  32. package/dist/chunk-WO3N5FH2.js +5 -0
  33. package/dist/chunk-WSCGYRWA.js +2595 -0
  34. package/dist/chunk-X54CQLK2.js +1 -0
  35. package/dist/chunk-XLWNEVUZ.js +27 -0
  36. package/dist/chunk-XPZVJIZ2.js +100 -0
  37. package/dist/chunk-YQZGZBB4.js +1 -0
  38. package/dist/chunk-Z2QGQJS2.js +1 -0
  39. package/dist/db-F2MX63GI.js +1 -0
  40. package/dist/extract-bundle-SNUIHM3J.js +26 -0
  41. package/dist/gmail-service-BZ3H75XC.js +1 -0
  42. package/dist/index.cjs +21750 -6045
  43. package/dist/index.d.cts +14 -14
  44. package/dist/index.d.ts +14 -14
  45. package/dist/index.js +18 -315
  46. package/dist/lead-list-enrichment-repository-S2H3U7T7.js +1 -0
  47. package/dist/location-data-repository-OTWHWMV6.js +1 -0
  48. package/dist/server-RFR2A5UJ.js +7303 -0
  49. package/dist/site-extract-repository-SE776XDC.js +1 -0
  50. package/dist/worker-XUDSM3AL.js +1 -0
  51. package/package.json +17 -124
  52. package/dist/analytics-repository-GGJJCVVP.js +0 -194
  53. package/dist/chunk-4QMUF6XM.js +0 -1013
  54. package/dist/chunk-6HAV7LCE.js +0 -265
  55. package/dist/chunk-ABF2CGOZ.js +0 -113
  56. package/dist/chunk-C5Z4OFKW.js +0 -404
  57. package/dist/chunk-DNM65UCK.js +0 -299
  58. package/dist/chunk-EQGTEHLZ.js +0 -592
  59. package/dist/chunk-F5GQJWZU.js +0 -732
  60. package/dist/chunk-GGZEC22A.js +0 -215
  61. package/dist/chunk-GXBZXWXB.js +0 -184
  62. package/dist/chunk-IHXAXYIS.js +0 -843
  63. package/dist/chunk-K3Z5AQYE.js +0 -683
  64. package/dist/chunk-K45K75OF.js +0 -6
  65. package/dist/chunk-LFW2FRPJ.js +0 -224
  66. package/dist/chunk-MZDNZQWT.js +0 -2078
  67. package/dist/chunk-NVUKO5NN.js +0 -256
  68. package/dist/chunk-OM7HVEJ3.js +0 -26
  69. package/dist/chunk-OPQIGAFB.js +0 -286
  70. package/dist/chunk-OZJMVCDK.js +0 -16
  71. package/dist/chunk-P7FWOMU7.js +0 -505
  72. package/dist/chunk-PGJQDMC2.js +0 -383
  73. package/dist/chunk-PKZS6SHW.js +0 -33139
  74. package/dist/chunk-RJ7JVYKU.js +0 -68
  75. package/dist/chunk-S24LFPL7.js +0 -5262
  76. package/dist/chunk-T3MZISOF.js +0 -240
  77. package/dist/chunk-UZPTGUDV.js +0 -1915
  78. package/dist/chunk-X623GTBV.js +0 -8290
  79. package/dist/chunk-YXNDOQXN.js +0 -4018
  80. package/dist/db-Z34LPZNR.js +0 -284
  81. package/dist/extract-bundle-565SBZCR.js +0 -1003
  82. package/dist/gmail-service-E6ALS7JG.js +0 -25
  83. package/dist/lead-list-enrichment-repository-36RPVV6N.js +0 -67
  84. package/dist/location-data-repository-WPRG62GE.js +0 -34
  85. package/dist/server-SQZ3A7SY.js +0 -86606
  86. package/dist/site-extract-repository-VYFZASPU.js +0 -69
  87. package/dist/worker-LDCAULWL.js +0 -146
@@ -1,592 +0,0 @@
1
- import {
2
- getDb
3
- } from "./chunk-YXNDOQXN.js";
4
-
5
- // src/api/lead-list-enrichment-repository.ts
6
- import { createHash, randomUUID } from "crypto";
7
- var EVENT_NAME = "mcp-scraper/lead-list-enrichment.requested";
8
- var TERMINAL_JOB_STATUSES = /* @__PURE__ */ new Set(["complete", "partial", "empty", "failed", "cancelled"]);
9
- var TERMINAL_ROW_STATUSES = /* @__PURE__ */ new Set(["complete", "failed", "cancelled"]);
10
- var LEAD_LIST_MAX_DISPATCH_ATTEMPTS = 10;
11
- var LEAD_LIST_START_DEADLINE_MINUTES = 6 * 60;
12
- var LEAD_LIST_RUNNING_STALE_MINUTES = 60;
13
- var schemaDb = null;
14
- var schemaPromise = null;
15
- function requiredBoundedString(value, field, max) {
16
- const trimmed = value.trim();
17
- if (!trimmed || trimmed.length > max) throw new Error(`${field} is required and must be ${max} characters or fewer`);
18
- return trimmed;
19
- }
20
- function nonNegativeSafeInteger(value, field) {
21
- if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${field} must be a non-negative safe integer`);
22
- return value;
23
- }
24
- function parseJson(value) {
25
- if (value == null) return null;
26
- try {
27
- return JSON.parse(String(value));
28
- } catch {
29
- return null;
30
- }
31
- }
32
- function stableCanonicalJson(value) {
33
- const seen = /* @__PURE__ */ new WeakSet();
34
- const normalize = (input) => {
35
- if (input === null || typeof input === "string" || typeof input === "boolean") return input;
36
- if (typeof input === "number") {
37
- if (!Number.isFinite(input)) throw new Error("canonical fingerprint input must contain finite numbers");
38
- return Object.is(input, -0) ? 0 : input;
39
- }
40
- if (Array.isArray(input)) return input.map(normalize);
41
- if (typeof input === "object") {
42
- if (seen.has(input)) throw new Error("canonical fingerprint input must not be cyclic");
43
- seen.add(input);
44
- const out = {};
45
- for (const key of Object.keys(input).sort()) {
46
- const child = input[key];
47
- if (child !== void 0) out[key] = normalize(child);
48
- }
49
- seen.delete(input);
50
- return out;
51
- }
52
- throw new Error(`canonical fingerprint input contains unsupported ${typeof input}`);
53
- };
54
- return JSON.stringify(normalize(value));
55
- }
56
- function leadListEnrichmentFingerprint(input) {
57
- return createHash("sha256").update(stableCanonicalJson(input)).digest("hex");
58
- }
59
- function leadListRowInputDigest(row) {
60
- return createHash("sha256").update(stableCanonicalJson(row)).digest("hex");
61
- }
62
- function emptyUsage() {
63
- return { mapsAttempts: 0, pageAttempts: 0, pageSuccesses: 0, serpSearches: 0 };
64
- }
65
- function rowToJob(row) {
66
- const input = parseJson(row.input_json);
67
- if (!input) throw new Error(`lead list enrichment job ${String(row.id ?? "")} has invalid durable input`);
68
- return {
69
- id: String(row.id),
70
- userId: Number(row.user_id),
71
- idempotencyKey: String(row.idempotency_key),
72
- requestFingerprint: String(row.request_fingerprint),
73
- input,
74
- status: String(row.status),
75
- totalRows: Number(row.total_rows ?? 0),
76
- doneRows: Number(row.done_rows ?? 0),
77
- failedRows: Number(row.failed_rows ?? 0),
78
- cancelledRows: Number(row.cancelled_rows ?? 0),
79
- usage: {
80
- mapsAttempts: Number(row.maps_attempts ?? 0),
81
- pageAttempts: Number(row.page_attempts ?? 0),
82
- pageSuccesses: Number(row.page_successes ?? 0),
83
- serpSearches: Number(row.serp_searches ?? 0)
84
- },
85
- concurrencyLockId: row.concurrency_lock_id == null ? null : String(row.concurrency_lock_id),
86
- debitKey: String(row.debit_key),
87
- holdMc: Number(row.hold_mc ?? 0),
88
- billedMc: row.billed_mc == null ? null : Number(row.billed_mc),
89
- result: parseJson(row.result_json),
90
- csvArtifact: parseJson(row.csv_artifact_json),
91
- xlsxArtifact: parseJson(row.xlsx_artifact_json),
92
- warnings: parseJson(row.warnings_json) ?? [],
93
- cancelRequestedAt: row.cancel_requested_at == null ? null : String(row.cancel_requested_at),
94
- errorCode: row.error_code == null ? null : String(row.error_code),
95
- errorMessage: row.error_message == null ? null : String(row.error_message),
96
- retryable: row.retryable == null ? null : Number(row.retryable) === 1,
97
- createdAt: String(row.created_at ?? ""),
98
- startedAt: row.started_at == null ? null : String(row.started_at),
99
- completedAt: row.completed_at == null ? null : String(row.completed_at),
100
- updatedAt: String(row.updated_at ?? "")
101
- };
102
- }
103
- function rowToCheckpoint(row) {
104
- return {
105
- jobId: String(row.job_id),
106
- ordinal: Number(row.ordinal),
107
- inputDigest: String(row.input_digest),
108
- status: String(row.status),
109
- result: parseJson(row.result_json),
110
- usage: {
111
- mapsAttempts: Number(row.maps_attempts ?? 0),
112
- pageAttempts: Number(row.page_attempts ?? 0),
113
- pageSuccesses: Number(row.page_successes ?? 0),
114
- serpSearches: Number(row.serp_searches ?? 0)
115
- },
116
- errorCode: row.error_code == null ? null : String(row.error_code),
117
- errorMessage: row.error_message == null ? null : String(row.error_message),
118
- retryable: Number(row.retryable ?? 0) === 1,
119
- durationMs: Number(row.duration_ms ?? 0),
120
- startedAt: row.started_at == null ? null : String(row.started_at),
121
- completedAt: row.completed_at == null ? null : String(row.completed_at),
122
- updatedAt: String(row.updated_at ?? "")
123
- };
124
- }
125
- function rowToOutbox(row) {
126
- return {
127
- id: String(row.id),
128
- jobId: String(row.job_id),
129
- eventName: EVENT_NAME,
130
- status: String(row.status),
131
- attempts: Number(row.attempts ?? 0),
132
- claimToken: row.locked_by == null ? null : String(row.locked_by),
133
- lockedUntil: row.locked_until == null ? null : String(row.locked_until),
134
- nextAttemptAt: row.next_attempt_at == null ? null : String(row.next_attempt_at),
135
- lastError: row.last_error == null ? null : String(row.last_error),
136
- createdAt: String(row.created_at ?? ""),
137
- dispatchedAt: row.dispatched_at == null ? null : String(row.dispatched_at),
138
- updatedAt: String(row.updated_at ?? "")
139
- };
140
- }
141
- async function ensureLeadListEnrichmentRepositorySchema() {
142
- const db = getDb();
143
- if (schemaDb !== db) {
144
- schemaDb = db;
145
- schemaPromise = null;
146
- }
147
- if (!schemaPromise) schemaPromise = (async () => {
148
- await db.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_jobs (
149
- id TEXT PRIMARY KEY, user_id INTEGER NOT NULL REFERENCES users(id), idempotency_key TEXT NOT NULL,
150
- request_fingerprint TEXT NOT NULL, input_json TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'initializing',
151
- total_rows INTEGER NOT NULL DEFAULT 0, done_rows INTEGER NOT NULL DEFAULT 0,
152
- failed_rows INTEGER NOT NULL DEFAULT 0, cancelled_rows INTEGER NOT NULL DEFAULT 0,
153
- maps_attempts INTEGER NOT NULL DEFAULT 0, page_attempts INTEGER NOT NULL DEFAULT 0,
154
- page_successes INTEGER NOT NULL DEFAULT 0, serp_searches INTEGER NOT NULL DEFAULT 0,
155
- concurrency_lock_id TEXT, debit_key TEXT NOT NULL,
156
- hold_mc INTEGER NOT NULL DEFAULT 0, billed_mc INTEGER, result_json TEXT,
157
- csv_artifact_json TEXT, xlsx_artifact_json TEXT, warnings_json TEXT NOT NULL DEFAULT '[]',
158
- cancel_requested_at TEXT, error_code TEXT, error_message TEXT, retryable INTEGER,
159
- created_at TEXT NOT NULL DEFAULT (datetime('now')), started_at TEXT, completed_at TEXT,
160
- updated_at TEXT NOT NULL DEFAULT (datetime('now')), UNIQUE(user_id, idempotency_key)
161
- )`);
162
- try {
163
- await db.execute("ALTER TABLE lead_list_enrichment_jobs ADD COLUMN serp_searches INTEGER NOT NULL DEFAULT 0");
164
- } catch {
165
- }
166
- await db.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_jobs_user_created ON lead_list_enrichment_jobs(user_id, created_at DESC)");
167
- await db.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_jobs_status ON lead_list_enrichment_jobs(status, updated_at)");
168
- await db.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_rows (
169
- job_id TEXT NOT NULL REFERENCES lead_list_enrichment_jobs(id) ON DELETE CASCADE,
170
- ordinal INTEGER NOT NULL, input_digest TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending',
171
- result_json TEXT, maps_attempts INTEGER NOT NULL DEFAULT 0, page_attempts INTEGER NOT NULL DEFAULT 0,
172
- page_successes INTEGER NOT NULL DEFAULT 0, serp_searches INTEGER NOT NULL DEFAULT 0,
173
- error_code TEXT, error_message TEXT,
174
- retryable INTEGER NOT NULL DEFAULT 0, duration_ms INTEGER NOT NULL DEFAULT 0,
175
- started_at TEXT, completed_at TEXT, updated_at TEXT NOT NULL DEFAULT (datetime('now')),
176
- PRIMARY KEY(job_id, ordinal)
177
- )`);
178
- try {
179
- await db.execute("ALTER TABLE lead_list_enrichment_rows ADD COLUMN serp_searches INTEGER NOT NULL DEFAULT 0");
180
- } catch {
181
- }
182
- await db.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_rows_job_status ON lead_list_enrichment_rows(job_id, status, ordinal)");
183
- await db.execute(`CREATE TABLE IF NOT EXISTS lead_list_enrichment_outbox (
184
- id TEXT PRIMARY KEY, job_id TEXT NOT NULL UNIQUE REFERENCES lead_list_enrichment_jobs(id) ON DELETE CASCADE,
185
- event_name TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'held', attempts INTEGER NOT NULL DEFAULT 0,
186
- locked_by TEXT, locked_until TEXT, next_attempt_at TEXT, last_error TEXT,
187
- created_at TEXT NOT NULL DEFAULT (datetime('now')), dispatched_at TEXT,
188
- updated_at TEXT NOT NULL DEFAULT (datetime('now'))
189
- )`);
190
- await db.execute("CREATE INDEX IF NOT EXISTS lead_list_enrichment_outbox_dispatch ON lead_list_enrichment_outbox(status, next_attempt_at, locked_until, created_at)");
191
- })();
192
- return schemaPromise;
193
- }
194
- async function getLeadListEnrichmentJob(id, userId) {
195
- await ensureLeadListEnrichmentRepositorySchema();
196
- const result = userId == null ? await getDb().execute({ sql: "SELECT * FROM lead_list_enrichment_jobs WHERE id = ? LIMIT 1", args: [id] }) : await getDb().execute({ sql: "SELECT * FROM lead_list_enrichment_jobs WHERE id = ? AND user_id = ? LIMIT 1", args: [id, userId] });
197
- return result.rows[0] ? rowToJob(result.rows[0]) : null;
198
- }
199
- async function getLeadListEnrichmentJobByIdempotencyKey(userId, idempotencyKey) {
200
- await ensureLeadListEnrichmentRepositorySchema();
201
- const result = await getDb().execute({
202
- sql: "SELECT * FROM lead_list_enrichment_jobs WHERE user_id = ? AND idempotency_key = ? LIMIT 1",
203
- args: [userId, idempotencyKey]
204
- });
205
- return result.rows[0] ? rowToJob(result.rows[0]) : null;
206
- }
207
- async function createOrGetLeadListEnrichmentJob(input) {
208
- await ensureLeadListEnrichmentRepositorySchema();
209
- const id = requiredBoundedString(input.id, "job id", 200);
210
- const idempotencyKey = requiredBoundedString(input.idempotencyKey, "idempotency key", 200);
211
- const requestFingerprint = input.requestFingerprint ?? leadListEnrichmentFingerprint(input.input);
212
- requiredBoundedString(requestFingerprint, "request fingerprint", 200);
213
- const debitKey = requiredBoundedString(input.debitKey, "debit key", 500);
214
- const holdMc = nonNegativeSafeInteger(input.holdMc, "holdMc");
215
- if (input.input.rows.length < 1 || input.input.rows.length > 250) throw new Error("lead list job must contain 1 to 250 rows");
216
- const commands = [{
217
- sql: `INSERT OR IGNORE INTO lead_list_enrichment_jobs (
218
- id,user_id,idempotency_key,request_fingerprint,input_json,total_rows,concurrency_lock_id,debit_key,hold_mc
219
- ) VALUES (?,?,?,?,?,?,?,?,?)`,
220
- args: [
221
- id,
222
- input.userId,
223
- idempotencyKey,
224
- requestFingerprint,
225
- JSON.stringify(input.input),
226
- input.input.rows.length,
227
- input.concurrencyLockId ?? null,
228
- debitKey,
229
- holdMc
230
- ]
231
- }, {
232
- sql: `INSERT OR IGNORE INTO lead_list_enrichment_outbox(id,job_id,event_name,status)
233
- SELECT 'lead-list-enrichment:' || id,id,?,'held' FROM lead_list_enrichment_jobs
234
- WHERE user_id=? AND idempotency_key=? AND request_fingerprint=?`,
235
- args: [EVENT_NAME, input.userId, idempotencyKey, requestFingerprint]
236
- }];
237
- input.input.rows.forEach((row, ordinal) => commands.push({
238
- sql: `INSERT OR IGNORE INTO lead_list_enrichment_rows(job_id,ordinal,input_digest)
239
- SELECT id,?,? FROM lead_list_enrichment_jobs
240
- WHERE user_id=? AND idempotency_key=? AND request_fingerprint=?`,
241
- args: [ordinal, leadListRowInputDigest(row), input.userId, idempotencyKey, requestFingerprint]
242
- }));
243
- const results = await getDb().batch(commands, "write");
244
- const job = await getLeadListEnrichmentJobByIdempotencyKey(input.userId, idempotencyKey);
245
- if (!job) throw new Error("lead list enrichment job could not be created");
246
- return { job, created: Number(results[0]?.rowsAffected ?? 0) > 0, conflict: job.requestFingerprint !== requestFingerprint };
247
- }
248
- async function setInitializingLeadListEnrichmentConcurrencyLock(jobId, concurrencyLockId) {
249
- await ensureLeadListEnrichmentRepositorySchema();
250
- await getDb().execute({
251
- sql: `UPDATE lead_list_enrichment_jobs SET concurrency_lock_id=?, updated_at=datetime('now') WHERE id=? AND status='initializing'`,
252
- args: [concurrencyLockId, jobId]
253
- });
254
- return getLeadListEnrichmentJob(jobId);
255
- }
256
- async function markLeadListEnrichmentJobQueued(jobId) {
257
- await ensureLeadListEnrichmentRepositorySchema();
258
- await getDb().batch([{
259
- sql: `UPDATE lead_list_enrichment_jobs SET status='queued',updated_at=datetime('now') WHERE id=? AND status='initializing'`,
260
- args: [jobId]
261
- }, {
262
- sql: `UPDATE lead_list_enrichment_outbox SET status='pending',next_attempt_at=NULL,updated_at=datetime('now')
263
- WHERE job_id=? AND status='held' AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='queued')`,
264
- args: [jobId, jobId]
265
- }], "write");
266
- return getLeadListEnrichmentJob(jobId);
267
- }
268
- async function markLeadListEnrichmentJobRunning(jobId) {
269
- await ensureLeadListEnrichmentRepositorySchema();
270
- await getDb().execute({
271
- sql: `UPDATE lead_list_enrichment_jobs SET status='running',started_at=COALESCE(started_at,datetime('now')),
272
- updated_at=datetime('now') WHERE id=? AND status IN ('queued','running') AND cancel_requested_at IS NULL`,
273
- args: [jobId]
274
- });
275
- const job = await getLeadListEnrichmentJob(jobId);
276
- return job?.status === "running" ? job : null;
277
- }
278
- async function claimLeadListEnrichmentRow(jobId, ordinal) {
279
- await ensureLeadListEnrichmentRepositorySchema();
280
- nonNegativeSafeInteger(ordinal, "row ordinal");
281
- await getDb().execute({
282
- sql: `UPDATE lead_list_enrichment_rows SET status='running',started_at=COALESCE(started_at,datetime('now')),
283
- updated_at=datetime('now') WHERE job_id=? AND ordinal=? AND status IN ('pending','running')
284
- AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='running' AND cancel_requested_at IS NULL)`,
285
- args: [jobId, ordinal, jobId]
286
- });
287
- return getLeadListEnrichmentRow(jobId, ordinal);
288
- }
289
- async function getLeadListEnrichmentRow(jobId, ordinal) {
290
- await ensureLeadListEnrichmentRepositorySchema();
291
- const result = await getDb().execute({ sql: "SELECT * FROM lead_list_enrichment_rows WHERE job_id=? AND ordinal=? LIMIT 1", args: [jobId, ordinal] });
292
- return result.rows[0] ? rowToCheckpoint(result.rows[0]) : null;
293
- }
294
- async function listLeadListEnrichmentRows(jobId) {
295
- await ensureLeadListEnrichmentRepositorySchema();
296
- const result = await getDb().execute({ sql: "SELECT * FROM lead_list_enrichment_rows WHERE job_id=? ORDER BY ordinal ASC", args: [jobId] });
297
- return result.rows.map((row) => rowToCheckpoint(row));
298
- }
299
- async function saveLeadListEnrichmentRowResult(input) {
300
- await ensureLeadListEnrichmentRepositorySchema();
301
- const usage = input.result.usage ?? emptyUsage();
302
- const rowStatus = input.result.rowStatus === "failed" ? "failed" : "complete";
303
- const existing = await getLeadListEnrichmentRow(input.jobId, input.ordinal);
304
- if (existing && TERMINAL_ROW_STATUSES.has(existing.status)) return existing;
305
- await getDb().batch([{
306
- sql: `UPDATE lead_list_enrichment_rows SET status=?,result_json=?,maps_attempts=?,page_attempts=?,page_successes=?,serp_searches=?,
307
- error_code=?,error_message=?,retryable=?,duration_ms=?,completed_at=datetime('now'),updated_at=datetime('now')
308
- WHERE job_id=? AND ordinal=? AND status IN ('pending','running')
309
- AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND status='running')`,
310
- args: [
311
- rowStatus,
312
- JSON.stringify(input.result),
313
- nonNegativeSafeInteger(usage.mapsAttempts, "mapsAttempts"),
314
- nonNegativeSafeInteger(usage.pageAttempts, "pageAttempts"),
315
- nonNegativeSafeInteger(usage.pageSuccesses, "pageSuccesses"),
316
- nonNegativeSafeInteger(usage.serpSearches, "serpSearches"),
317
- input.result.error?.code ?? null,
318
- input.result.error?.message ?? null,
319
- input.result.error ? 1 : 0,
320
- nonNegativeSafeInteger(input.durationMs ?? 0, "durationMs"),
321
- input.jobId,
322
- input.ordinal,
323
- input.jobId
324
- ]
325
- }, {
326
- sql: `UPDATE lead_list_enrichment_jobs SET
327
- done_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status IN ('complete','failed','cancelled')),
328
- failed_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='failed'),
329
- cancelled_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='cancelled'),
330
- maps_attempts=(SELECT COALESCE(SUM(maps_attempts),0) FROM lead_list_enrichment_rows WHERE job_id=?),
331
- page_attempts=(SELECT COALESCE(SUM(page_attempts),0) FROM lead_list_enrichment_rows WHERE job_id=?),
332
- page_successes=(SELECT COALESCE(SUM(page_successes),0) FROM lead_list_enrichment_rows WHERE job_id=?),
333
- serp_searches=(SELECT COALESCE(SUM(serp_searches),0) FROM lead_list_enrichment_rows WHERE job_id=?),
334
- updated_at=datetime('now') WHERE id=? AND status='running'`,
335
- args: [input.jobId, input.jobId, input.jobId, input.jobId, input.jobId, input.jobId, input.jobId, input.jobId]
336
- }], "write");
337
- return getLeadListEnrichmentRow(input.jobId, input.ordinal);
338
- }
339
- async function requestLeadListEnrichmentCancellation(jobId, userId) {
340
- await ensureLeadListEnrichmentRepositorySchema();
341
- const ownerClause = userId == null ? "" : " AND user_id=?";
342
- const args = [jobId];
343
- if (userId != null) args.push(userId);
344
- await getDb().batch([{
345
- sql: `UPDATE lead_list_enrichment_jobs SET cancel_requested_at=COALESCE(cancel_requested_at,datetime('now')),
346
- status=CASE WHEN status IN ('initializing','queued') THEN 'settling' ELSE status END,updated_at=datetime('now')
347
- WHERE id=?${ownerClause} AND status IN ('initializing','queued','running')`,
348
- args
349
- }, {
350
- sql: `UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
351
- last_error='Lead list enrichment was cancelled before execution.',updated_at=datetime('now')
352
- WHERE job_id=? AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND cancel_requested_at IS NOT NULL AND status='settling')`,
353
- args: [jobId, jobId]
354
- }], "write");
355
- return getLeadListEnrichmentJob(jobId, userId);
356
- }
357
- async function cancelPendingLeadListEnrichmentRows(jobId) {
358
- await ensureLeadListEnrichmentRepositorySchema();
359
- const results = await getDb().batch([{
360
- sql: `UPDATE lead_list_enrichment_rows SET status='cancelled',completed_at=datetime('now'),updated_at=datetime('now')
361
- WHERE job_id=? AND status='pending' AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND cancel_requested_at IS NOT NULL)`,
362
- args: [jobId, jobId]
363
- }, {
364
- sql: `UPDATE lead_list_enrichment_jobs SET
365
- done_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status IN ('complete','failed','cancelled')),
366
- failed_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='failed'),
367
- cancelled_rows=(SELECT COUNT(*) FROM lead_list_enrichment_rows WHERE job_id=? AND status='cancelled'),
368
- status=CASE WHEN status='running' THEN 'settling' ELSE status END,updated_at=datetime('now')
369
- WHERE id=? AND cancel_requested_at IS NOT NULL`,
370
- args: [jobId, jobId, jobId, jobId]
371
- }], "write");
372
- return Number(results[0]?.rowsAffected ?? 0);
373
- }
374
- async function saveLeadListEnrichmentDeliverable(input) {
375
- await ensureLeadListEnrichmentRepositorySchema();
376
- await getDb().execute({
377
- sql: `UPDATE lead_list_enrichment_jobs SET status='settling',result_json=?,csv_artifact_json=?,xlsx_artifact_json=?,
378
- warnings_json=?,updated_at=datetime('now') WHERE id=? AND status IN ('running','settling')`,
379
- args: [
380
- JSON.stringify(input.result),
381
- input.csvArtifact == null ? null : JSON.stringify(input.csvArtifact),
382
- input.xlsxArtifact == null ? null : JSON.stringify(input.xlsxArtifact),
383
- JSON.stringify(input.warnings ?? input.result.warnings),
384
- input.jobId
385
- ]
386
- });
387
- return getLeadListEnrichmentJob(input.jobId);
388
- }
389
- async function completeLeadListEnrichmentJob(input) {
390
- await ensureLeadListEnrichmentRepositorySchema();
391
- const current = await getLeadListEnrichmentJob(input.jobId);
392
- if (!current) throw new Error("lead list enrichment job not found");
393
- const billedMc = nonNegativeSafeInteger(input.billedMc, "billedMc");
394
- if (billedMc > current.holdMc) throw new Error("billedMc cannot exceed the lead list enrichment hold");
395
- if (TERMINAL_JOB_STATUSES.has(current.status)) {
396
- if (current.status === input.status && current.billedMc === billedMc) return current;
397
- throw new Error("lead list enrichment job already has a conflicting terminal result");
398
- }
399
- await getDb().execute({
400
- sql: `UPDATE lead_list_enrichment_jobs SET status=?,billed_mc=?,result_json=COALESCE(?,result_json),
401
- csv_artifact_json=CASE WHEN ?=1 THEN ? ELSE csv_artifact_json END,
402
- xlsx_artifact_json=CASE WHEN ?=1 THEN ? ELSE xlsx_artifact_json END,
403
- warnings_json=CASE WHEN ?=1 THEN ? ELSE warnings_json END,error_code=NULL,error_message=NULL,retryable=NULL,
404
- completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status NOT IN ('complete','partial','empty','failed','cancelled')`,
405
- args: [
406
- input.status,
407
- billedMc,
408
- input.result === void 0 ? null : JSON.stringify(input.result),
409
- input.csvArtifact === void 0 ? 0 : 1,
410
- input.csvArtifact == null ? null : JSON.stringify(input.csvArtifact),
411
- input.xlsxArtifact === void 0 ? 0 : 1,
412
- input.xlsxArtifact == null ? null : JSON.stringify(input.xlsxArtifact),
413
- input.warnings === void 0 ? 0 : 1,
414
- JSON.stringify(input.warnings ?? []),
415
- input.jobId
416
- ]
417
- });
418
- const completed = await getLeadListEnrichmentJob(input.jobId);
419
- if (!completed || completed.status !== input.status || completed.billedMc !== billedMc) throw new Error("lead list enrichment job completion conflicted");
420
- return completed;
421
- }
422
- async function failLeadListEnrichmentJob(input) {
423
- await ensureLeadListEnrichmentRepositorySchema();
424
- const current = await getLeadListEnrichmentJob(input.jobId);
425
- if (!current) throw new Error("lead list enrichment job not found");
426
- if (current.status !== "failed" && TERMINAL_JOB_STATUSES.has(current.status)) throw new Error("completed lead list enrichment job cannot be failed");
427
- if (current.status === "failed") return current;
428
- const billedMc = input.billedMc == null ? null : nonNegativeSafeInteger(input.billedMc, "billedMc");
429
- if (billedMc != null && billedMc > current.holdMc) throw new Error("billedMc cannot exceed the lead list enrichment hold");
430
- await getDb().execute({
431
- sql: `UPDATE lead_list_enrichment_jobs SET status='failed',billed_mc=COALESCE(?,billed_mc),error_code=?,error_message=?,retryable=?,
432
- completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status NOT IN ('complete','partial','empty','failed','cancelled')`,
433
- args: [
434
- billedMc,
435
- requiredBoundedString(input.errorCode, "error code", 200),
436
- requiredBoundedString(input.errorMessage, "error message", 2e3),
437
- input.retryable ? 1 : 0,
438
- input.jobId
439
- ]
440
- });
441
- const failed = await getLeadListEnrichmentJob(input.jobId);
442
- if (!failed || failed.status !== "failed") throw new Error("lead list enrichment job failure conflicted");
443
- return failed;
444
- }
445
- async function recordLeadListEnrichmentSettlement(jobId, billedMc) {
446
- await ensureLeadListEnrichmentRepositorySchema();
447
- const current = await getLeadListEnrichmentJob(jobId);
448
- if (!current) throw new Error("lead list enrichment job not found");
449
- const amount = nonNegativeSafeInteger(billedMc, "billedMc");
450
- if (amount > current.holdMc) throw new Error("billedMc cannot exceed the lead list enrichment hold");
451
- await getDb().execute({
452
- sql: `UPDATE lead_list_enrichment_jobs SET billed_mc=COALESCE(billed_mc,?),updated_at=datetime('now')
453
- WHERE id=? AND status IN ('settling','failed','cancelled')`,
454
- args: [amount, jobId]
455
- });
456
- const updated = await getLeadListEnrichmentJob(jobId);
457
- if (!updated || updated.billedMc !== amount) throw new Error("lead list enrichment settlement conflicts with durable amount");
458
- return updated;
459
- }
460
- async function listLeadListEnrichmentJobsNeedingSettlement(limit = 25) {
461
- await ensureLeadListEnrichmentRepositorySchema();
462
- const result = await getDb().execute({
463
- sql: `SELECT * FROM lead_list_enrichment_jobs WHERE status='settling' OR (status IN ('failed','cancelled') AND billed_mc IS NULL)
464
- ORDER BY updated_at ASC LIMIT ?`,
465
- args: [Math.max(1, Math.min(100, Math.trunc(limit)))]
466
- });
467
- return result.rows.map((row) => rowToJob(row));
468
- }
469
- async function listExpiredLeadListEnrichmentStarts(limit = 25, maxAttempts = LEAD_LIST_MAX_DISPATCH_ATTEMPTS, maxAgeMinutes = LEAD_LIST_START_DEADLINE_MINUTES) {
470
- await ensureLeadListEnrichmentRepositorySchema();
471
- const result = await getDb().execute({
472
- sql: `SELECT j.* FROM lead_list_enrichment_jobs j JOIN lead_list_enrichment_outbox o ON o.job_id=j.id
473
- WHERE j.status IN ('initializing','queued') AND j.billed_mc IS NULL AND (o.attempts>=? OR j.created_at<=datetime('now',?))
474
- AND (o.status IN ('held','pending','failed','dispatched') OR (o.status='dispatching' AND o.locked_until<=datetime('now')))
475
- ORDER BY j.updated_at ASC LIMIT ?`,
476
- args: [Math.max(1, Math.trunc(maxAttempts)), `-${Math.max(15, Math.trunc(maxAgeMinutes))} minutes`, Math.max(1, Math.min(100, Math.trunc(limit)))]
477
- });
478
- return result.rows.map((row) => rowToJob(row));
479
- }
480
- async function failExpiredLeadListEnrichmentStart(jobId, maxAttempts = LEAD_LIST_MAX_DISPATCH_ATTEMPTS, maxAgeMinutes = LEAD_LIST_START_DEADLINE_MINUTES) {
481
- await ensureLeadListEnrichmentRepositorySchema();
482
- const results = await getDb().batch([{
483
- sql: `UPDATE lead_list_enrichment_jobs SET status='failed',billed_mc=CASE
484
- WHEN EXISTS(SELECT 1 FROM billing_debits d WHERE d.idempotency_key=lead_list_enrichment_jobs.debit_key
485
- AND d.user_id=lead_list_enrichment_jobs.user_id AND d.status IN ('applied','settled')) THEN NULL ELSE 0 END,
486
- error_code='lead_list_dispatch_expired',
487
- error_message='Lead list enrichment did not start before its dispatch deadline; the unused hold will be refunded.',retryable=1,
488
- completed_at=datetime('now'),updated_at=datetime('now') WHERE id=? AND status IN ('initializing','queued') AND billed_mc IS NULL
489
- AND EXISTS(SELECT 1 FROM lead_list_enrichment_outbox o WHERE o.job_id=lead_list_enrichment_jobs.id AND
490
- (o.attempts>=? OR lead_list_enrichment_jobs.created_at<=datetime('now',?)))`,
491
- args: [jobId, Math.max(1, Math.trunc(maxAttempts)), `-${Math.max(15, Math.trunc(maxAgeMinutes))} minutes`]
492
- }, {
493
- sql: `UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
494
- last_error='Lead list enrichment dispatch deadline exhausted.',updated_at=datetime('now') WHERE job_id=?
495
- AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND error_code='lead_list_dispatch_expired')`,
496
- args: [jobId, jobId]
497
- }], "write");
498
- return Number(results[0]?.rowsAffected ?? 0) === 1 ? getLeadListEnrichmentJob(jobId) : null;
499
- }
500
- async function listStaleRunningLeadListEnrichmentJobs(limit = 25, staleMinutes = LEAD_LIST_RUNNING_STALE_MINUTES) {
501
- await ensureLeadListEnrichmentRepositorySchema();
502
- const result = await getDb().execute({ sql: `SELECT * FROM lead_list_enrichment_jobs WHERE status='running' AND billed_mc IS NULL
503
- AND updated_at<=datetime('now',?) ORDER BY updated_at ASC LIMIT ?`, args: [`-${Math.max(15, Math.trunc(staleMinutes))} minutes`, Math.max(1, Math.min(100, Math.trunc(limit)))] });
504
- return result.rows.map((row) => rowToJob(row));
505
- }
506
- async function failStaleRunningLeadListEnrichmentJob(jobId, staleMinutes = LEAD_LIST_RUNNING_STALE_MINUTES) {
507
- await ensureLeadListEnrichmentRepositorySchema();
508
- const results = await getDb().batch([{
509
- sql: `UPDATE lead_list_enrichment_jobs SET
510
- status=CASE WHEN cancel_requested_at IS NULL THEN 'failed' ELSE 'settling' END,
511
- error_code=CASE WHEN cancel_requested_at IS NULL THEN 'lead_list_worker_stale' ELSE NULL END,
512
- error_message=CASE WHEN cancel_requested_at IS NULL
513
- THEN 'Lead list enrichment stopped without a heartbeat; completed row checkpoints were preserved.' ELSE NULL END,
514
- retryable=CASE WHEN cancel_requested_at IS NULL THEN 1 ELSE NULL END,
515
- completed_at=CASE WHEN cancel_requested_at IS NULL THEN datetime('now') ELSE NULL END,updated_at=datetime('now')
516
- WHERE id=? AND status='running' AND billed_mc IS NULL
517
- AND updated_at<=datetime('now',?)`,
518
- args: [jobId, `-${Math.max(15, Math.trunc(staleMinutes))} minutes`]
519
- }, {
520
- sql: `UPDATE lead_list_enrichment_outbox SET status='exhausted',locked_by=NULL,locked_until=NULL,next_attempt_at=NULL,
521
- last_error='Lead list enrichment worker heartbeat expired.',updated_at=datetime('now') WHERE job_id=?
522
- AND EXISTS(SELECT 1 FROM lead_list_enrichment_jobs WHERE id=? AND (error_code='lead_list_worker_stale'
523
- OR (status='settling' AND cancel_requested_at IS NOT NULL)))`,
524
- args: [jobId, jobId]
525
- }], "write");
526
- return Number(results[0]?.rowsAffected ?? 0) === 1 ? getLeadListEnrichmentJob(jobId) : null;
527
- }
528
- async function claimLeadListEnrichmentOutbox(input) {
529
- await ensureLeadListEnrichmentRepositorySchema();
530
- const workerId = requiredBoundedString(input.workerId, "outbox worker id", 160);
531
- const limit = Math.max(1, Math.min(100, Math.trunc(input.limit ?? 25)));
532
- const leaseSeconds = Math.max(30, Math.min(3600, Math.trunc(input.leaseSeconds ?? 120)));
533
- const claimToken = `${workerId}:${randomUUID()}`;
534
- const results = await getDb().batch([{
535
- sql: `UPDATE lead_list_enrichment_outbox SET status='dispatching',attempts=attempts+1,locked_by=?,
536
- locked_until=datetime('now',?),updated_at=datetime('now') WHERE id IN (SELECT id FROM lead_list_enrichment_outbox
537
- WHERE attempts<${LEAD_LIST_MAX_DISPATCH_ATTEMPTS} AND created_at>datetime('now','-${LEAD_LIST_START_DEADLINE_MINUTES} minutes')
538
- AND (status='pending' OR (status='failed' AND (next_attempt_at IS NULL OR next_attempt_at<=datetime('now')))
539
- OR (status='dispatching' AND locked_until<=datetime('now'))) ORDER BY created_at ASC LIMIT ?)`,
540
- args: [claimToken, `+${leaseSeconds} seconds`, limit]
541
- }, { sql: `SELECT * FROM lead_list_enrichment_outbox WHERE locked_by=? AND status='dispatching' ORDER BY created_at ASC`, args: [claimToken] }], "write");
542
- return results[1]?.rows.map((row) => rowToOutbox(row)) ?? [];
543
- }
544
- async function markLeadListEnrichmentOutboxDispatched(id, claimToken) {
545
- await ensureLeadListEnrichmentRepositorySchema();
546
- const result = await getDb().execute({ sql: `UPDATE lead_list_enrichment_outbox SET status='dispatched',locked_by=NULL,
547
- locked_until=NULL,next_attempt_at=NULL,last_error=NULL,dispatched_at=datetime('now'),updated_at=datetime('now')
548
- WHERE id=? AND status='dispatching' AND locked_by=?`, args: [id, claimToken] });
549
- return Number(result.rowsAffected ?? 0) > 0;
550
- }
551
- async function markLeadListEnrichmentOutboxFailed(input) {
552
- await ensureLeadListEnrichmentRepositorySchema();
553
- const retryAfterSeconds = Math.max(1, Math.min(86400, Math.trunc(input.retryAfterSeconds ?? 30)));
554
- const result = await getDb().execute({ sql: `UPDATE lead_list_enrichment_outbox SET status='failed',locked_by=NULL,
555
- locked_until=NULL,next_attempt_at=datetime('now',?),last_error=?,updated_at=datetime('now')
556
- WHERE id=? AND status='dispatching' AND locked_by=?`, args: [`+${retryAfterSeconds} seconds`, String(input.error).slice(0, 2e3), input.id, input.claimToken] });
557
- return Number(result.rowsAffected ?? 0) > 0;
558
- }
559
-
560
- export {
561
- LEAD_LIST_MAX_DISPATCH_ATTEMPTS,
562
- LEAD_LIST_START_DEADLINE_MINUTES,
563
- LEAD_LIST_RUNNING_STALE_MINUTES,
564
- stableCanonicalJson,
565
- leadListEnrichmentFingerprint,
566
- leadListRowInputDigest,
567
- ensureLeadListEnrichmentRepositorySchema,
568
- getLeadListEnrichmentJob,
569
- getLeadListEnrichmentJobByIdempotencyKey,
570
- createOrGetLeadListEnrichmentJob,
571
- setInitializingLeadListEnrichmentConcurrencyLock,
572
- markLeadListEnrichmentJobQueued,
573
- markLeadListEnrichmentJobRunning,
574
- claimLeadListEnrichmentRow,
575
- getLeadListEnrichmentRow,
576
- listLeadListEnrichmentRows,
577
- saveLeadListEnrichmentRowResult,
578
- requestLeadListEnrichmentCancellation,
579
- cancelPendingLeadListEnrichmentRows,
580
- saveLeadListEnrichmentDeliverable,
581
- completeLeadListEnrichmentJob,
582
- failLeadListEnrichmentJob,
583
- recordLeadListEnrichmentSettlement,
584
- listLeadListEnrichmentJobsNeedingSettlement,
585
- listExpiredLeadListEnrichmentStarts,
586
- failExpiredLeadListEnrichmentStart,
587
- listStaleRunningLeadListEnrichmentJobs,
588
- failStaleRunningLeadListEnrichmentJob,
589
- claimLeadListEnrichmentOutbox,
590
- markLeadListEnrichmentOutboxDispatched,
591
- markLeadListEnrichmentOutboxFailed
592
- };