mcp-scraper 0.66.0 → 0.66.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +32 -1
  2. package/README.md +2 -2
  3. package/dist/{analytics-repository-IKE53S3E.js → analytics-repository-RUXBFSFX.js} +1 -2
  4. package/dist/bin/api-server.cjs +2898 -2075
  5. package/dist/bin/api-server.js +4 -5
  6. package/dist/bin/mcp-scraper-cli.cjs +1 -2
  7. package/dist/bin/mcp-scraper-cli.js +6 -7
  8. package/dist/bin/mcp-scraper-install.cjs +7 -3
  9. package/dist/bin/mcp-scraper-install.js +2 -3
  10. package/dist/bin/mcp-stdio-server.cjs +1669 -97
  11. package/dist/bin/mcp-stdio-server.js +11 -12
  12. package/dist/bin/paa-harvest.cjs +0 -1
  13. package/dist/bin/paa-harvest.js +4 -5
  14. package/dist/{chunk-VIWT35S6.js → chunk-3EYALCYK.js} +1 -2
  15. package/dist/{chunk-4LINSJNI.js → chunk-3TKKJQB4.js} +1 -2
  16. package/dist/{chunk-4VG3NUUY.js → chunk-55B3T5L2.js} +23 -28
  17. package/dist/{chunk-FG6LJGJ7.js → chunk-77LQV73B.js} +0 -1
  18. package/dist/{chunk-N54JS5WP.js → chunk-ABYBPODM.js} +1 -2
  19. package/dist/chunk-ALEBMPOW.js +6 -0
  20. package/dist/{chunk-2XUXVKT4.js → chunk-AYJHI2FK.js} +0 -1
  21. package/dist/{chunk-AAGVOPP5.js → chunk-CETDPNMZ.js} +24 -25
  22. package/dist/{chunk-5BEPOFKG.js → chunk-CNLLTD5B.js} +0 -1
  23. package/dist/{chunk-P7YFUNKR.js → chunk-DBWWIK7O.js} +3 -4
  24. package/dist/{chunk-EKJ65E3Z.js → chunk-DNM65UCK.js} +0 -1
  25. package/dist/{chunk-3YSEJTMC.js → chunk-GGZEC22A.js} +0 -1
  26. package/dist/{chunk-AJ4YGHG2.js → chunk-K7Y27ND3.js} +3 -14
  27. package/dist/{chunk-XPFRP5QT.js → chunk-KBTUPRTF.js} +3 -4
  28. package/dist/{chunk-5PIO7QBG.js → chunk-KXRWY7DZ.js} +3 -4
  29. package/dist/{chunk-DCWXVAQT.js → chunk-OZDCTNUV.js} +6 -2
  30. package/dist/{chunk-BIUQ6O2Y.js → chunk-OZJMVCDK.js} +0 -1
  31. package/dist/{chunk-X7UG2JCF.js → chunk-PGJQDMC2.js} +1 -2
  32. package/dist/{chunk-DKIG2TEE.js → chunk-PMCT2YQM.js} +2 -3
  33. package/dist/{chunk-G3P3ZDB4.js → chunk-RJ7JVYKU.js} +0 -1
  34. package/dist/{chunk-WD73OFL3.js → chunk-VXLU74YZ.js} +1 -2
  35. package/dist/{chunk-SRRI2HJL.js → chunk-YKL3O7IU.js} +1742 -104
  36. package/dist/{db-BRI465XQ.js → db-4RVZ3NYZ.js} +9 -10
  37. package/dist/{extract-bundle-QAI7JYV2.js → extract-bundle-EA5URLXN.js} +8 -9
  38. package/dist/index.cjs +0 -1
  39. package/dist/index.js +4 -5
  40. package/dist/{lead-list-enrichment-repository-ODWGRVCQ.js → lead-list-enrichment-repository-IBXNWM6H.js} +2 -3
  41. package/dist/{location-data-repository-67MMEVZQ.js → location-data-repository-WHHQUCUI.js} +3 -4
  42. package/dist/{server-MTKWBSAR.js → server-Q7VYGSGQ.js} +450 -1264
  43. package/dist/{site-extract-repository-J3RH3MOS.js → site-extract-repository-NTM4MDQN.js} +4 -5
  44. package/dist/{worker-K7MPPQ7I.js → worker-UTBUWEXQ.js} +8 -9
  45. package/package.json +4 -1
  46. package/dist/analytics-repository-IKE53S3E.js.map +0 -1
  47. package/dist/bin/api-server.cjs.map +0 -1
  48. package/dist/bin/api-server.js.map +0 -1
  49. package/dist/bin/mcp-scraper-cli.cjs.map +0 -1
  50. package/dist/bin/mcp-scraper-cli.js.map +0 -1
  51. package/dist/bin/mcp-scraper-install.cjs.map +0 -1
  52. package/dist/bin/mcp-scraper-install.js.map +0 -1
  53. package/dist/bin/mcp-stdio-server.cjs.map +0 -1
  54. package/dist/bin/mcp-stdio-server.js.map +0 -1
  55. package/dist/bin/paa-harvest.cjs.map +0 -1
  56. package/dist/bin/paa-harvest.js.map +0 -1
  57. package/dist/chunk-2XUXVKT4.js.map +0 -1
  58. package/dist/chunk-3YSEJTMC.js.map +0 -1
  59. package/dist/chunk-4LINSJNI.js.map +0 -1
  60. package/dist/chunk-4VG3NUUY.js.map +0 -1
  61. package/dist/chunk-5BEPOFKG.js.map +0 -1
  62. package/dist/chunk-5PIO7QBG.js.map +0 -1
  63. package/dist/chunk-AAGVOPP5.js.map +0 -1
  64. package/dist/chunk-AJ4YGHG2.js.map +0 -1
  65. package/dist/chunk-BIUQ6O2Y.js.map +0 -1
  66. package/dist/chunk-DCWXVAQT.js.map +0 -1
  67. package/dist/chunk-DKIG2TEE.js.map +0 -1
  68. package/dist/chunk-EKJ65E3Z.js.map +0 -1
  69. package/dist/chunk-FG6LJGJ7.js.map +0 -1
  70. package/dist/chunk-G3P3ZDB4.js.map +0 -1
  71. package/dist/chunk-HDYOVGC6.js +0 -7
  72. package/dist/chunk-HDYOVGC6.js.map +0 -1
  73. package/dist/chunk-N54JS5WP.js.map +0 -1
  74. package/dist/chunk-P7YFUNKR.js.map +0 -1
  75. package/dist/chunk-SRRI2HJL.js.map +0 -1
  76. package/dist/chunk-VIWT35S6.js.map +0 -1
  77. package/dist/chunk-WD73OFL3.js.map +0 -1
  78. package/dist/chunk-X7UG2JCF.js.map +0 -1
  79. package/dist/chunk-XPFRP5QT.js.map +0 -1
  80. package/dist/db-BRI465XQ.js.map +0 -1
  81. package/dist/extract-bundle-QAI7JYV2.js.map +0 -1
  82. package/dist/index.cjs.map +0 -1
  83. package/dist/index.js.map +0 -1
  84. package/dist/lead-list-enrichment-repository-ODWGRVCQ.js.map +0 -1
  85. package/dist/location-data-repository-67MMEVZQ.js.map +0 -1
  86. package/dist/server-MTKWBSAR.js.map +0 -1
  87. package/dist/site-extract-repository-J3RH3MOS.js.map +0 -1
  88. package/dist/worker-K7MPPQ7I.js.map +0 -1
@@ -10,13 +10,13 @@ import {
10
10
  queuedCommonsDocumentCount,
11
11
  semanticCommonsEntityScores,
12
12
  typeFromMime
13
- } from "./chunk-P7YFUNKR.js";
13
+ } from "./chunk-DBWWIK7O.js";
14
14
  import {
15
15
  REQUIRED_LOCATION_STATE_CODES,
16
16
  importHostedCensusPlacesCsv,
17
17
  importHostedZipGroupsCsv,
18
18
  listActiveHostedLocationDatasets
19
- } from "./chunk-DKIG2TEE.js";
19
+ } from "./chunk-PMCT2YQM.js";
20
20
  import {
21
21
  cancelPendingLeadListEnrichmentRows,
22
22
  claimLeadListEnrichmentOutbox,
@@ -43,7 +43,7 @@ import {
43
43
  saveLeadListEnrichmentRowResult,
44
44
  setInitializingLeadListEnrichmentConcurrencyLock,
45
45
  stableCanonicalJson
46
- } from "./chunk-N54JS5WP.js";
46
+ } from "./chunk-ABYBPODM.js";
47
47
  import {
48
48
  ANALYTICS_CONTENT_SORTS,
49
49
  AnalyticsRepositoryError,
@@ -99,7 +99,7 @@ import {
99
99
  updateAnalyticsPixel,
100
100
  upsertAnalyticsAdSpend,
101
101
  verifySession
102
- } from "./chunk-5BEPOFKG.js";
102
+ } from "./chunk-CNLLTD5B.js";
103
103
  import {
104
104
  classifyHarvestProblem,
105
105
  createHarvestAttemptRecorder,
@@ -108,20 +108,12 @@ import {
108
108
  harvestProblemResponse,
109
109
  publicErrorBoundary,
110
110
  serializeHarvestProblem
111
- } from "./chunk-AAGVOPP5.js";
112
- import "./chunk-BIUQ6O2Y.js";
111
+ } from "./chunk-CETDPNMZ.js";
112
+ import "./chunk-OZJMVCDK.js";
113
113
  import {
114
- AIModeSelectors,
115
- AIOverviewSelectors,
116
114
  BACKUP_LAST_RESORT_ATTEMPTS,
117
115
  BrowserDriver,
118
- ForumSelectors,
119
- LocalPackSelectors,
120
116
  MapsSelectors,
121
- OrganicSelectors,
122
- PAASelectors,
123
- VideoSelectors,
124
- WhatPeopleSayingSelectors,
125
117
  backupProxyAvailable,
126
118
  brightDataSerpEnabled,
127
119
  buildYouTubeChannelVideosUrl,
@@ -130,14 +122,12 @@ import {
130
122
  deleteKernelProxyId,
131
123
  detectPageUserAgent,
132
124
  detectRuntimeUserAgent,
133
- encodeUule,
134
125
  harvest,
135
- normalizeLocation,
136
126
  resolveBackupKernelProxyId,
137
127
  resolveKernelProxyId,
138
128
  serpArrivalUrl,
139
129
  withRuntimeUserAgent
140
- } from "./chunk-AJ4YGHG2.js";
130
+ } from "./chunk-K7Y27ND3.js";
141
131
  import {
142
132
  ApiErrorResponseSchema,
143
133
  ArtifactTemplatePresetKeySchema,
@@ -209,18 +199,18 @@ import {
209
199
  resolveDeploymentProfile,
210
200
  resolveLocalSourcebookSchemaType,
211
201
  transcribeMediaUrl
212
- } from "./chunk-SRRI2HJL.js";
202
+ } from "./chunk-YKL3O7IU.js";
213
203
  import {
214
204
  auditImageUrls,
215
205
  auditImages,
216
206
  buildLinkReport,
217
207
  computeIssues
218
- } from "./chunk-X7UG2JCF.js";
208
+ } from "./chunk-PGJQDMC2.js";
219
209
  import {
220
210
  sanitizeAttempts,
221
211
  sanitizeHarvestResult,
222
212
  sanitizeOutboundDiagnostics
223
- } from "./chunk-VIWT35S6.js";
213
+ } from "./chunk-3EYALCYK.js";
224
214
  import {
225
215
  browserServiceApiKey,
226
216
  browserServiceProfileName,
@@ -230,7 +220,7 @@ import {
230
220
  recordKernelSession,
231
221
  recordVendorUsage,
232
222
  runWithCostContext
233
- } from "./chunk-4LINSJNI.js";
223
+ } from "./chunk-3TKKJQB4.js";
234
224
  import {
235
225
  listWorkflowDefinitions,
236
226
  runWorkflow,
@@ -240,12 +230,12 @@ import {
240
230
  workflowOutputBaseDir,
241
231
  workflowStepCount,
242
232
  workflowSupportsSteps
243
- } from "./chunk-XPFRP5QT.js";
233
+ } from "./chunk-KBTUPRTF.js";
244
234
  import {
245
235
  csvRecords,
246
236
  parseCsv,
247
237
  rowsToCsv
248
- } from "./chunk-G3P3ZDB4.js";
238
+ } from "./chunk-RJ7JVYKU.js";
249
239
  import {
250
240
  DEFAULT_MAPS_PROXY_MODE,
251
241
  DEFAULT_PROXY_MODE,
@@ -255,18 +245,18 @@ import {
255
245
  RawMapsHoursRowSchema,
256
246
  RawMapsOverviewSchema,
257
247
  RawMapsReviewStatsSchema
258
- } from "./chunk-FG6LJGJ7.js";
248
+ } from "./chunk-77LQV73B.js";
259
249
  import {
260
250
  extractContactEvidence
261
- } from "./chunk-WD73OFL3.js";
251
+ } from "./chunk-VXLU74YZ.js";
262
252
  import {
263
253
  isPrivateIpAddress,
264
254
  loadHtmlDocument,
265
255
  validatePublicHttpUrl
266
- } from "./chunk-3YSEJTMC.js";
256
+ } from "./chunk-GGZEC22A.js";
267
257
  import {
268
258
  PACKAGE_VERSION
269
- } from "./chunk-HDYOVGC6.js";
259
+ } from "./chunk-ALEBMPOW.js";
270
260
  import {
271
261
  SITE_EXTRACT_ARTIFACT_PREFIX,
272
262
  abandonExtractSettlement,
@@ -297,14 +287,14 @@ import {
297
287
  setExtractJobTotal,
298
288
  settleExtractJob,
299
289
  terminalExtractJobStatus
300
- } from "./chunk-5PIO7QBG.js";
290
+ } from "./chunk-KXRWY7DZ.js";
301
291
  import {
302
292
  createPrivateArtifact,
303
293
  privateArtifactOwnerId,
304
294
  readPrivateArtifactBuffer,
305
295
  readPrivateArtifactWindow,
306
296
  renewPrivateArtifactDownload
307
- } from "./chunk-EKJ65E3Z.js";
297
+ } from "./chunk-DNM65UCK.js";
308
298
  import {
309
299
  BROWSER_OPEN_MIN_BALANCE_MC,
310
300
  CONCURRENCY_PACK_INVOICE_DESCRIPTION,
@@ -357,7 +347,7 @@ import {
357
347
  tierChangeLeavesPrivateOffer,
358
348
  tierCreditsPerInterval,
359
349
  vendorCostUsdToMc
360
- } from "./chunk-2XUXVKT4.js";
350
+ } from "./chunk-AYJHI2FK.js";
361
351
  import {
362
352
  CaptchaError,
363
353
  MapsSearchIncompletePageError,
@@ -369,7 +359,7 @@ import {
369
359
  cancelJob,
370
360
  checkRateLimit,
371
361
  checkpointJob,
372
- claimInngestPaaJob,
362
+ claimDurablePaaJob,
373
363
  claimStripeEvent,
374
364
  completeJob,
375
365
  completeStripeEvent,
@@ -392,9 +382,9 @@ import {
392
382
  deleteWorkflowSchedule,
393
383
  expireOldLots,
394
384
  failJob,
385
+ failRunningJob,
395
386
  failStripeEvent,
396
387
  failWorkflowRunRecord,
397
- finishHarvestAttempt,
398
388
  getBillingLedgerSummary,
399
389
  getClient,
400
390
  getDb,
@@ -424,10 +414,9 @@ import {
424
414
  listHarvestAttempts,
425
415
  listInboxMessages,
426
416
  listJobs,
427
- listPendingInngestPaaJobs,
428
- listRecentTerminalInngestPaaJobs,
417
+ listRecentTerminalDurablePaaJobs,
429
418
  listRequestEvents,
430
- listStaleRunningInngestPaaJobs,
419
+ listStaleRunningDurablePaaJobs,
431
420
  listUsers,
432
421
  listUsersDueForConnectedAccountBillingReconciliation,
433
422
  listWorkflowArtifacts,
@@ -460,10 +449,9 @@ import {
460
449
  setUserMemoryCreds,
461
450
  settleDebitMcIdempotent,
462
451
  softDeleteInboxMessage,
463
- startHarvestAttempt,
464
452
  upsertPageSnapshot,
465
453
  verifyPassword
466
- } from "./chunk-4VG3NUUY.js";
454
+ } from "./chunk-55B3T5L2.js";
467
455
 
468
456
  // src/blog/registry.ts
469
457
  var posts = [
@@ -7795,12 +7783,12 @@ function buildIngestValidatePrompt(input) {
7795
7783
  `{"error_severity":"blocking","error_code":"dep-001","message":"Required Python package missing: {package}. Install with: pip install networkx pandas lxml","file":null,"remediation":"Run: pip install networkx pandas lxml"}`,
7796
7784
  `\`\`\``,
7797
7785
  ``,
7798
- `Also verify uszips.csv is accessible (required for Phase 3 location classification):`,
7786
+ `Also check whether the optional uszips.csv classifier dataset is accessible. When unavailable, Phase 3 remains functional and falls back to LLM classification:`,
7799
7787
  ``,
7800
7788
  `\`\`\`bash`,
7801
7789
  `python3 -c "`,
7802
7790
  `import csv, os`,
7803
- `path = '/Users/vilovieta/Downloads/sales-magician-api-leads-magician-01c6cff78e31/tools/analytics/data/uszips.csv'`,
7791
+ `path = os.environ.get('US_ZIPS_CSV_PATH') or os.environ.get('MCP_SCRAPER_USZIPS_CSV_PATH') or '/tmp/uszips.csv'`,
7804
7792
  `if not os.path.exists(path):`,
7805
7793
  ` print('uszips:missing')`,
7806
7794
  `else:`,
@@ -7814,12 +7802,12 @@ function buildIngestValidatePrompt(input) {
7814
7802
  `If \`uszips:missing\`, write a **warning** (non-blocking) to \`ingestion_error_log.jsonl\`:`,
7815
7803
  ``,
7816
7804
  `\`\`\`json`,
7817
- `{"error_severity":"warning","error_code":"dep-002","message":"uszips.csv not found at expected path. Location classifier will be disabled \u2014 all URL classification will use Claude. Phase 3 LLM costs will be higher.","file":"/Users/vilovieta/Downloads/sales-magician-api-leads-magician-01c6cff78e31/tools/analytics/data/uszips.csv","remediation":"Verify the sales-magician codebase is present at the expected path. See reference/05-location-classifier.md."}`,
7805
+ `{"error_severity":"warning","error_code":"dep-002","message":"uszips.csv was not found at the resolved path. The location classifier is disabled and Phase 3 will use LLM classification, which may increase cost.","file":"{resolved_uszips_path}","remediation":"Optionally set US_ZIPS_CSV_PATH or MCP_SCRAPER_USZIPS_CSV_PATH to a readable uszips.csv file with the expected schema."}`,
7818
7806
  `\`\`\``,
7819
7807
  ``,
7820
7808
  `If \`uszips:schema-mismatch\`, write the same warning with the mismatch detail in the message. The audit continues without the location classifier \u2014 all URLs fall through to Claude classification.`,
7821
7809
  ``,
7822
- `Proceed only if the check passes.`,
7810
+ `Continue after recording either warning; missing or mismatched uszips.csv data is not a blocking gate.`,
7823
7811
  ``,
7824
7812
  `---`,
7825
7813
  ``,
@@ -8733,6 +8721,9 @@ var CompareRecommendOutputSchema = z9.object({
8733
8721
  ).optional(),
8734
8722
  newPageCreationList: z9.array(z9.unknown()).optional()
8735
8723
  });
8724
+ function resolveSiteAuditUsZipsCsvPath(env = process.env) {
8725
+ return env["US_ZIPS_CSV_PATH"]?.trim() || env["MCP_SCRAPER_USZIPS_CSV_PATH"]?.trim() || "/tmp/uszips.csv";
8726
+ }
8736
8727
  var SiteAuditService = class {
8737
8728
  constructor(deps) {
8738
8729
  this.deps = deps;
@@ -8900,7 +8891,7 @@ var SiteAuditService = class {
8900
8891
  currentPayloadChars += itemChars;
8901
8892
  }
8902
8893
  if (currentBatch.length > 0) batches.push(currentBatch);
8903
- const usZipsCsvPath = process.env["US_ZIPS_CSV_PATH"] ?? "/Users/vilovieta/Downloads/sales-magician-api-leads-magician-01c6cff78e31/tools/analytics/data/uszips.csv";
8894
+ const usZipsCsvPath = resolveSiteAuditUsZipsCsvPath();
8904
8895
  await this.runLlmBilled(job.user_id, `${jobId2}:phase3-classify`, async () => {
8905
8896
  for (const batch of batches) {
8906
8897
  let classified = [];
@@ -9554,7 +9545,7 @@ var siteExtractFn = inngest.createFunction(
9554
9545
  });
9555
9546
  }) : null;
9556
9547
  const finalized = await step.run("finalize", async () => {
9557
- const { assembleExtractArtifacts } = await import("./extract-bundle-QAI7JYV2.js");
9548
+ const { assembleExtractArtifacts } = await import("./extract-bundle-EA5URLXN.js");
9558
9549
  const current = await getExtractJob(jobId2);
9559
9550
  if (!current) throw new Error("extract job disappeared before finalization");
9560
9551
  const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
@@ -10514,7 +10505,7 @@ async function resolveUsZipsPath(requestedPath) {
10514
10505
  async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
10515
10506
  const mode = resolvedLocationDataMode({ zipSource: sourceMode });
10516
10507
  if (mode === "hosted") {
10517
- const { getHostedZipGroups } = await import("./location-data-repository-67MMEVZQ.js");
10508
+ const { getHostedZipGroups } = await import("./location-data-repository-WHHQUCUI.js");
10518
10509
  const hosted = await getHostedZipGroups(stateAbbr);
10519
10510
  if (!hosted.dataset) {
10520
10511
  throw new DirectoryLocationError(
@@ -10577,7 +10568,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
10577
10568
  const sourceUrl = censusStateUrl(state.fips);
10578
10569
  const warnings = [];
10579
10570
  if (resolvedLocationDataMode(config) === "hosted") {
10580
- const { queryHostedLocationMarkets } = await import("./location-data-repository-67MMEVZQ.js");
10571
+ const { queryHostedLocationMarkets } = await import("./location-data-repository-WHHQUCUI.js");
10581
10572
  const hosted = await queryHostedLocationMarkets({
10582
10573
  state: state.abbr,
10583
10574
  minPopulation: options.minPopulation,
@@ -12064,7 +12055,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
12064
12055
  async function settlePersistedLeadListEnrichment(jobId2) {
12065
12056
  const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
12066
12057
  const job = jobs.find((candidate) => candidate.id === jobId2);
12067
- const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-ODWGRVCQ.js");
12058
+ const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-IBXNWM6H.js");
12068
12059
  const current = job ?? await getLeadListEnrichmentJob2(jobId2);
12069
12060
  if (!current) return null;
12070
12061
  const rows = await listLeadListEnrichmentRows(jobId2);
@@ -12120,7 +12111,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
12120
12111
  const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
12121
12112
  const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
12122
12113
  if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
12123
- const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-ODWGRVCQ.js");
12114
+ const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-IBXNWM6H.js");
12124
12115
  await failLeadListEnrichmentJob2({
12125
12116
  jobId: job.id,
12126
12117
  errorCode: "lead_list_all_rows_failed",
@@ -12421,1067 +12412,8 @@ var leadListEnrichmentFn = inngest.createFunction(
12421
12412
  }
12422
12413
  );
12423
12414
 
12424
- // src/api/paa-capture-artifacts.ts
12425
- var DAY_MS = 24 * 60 * 60 * 1e3;
12426
- var PAA_CAPTURE_ARTIFACT_PREFIX = "paa-captures";
12427
- function paaCaptureArtifactPolicy() {
12428
- return {
12429
- prefix: PAA_CAPTURE_ARTIFACT_PREFIX,
12430
- artifactTtlMs: 7 * DAY_MS,
12431
- downloadTtlMs: 15 * 60 * 1e3,
12432
- token: process.env.PAA_CAPTURE_BLOB_READ_WRITE_TOKEN ?? process.env.BLOB_READ_WRITE_TOKEN ?? null
12433
- };
12434
- }
12435
- async function createPaaRawDomArtifact(input) {
12436
- return createPrivateArtifact({
12437
- policy: paaCaptureArtifactPolicy(),
12438
- ownerId: String(input.userId),
12439
- artifactKey: `${input.jobId}-attempt-${input.attempt}.html.gz`,
12440
- createdAt: input.createdAt ?? /* @__PURE__ */ new Date(),
12441
- filename: `${input.jobId}-serp.html.gz`,
12442
- contentType: "application/gzip",
12443
- content: input.gzip
12444
- });
12445
- }
12446
- async function tryCreatePaaRawDomArtifact(input) {
12447
- try {
12448
- return {
12449
- artifact: await createPaaRawDomArtifact(input),
12450
- status: "stored"
12451
- };
12452
- } catch (error) {
12453
- console.error("[paa-capture-artifact] private DOM preservation unavailable:", error instanceof Error ? error.message : String(error));
12454
- return { artifact: null, status: "unavailable" };
12455
- }
12456
- }
12457
-
12458
- // src/api/paa-harvest-settlement.ts
12459
- function capturedQuestionCount(result) {
12460
- if (!result || typeof result !== "object") return 0;
12461
- const value = result;
12462
- if (typeof value.totalQuestions === "number") return value.totalQuestions;
12463
- return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
12464
- }
12465
- function finalCost(result, heldMc) {
12466
- const questions = capturedQuestionCount(result);
12467
- if (questions <= 0) return 0;
12468
- return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
12469
- }
12470
- async function settlePaaHarvestJob(jobOrId) {
12471
- const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
12472
- if (!job) return false;
12473
- const options = job.options;
12474
- const debitKey2 = options.billingDebitKey;
12475
- const heldMc = Number(options.billingHoldMc ?? 0);
12476
- if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
12477
- const settlement = await settleDebitMcIdempotent(
12478
- job.user_id,
12479
- debitKey2,
12480
- finalCost(job.result, heldMc),
12481
- LedgerOperation.PAA_REFUND,
12482
- "durable PAA harvest settlement",
12483
- "paa_harvest"
12484
- );
12485
- await markJobBillingSettled(job.id, settlement.final_amount_mc);
12486
- return true;
12487
- }
12488
-
12489
- // src/api/serp-identity-db.ts
12490
- import { randomUUID as randomUUID3 } from "crypto";
12491
- async function createSerpIdentityRow(input) {
12492
- const id = `serpi_${randomUUID3().replace(/-/g, "").slice(0, 20)}`;
12493
- await getDb().execute({
12494
- sql: `INSERT INTO serp_identities
12495
- (id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
12496
- VALUES (?, ?, ?, ?, ?, 'isp', ?, 'ready')`,
12497
- args: [id, input.userId, input.name, input.kernelProfileName, input.kernelProxyId, input.country]
12498
- });
12499
- const row = await getSerpIdentityRow(input.userId, input.name);
12500
- if (!row) throw new Error("SERP identity insert failed");
12501
- return row;
12502
- }
12503
- async function getSerpIdentityRow(userId, name) {
12504
- const result = await getDb().execute({
12505
- sql: `SELECT * FROM serp_identities WHERE user_id = ? AND name = ? LIMIT 1`,
12506
- args: [userId, name]
12507
- });
12508
- return result.rows[0] ?? null;
12509
- }
12510
- async function listSerpIdentityRows(userId) {
12511
- const result = await getDb().execute({
12512
- sql: `SELECT * FROM serp_identities WHERE user_id = ? ORDER BY created_at DESC`,
12513
- args: [userId]
12514
- });
12515
- return result.rows;
12516
- }
12517
- async function touchSerpIdentity(userId, name) {
12518
- await getDb().execute({
12519
- sql: `UPDATE serp_identities
12520
- SET last_used_at = datetime('now'), updated_at = datetime('now')
12521
- WHERE user_id = ? AND name = ?`,
12522
- args: [userId, name]
12523
- });
12524
- }
12525
- async function setSerpIdentityStatus(userId, name, status) {
12526
- await getDb().execute({
12527
- sql: `UPDATE serp_identities SET status = ?, updated_at = datetime('now') WHERE user_id = ? AND name = ?`,
12528
- args: [status, userId, name]
12529
- });
12530
- }
12531
- async function deleteSerpIdentityRow(userId, name) {
12532
- await getDb().execute({
12533
- sql: `DELETE FROM serp_identities WHERE user_id = ? AND name = ?`,
12534
- args: [userId, name]
12535
- });
12536
- }
12537
-
12538
- // src/paa/durable-capture.ts
12539
- import Kernel4 from "@onkernel/sdk";
12540
- import { createHash as createHash2 } from "crypto";
12541
- var PAA_INVOCATION_BUDGET_MS = 28e4;
12542
- var PAA_BROWSER_WORK_BUDGET_MS = 25e4;
12543
- var PAA_SOLVER_BUDGET_MS = 6e4;
12544
- function remainingSeconds(deadlineMs, ceiling) {
12545
- return Math.max(1, Math.min(ceiling, Math.floor((deadlineMs - Date.now()) / 1e3)));
12546
- }
12547
- async function execute(kernel, sessionId, code, deadlineMs, ceilingSeconds) {
12548
- if (Date.now() >= deadlineMs) throw new Error("paa_work_deadline_exhausted");
12549
- const response = await kernel.browsers.playwright.execute(sessionId, {
12550
- code,
12551
- timeout_sec: remainingSeconds(deadlineMs, ceilingSeconds)
12552
- });
12553
- if (!response.success || response.result === null || response.result === void 0) {
12554
- throw new Error(response.error || response.stderr || "co-located browser execution failed");
12555
- }
12556
- return response.result;
12557
- }
12558
- function navigationCode(searchUrl) {
12559
- return `
12560
- const wait = ms => page.waitForTimeout(ms);
12561
- const block = async () => {
12562
- if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
12563
- if (await page.locator(${JSON.stringify(PAASelectors.captchaMarker)}).count().catch(() => 0)) return 'captcha';
12564
- const title = await page.title().catch(() => '');
12565
- const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
12566
- if (/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(text)) return 'captcha';
12567
- if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(title + '\\n' + text.slice(0, 3000))) return 'soft_block';
12568
- return null;
12569
- };
12570
- await page.goto(${JSON.stringify(searchUrl)}, { waitUntil: 'domcontentloaded', timeout: 45000 });
12571
- let captchaObserved = false;
12572
- let solverCleared = false;
12573
- let blocked = await block();
12574
- const solverDeadline = Date.now() + ${PAA_SOLVER_BUDGET_MS};
12575
- while (blocked === 'captcha' && Date.now() < solverDeadline) {
12576
- captchaObserved = true;
12577
- await wait(4000 + Math.floor(Math.random() * 1500));
12578
- blocked = await block();
12579
- if (!blocked) solverCleared = true;
12580
- }
12581
- let count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
12582
- for (let scan = 0; !blocked && count === 0 && scan < 7; scan += 1) {
12583
- await page.mouse.wheel(0, 520 + Math.floor(Math.random() * 420));
12584
- await wait(350 + Math.floor(Math.random() * 500));
12585
- count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
12586
- }
12587
- await page.evaluate((itemSelector) => {
12588
- const state = window;
12589
- if (state.__mcpTrustedListenerInstalled) return;
12590
- document.addEventListener('click', event => {
12591
- if (event.target instanceof Element && event.target.closest(itemSelector)) {
12592
- state.__mcpLastPaaClickTrusted = event.isTrusted;
12593
- }
12594
- }, { capture: true });
12595
- state.__mcpTrustedListenerInstalled = true;
12596
- }, ${JSON.stringify(PAASelectors.item)}).catch(() => {});
12597
- return {
12598
- outcome: blocked || (count > 0 ? 'ready' : 'no_paa'),
12599
- finalUrl: page.url(),
12600
- title: await page.title().catch(() => ''),
12601
- captchaObserved,
12602
- solverCleared,
12603
- environment: await page.evaluate(() => ({
12604
- userAgent: navigator.userAgent,
12605
- language: navigator.language,
12606
- platform: navigator.platform,
12607
- timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
12608
- innerWidth: window.innerWidth,
12609
- innerHeight: window.innerHeight,
12610
- outerWidth: window.outerWidth,
12611
- outerHeight: window.outerHeight,
12612
- screenWidth: window.screen.width,
12613
- screenHeight: window.screen.height,
12614
- devicePixelRatio: window.devicePixelRatio,
12615
- })).catch(() => null),
12616
- };
12617
- `;
12618
- }
12619
- function recordReaderSource() {
12620
- return `
12621
- const clean = value => (value || '').replace(/\\u00a0/g, ' ').replace(/[ \\t]+/g, ' ').replace(/\\n{3,}/g, '\\n\\n').trim();
12622
- const externalHref = anchor => {
12623
- const href = anchor && anchor.href || '';
12624
- if (!/^https?:\\/\\//i.test(href)) return '';
12625
- try {
12626
- const url = new URL(href);
12627
- if (/(^|\\.)google(usercontent)?\\.[a-z.]+$/i.test(url.hostname)) {
12628
- const redirected = url.searchParams.get('q') || url.searchParams.get('url');
12629
- return redirected && redirected.startsWith('http') ? redirected : '';
12630
- }
12631
- } catch {}
12632
- return href;
12633
- };
12634
- const sourceLabel = value => clean(value)
12635
- .replace(/\\s*\\(\\+\\d+\\)\\s*-\\s*View related links.*$/i, '')
12636
- .replace(/\\s*-\\s*Opens in new tab.*$/i, '').trim();
12637
- const readRecords = () => Array.from(document.querySelectorAll(SEL.item)).map(item => {
12638
- const question = clean(item.getAttribute(SEL.itemDataQ)
12639
- || item.getAttribute(SEL.itemDataInitQ)
12640
- || item.querySelector(SEL.itemQuestionEl)?.innerText);
12641
- const expanded = item.classList.contains(SEL.expandedClass)
12642
- || item.querySelector(SEL.clickTarget)?.getAttribute('aria-expanded') === 'true';
12643
- const container = expanded ? item.querySelector(SEL.answerContainer) : null;
12644
- const answerElement = container?.querySelector(SEL.aiAnswerBody) || container;
12645
- const anchors = expanded ? Array.from(answerElement?.querySelectorAll(
12646
- SEL.aiCitation + ',' + SEL.aiSourceCard + ',a[href]'
12647
- ) || []) : [];
12648
- const byUrl = new Map();
12649
- for (const anchor of anchors) {
12650
- const url = externalHref(anchor);
12651
- if (!url || byUrl.has(url)) continue;
12652
- let site = '';
12653
- try { site = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
12654
- byUrl.set(url, { title: sourceLabel(anchor.getAttribute('aria-label')) || clean(anchor.textContent), site, url });
12655
- }
12656
- const sources = Array.from(byUrl.values());
12657
- let sourceTitle = expanded ? clean(item.querySelector(SEL.sourceTitle)?.textContent) : '';
12658
- let sourceSite = expanded ? clean(item.querySelector(SEL.sourceSite)?.textContent) : '';
12659
- let sourceCite = expanded ? externalHref(item.querySelector(SEL.sourceCite)) : '';
12660
- const primary = sources.find(source => source.title.length > 2) || sources[0];
12661
- if (!sourceCite && primary) {
12662
- sourceTitle ||= primary.title;
12663
- sourceSite ||= primary.site;
12664
- sourceCite = primary.url;
12665
- }
12666
- return {
12667
- question,
12668
- answer: clean(answerElement?.innerText || answerElement?.textContent),
12669
- sourceTitle,
12670
- sourceSite,
12671
- sourceCite,
12672
- sources,
12673
- depth: 1,
12674
- parentQuestion: null,
12675
- };
12676
- }).filter(record => record.question);
12677
- `;
12678
- }
12679
- function expansionCode(clickedQuestions) {
12680
- return `
12681
- const SEL = ${JSON.stringify(PAASelectors)};
12682
- ${recordReaderSource()}
12683
- const deliveryBlock = async () => {
12684
- if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
12685
- if (await page.locator(SEL.captchaMarker).count().catch(() => 0)) return 'captcha';
12686
- const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
12687
- if (/recaptcha|unusual traffic|are you a robot|about this page/i.test(text)) return 'captcha';
12688
- if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(text.slice(0, 3000))) return 'soft_block';
12689
- return null;
12690
- };
12691
- const blocked = await deliveryBlock();
12692
- const records = await page.evaluate(({ SEL }) => {
12693
- ${recordReaderSource()}
12694
- return readRecords();
12695
- }, { SEL });
12696
- if (blocked) return {
12697
- outcome: blocked, targetQuestion: null, records, newQuestions: [],
12698
- finalUrl: page.url(), title: await page.title().catch(() => ''),
12699
- interactionTrusted: null, clickError: null,
12700
- };
12701
- const clicked = new Set(${JSON.stringify(clickedQuestions)});
12702
- const targetQuestion = records.map(record => record.question).find(question => !clicked.has(question)) || null;
12703
- if (!targetQuestion) return {
12704
- outcome: 'exhausted', targetQuestion: null, records, newQuestions: [],
12705
- finalUrl: page.url(), title: await page.title().catch(() => ''),
12706
- interactionTrusted: null, clickError: null,
12707
- };
12708
- const items = page.locator(SEL.item);
12709
- let targetIndex = -1;
12710
- for (let index = 0, count = await items.count().catch(() => 0); index < count; index += 1) {
12711
- const question = await items.nth(index).evaluate((item, SEL) =>
12712
- item.getAttribute(SEL.itemDataQ)
12713
- || item.getAttribute(SEL.itemDataInitQ)
12714
- || item.querySelector(SEL.itemQuestionEl)?.innerText?.trim()
12715
- || '', SEL).catch(() => '');
12716
- if (question.replace(/\\s+/g, ' ').trim() === targetQuestion) { targetIndex = index; break; }
12717
- }
12718
- if (targetIndex < 0) return {
12719
- outcome: 'exhausted', targetQuestion, records, newQuestions: [],
12720
- finalUrl: page.url(), title: await page.title().catch(() => ''),
12721
- interactionTrusted: null, clickError: 'question disappeared before click',
12722
- };
12723
- const beforeQuestions = new Set(records.map(record => record.question));
12724
- const item = items.nth(targetIndex);
12725
- await item.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => {});
12726
- await page.waitForTimeout(180 + Math.floor(Math.random() * 420));
12727
- await page.evaluate(() => { window.__mcpLastPaaClickTrusted = null; }).catch(() => {});
12728
- let clickError = null;
12729
- try {
12730
- await item.locator(SEL.clickTarget).first().click({
12731
- timeout: 6000,
12732
- delay: 55 + Math.floor(Math.random() * 120),
12733
- });
12734
- } catch (error) {
12735
- clickError = String(error && error.message || error).slice(0, 240);
12736
- }
12737
- const interactionTrusted = await page.evaluate(() => window.__mcpLastPaaClickTrusted ?? null).catch(() => null);
12738
- await page.waitForTimeout(650 + Math.floor(Math.random() * 750));
12739
- const afterRecords = await page.evaluate(({ SEL }) => {
12740
- ${recordReaderSource()}
12741
- return readRecords();
12742
- }, { SEL });
12743
- return {
12744
- outcome: clickError ? 'exhausted' : 'clicked',
12745
- targetQuestion,
12746
- records: afterRecords,
12747
- newQuestions: afterRecords.map(record => record.question).filter(question => !beforeQuestions.has(question)),
12748
- finalUrl: page.url(),
12749
- title: await page.title().catch(() => ''),
12750
- interactionTrusted,
12751
- clickError,
12752
- };
12753
- `;
12754
- }
12755
- function finalCaptureCode(maxQuestions, parentByQuestion) {
12756
- const surfaces = {
12757
- aiOverview: AIOverviewSelectors,
12758
- aiMode: AIModeSelectors,
12759
- organic: OrganicSelectors,
12760
- localPack: LocalPackSelectors,
12761
- video: VideoSelectors,
12762
- forum: ForumSelectors,
12763
- whatPeopleSaying: WhatPeopleSayingSelectors
12764
- };
12765
- return `
12766
- const SEL = ${JSON.stringify(PAASelectors)};
12767
- const SURF = ${JSON.stringify(surfaces)};
12768
- const PARENTS = ${JSON.stringify(parentByQuestion)};
12769
- ${recordReaderSource()}
12770
- const capture = await page.evaluate(({ SEL, SURF, PARENTS, maxQuestions }) => {
12771
- ${recordReaderSource()}
12772
- const unique = (items, key) => {
12773
- const seen = new Set();
12774
- return items.filter(item => { const value = key(item); if (!value || seen.has(value)) return false; seen.add(value); return true; });
12775
- };
12776
- const paa = readRecords().map(record => {
12777
- const parentQuestion = PARENTS[record.question] || null;
12778
- let depth = 1, parent = parentQuestion;
12779
- const seen = new Set();
12780
- while (parent && !seen.has(parent)) { seen.add(parent); depth += 1; parent = PARENTS[parent] || null; }
12781
- return { ...record, depth, parentQuestion };
12782
- }).slice(0, maxQuestions);
12783
- const organicResults = [];
12784
- document.querySelectorAll(SURF.organic.result).forEach(card => {
12785
- const titleElement = card.querySelector(SURF.organic.title);
12786
- const title = clean(titleElement?.textContent);
12787
- const link = titleElement?.closest('a');
12788
- const url = externalHref(link);
12789
- if (!title || !url) return;
12790
- let domain = '';
12791
- try { domain = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
12792
- const rating = card.querySelector(SURF.organic.ratingWrap);
12793
- organicResults.push({
12794
- position: organicResults.length + 1, title, url, domain,
12795
- cite: clean(card.querySelector(SURF.organic.cite)?.textContent) || null,
12796
- snippet: clean(card.querySelector(SURF.organic.snippet)?.textContent) || null,
12797
- isRedditStyle: Boolean(card.querySelector(SURF.organic.redditCite)),
12798
- inlineRating: rating ? {
12799
- value: clean(rating.querySelector(SURF.organic.ratingValue)?.textContent),
12800
- count: clean(rating.querySelector(SURF.organic.reviewCount)?.textContent),
12801
- } : null,
12802
- });
12803
- });
12804
- const localPack = [];
12805
- document.querySelectorAll(SURF.localPack.card).forEach(card => {
12806
- const name = clean(card.querySelector(SURF.localPack.name)?.textContent);
12807
- if (!name) return;
12808
- const links = Array.from(card.querySelectorAll('a[href]'));
12809
- let cid = card.querySelector('a[data-cid]')?.getAttribute('data-cid') || null;
12810
- for (const link of links) {
12811
- if (cid) break;
12812
- const match = link.href.match(/[?&]cid=(\\d+)/);
12813
- if (match) cid = match[1];
12814
- }
12815
- localPack.push({
12816
- position: localPack.length + 1, name, cid,
12817
- rating: clean(card.querySelector(SURF.localPack.ratingValue)?.textContent) || null,
12818
- reviewCount: clean(card.querySelector(SURF.localPack.reviewCount)?.textContent).replace(/[()]/g, '') || null,
12819
- metadata: [], websiteUrl: links.map(externalHref).find(Boolean) || null,
12820
- directionsUrl: links.find(link => link.href.includes('google.com/maps'))?.href || null,
12821
- });
12822
- });
12823
- const videos = unique(Array.from(document.querySelectorAll(SURF.video.item)).map(link => {
12824
- const url = externalHref(link);
12825
- const raw = clean(link.textContent);
12826
- const platform = /youtu/i.test(url) ? 'YouTube' : /tiktok/i.test(url) ? 'TikTok' : /instagram/i.test(url) ? 'Instagram' : /facebook/i.test(url) ? 'Facebook' : '';
12827
- return { type: 'video', title: raw, channel: '', platform, duration: raw.match(/\\b\\d{1,2}:\\d{2}\\b/)?.[0] || '', url };
12828
- }).filter(item => item.title && item.url), item => item.url);
12829
- const forums = unique(Array.from(document.querySelectorAll(SURF.forum.item)).map(link => ({
12830
- title: clean(link.querySelector(SURF.forum.title)?.textContent),
12831
- source: clean(link.querySelector(SURF.forum.source)?.textContent),
12832
- url: externalHref(link),
12833
- })).filter(item => item.title && item.url), item => item.url);
12834
- const socialCards = Array.from(document.querySelectorAll(SURF.whatPeopleSaying.card));
12835
- const whatPeopleSaying = unique(socialCards.map(card => {
12836
- const link = card.querySelector(SURF.whatPeopleSaying.cardLink);
12837
- const url = externalHref(link);
12838
- const title = clean(card.querySelector(SURF.whatPeopleSaying.titleH1)?.textContent || card.querySelector(SURF.whatPeopleSaying.titleDiv)?.textContent);
12839
- const source = clean(card.querySelector(SURF.whatPeopleSaying.ytChannel)?.textContent || card.querySelector(SURF.whatPeopleSaying.source)?.textContent);
12840
- const platform = clean(card.querySelector(SURF.whatPeopleSaying.platformBadge)?.textContent) || (/youtube/i.test(url) ? 'YouTube' : '');
12841
- const identity = (platform + source + url).toLowerCase();
12842
- const type = identity.includes('reddit') ? 'reddit' : identity.includes('facebook') ? 'facebook' : identity.includes('instagram') ? 'instagram' : identity.includes('tiktok') ? 'tiktok' : identity.includes('youtube') ? 'youtube' : 'unknown';
12843
- return { type, title, url, source, platform, popularComment: null, engagement: '', date: '', duration: null, authorNote: null };
12844
- }).filter(item => item.title || item.url), item => item.url || item.type + ':' + item.title);
12845
- const aioRoot = document.querySelector(SURF.aiOverview.root) || document.querySelector(SURF.aiOverview.legacyRoot);
12846
- const aimRoot = document.querySelector(SURF.aiMode.root);
12847
- const citations = root => unique(Array.from(root?.querySelectorAll('a[href]') || []).map(link => ({
12848
- text: clean(link.textContent || link.getAttribute('aria-label')), href: externalHref(link),
12849
- })).filter(item => item.href), item => item.href);
12850
- const aiOverview = { detected: Boolean(aioRoot), text: aioRoot ? clean(aioRoot.innerText || aioRoot.textContent) : null, citations: citations(aioRoot), expanded: true, fullyExpanded: true, sections: [] };
12851
- const aiMode = { detected: Boolean(aimRoot), text: aimRoot ? clean(aimRoot.innerText || aimRoot.textContent) : null, citations: citations(aimRoot) };
12852
- const html = document.documentElement.outerHTML;
12853
- const kgIds = unique(Array.from(html.matchAll(/\\/g\\/[a-zA-Z0-9_-]{5,20}/g)).map(match => match[0]), value => value);
12854
- const cids = unique([
12855
- ...localPack.map(item => item.cid).filter(Boolean),
12856
- ...Array.from(html.matchAll(/[?&]cid=(\\d+)/g)).map(match => match[1]),
12857
- ], value => value);
12858
- const gcids = unique(Array.from(html.matchAll(/gcid:[a-zA-Z0-9_]+/g)).map(match => match[0]), value => value);
12859
- const entities = localPack.filter(item => item.cid).map(item => ({ name: item.name, kgId: null, cid: item.cid, gcid: null }));
12860
- const headings = unique(Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"]')).map(node => clean(node.textContent)).filter(text => text.length > 1 && text.length < 160), value => value);
12861
- const relatedSearches = unique(Array.from(document.querySelectorAll('a[href*="/search?"]')).map(link => ({
12862
- text: clean(link.textContent), url: link.href,
12863
- })).filter(item => item.text && /related/i.test(item.url + ' ' + item.text)), item => item.url);
12864
- const counts = {
12865
- peopleAlsoAsk: paa.length, organic: organicResults.length, localPack: localPack.length,
12866
- videos: videos.length, discussions: forums.length, whatPeopleSaying: whatPeopleSaying.length,
12867
- aiOverview: aiOverview.detected ? 1 : 0, aiMode: aiMode.detected ? 1 : 0,
12868
- relatedSearches: relatedSearches.length,
12869
- };
12870
- return {
12871
- paa, organicResults: unique(organicResults, item => item.url), localPack: unique(localPack, item => item.cid || item.name),
12872
- videos, forums, whatPeopleSaying, aiOverview, aiMode,
12873
- surface: window.google?.sn === 'aim' ? 'aim' : window.google?.sn === 'web' ? 'web' : 'unknown',
12874
- entityIds: { entities, kgIds, cids, gcids },
12875
- serpFeatures: { sectionHeadings: headings, detected: Object.fromEntries(Object.entries(counts).map(([key, count]) => [key, count > 0])), counts },
12876
- relatedSearches,
12877
- };
12878
- }, { SEL, SURF, PARENTS, maxQuestions: ${maxQuestions} });
12879
- return capture;
12880
- `;
12881
- }
12882
- var RAW_PREPARE_CODE = `
12883
- const result = await page.evaluate(async () => {
12884
- const source = new TextEncoder().encode(document.documentElement.outerHTML);
12885
- const compressed = typeof CompressionStream === 'undefined'
12886
- ? source
12887
- : new Uint8Array(await new Response(new Blob([source]).stream().pipeThrough(new CompressionStream('gzip'))).arrayBuffer());
12888
- const pieces = [];
12889
- for (let offset = 0; offset < compressed.length; offset += 32768) {
12890
- pieces.push(String.fromCharCode(...compressed.subarray(offset, offset + 32768)));
12891
- }
12892
- window.__mcpPaaRawDom = btoa(pieces.join(''));
12893
- return { encoding: typeof CompressionStream === 'undefined' ? 'plain' : 'gzip', chars: window.__mcpPaaRawDom.length };
12894
- });
12895
- return result;
12896
- `;
12897
- function rawChunkCode(offset, length) {
12898
- return `return await page.evaluate(({ offset, length }) => ({ offset, chunk: window.__mcpPaaRawDom.slice(offset, offset + length) }), { offset: ${offset}, length: ${length} });`;
12899
- }
12900
- function buildPublicResult(input) {
12901
- const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
12902
- const selected = input.records;
12903
- const flat = selected.map((record) => ({
12904
- seed_query: input.query,
12905
- question: record.question,
12906
- answer: record.answer,
12907
- source_title: record.sourceTitle,
12908
- source_site: record.sourceSite,
12909
- source_cite: record.sourceCite,
12910
- depth: record.depth,
12911
- parent_question: record.parentQuestion ?? "",
12912
- extracted_at: extractedAt
12913
- }));
12914
- const tree = selected.map((record) => ({
12915
- question: record.question,
12916
- answer: record.answer || null,
12917
- sourceTitle: record.sourceTitle || null,
12918
- sourceSite: record.sourceSite || null,
12919
- sourceCite: record.sourceCite || null,
12920
- depth: record.depth,
12921
- parentQuestion: record.parentQuestion,
12922
- children: []
12923
- }));
12924
- return {
12925
- seed: input.query,
12926
- location: input.location,
12927
- extractedAt,
12928
- diagnostics: {
12929
- completionStatus: selected.length > 0 ? "paa_found" : "no_paa",
12930
- problem: null,
12931
- resultQuality: input.partial ? "partial" : "complete",
12932
- degradedResult: input.partial,
12933
- retryRecommended: false,
12934
- completeness: {
12935
- paaWithoutAnswer: selected.filter((record) => !record.answer).length,
12936
- paaWithoutSource: selected.filter((record) => !record.sourceCite).length,
12937
- paaAnswersRecovered: 0,
12938
- aioShareCaptured: null
12939
- }
12940
- },
12941
- totalQuestions: selected.length,
12942
- surface: input.capture.surface,
12943
- aiOverview: input.capture.aiOverview,
12944
- aiMode: input.capture.aiMode,
12945
- whatPeopleSaying: input.capture.whatPeopleSaying,
12946
- tree,
12947
- flat,
12948
- videos: input.capture.videos,
12949
- forums: input.capture.forums,
12950
- organicResults: input.capture.organicResults,
12951
- localPack: input.capture.localPack,
12952
- entityIds: input.capture.entityIds,
12953
- serpFeatures: input.capture.serpFeatures,
12954
- relatedSearches: input.capture.relatedSearches,
12955
- stats: {
12956
- seed: input.query,
12957
- totalQuestions: selected.length,
12958
- maxDepthReached: selected.reduce((max, item) => Math.max(max, item.depth), 0),
12959
- durationMs: input.durationMs,
12960
- errorCount: input.partial ? 1 : 0
12961
- },
12962
- capture: {
12963
- execution: "kernel_colocated_playwright",
12964
- stealth: true,
12965
- managedProxy: input.persistentIdentity ? "serp_identity" : "kernel_default",
12966
- metadataOverrides: false,
12967
- locationSignal: input.location ? "uule_only" : "none",
12968
- attempt: input.attempt,
12969
- finalUrl: input.navigation.finalUrl,
12970
- captchaObserved: input.navigation.captchaObserved,
12971
- solverCleared: input.navigation.solverCleared,
12972
- environment: input.navigation.environment,
12973
- rawDomSha256: input.rawDomSha256
12974
- }
12975
- };
12976
- }
12977
- async function runDurablePaaCapture(input) {
12978
- const startedAt = input.startedAtMs ?? Date.now();
12979
- const workDeadline = startedAt + PAA_BROWSER_WORK_BUDGET_MS;
12980
- const location2 = input.location?.trim() || null;
12981
- const params = new URLSearchParams({
12982
- q: input.query,
12983
- gl: input.gl ?? "us",
12984
- hl: input.hl ?? "en",
12985
- pws: "0"
12986
- });
12987
- if (location2) params.set("uule", encodeUule(normalizeLocation(location2)));
12988
- const searchUrl = `https://www.google.com/search?${params.toString()}`;
12989
- const kernel = new Kernel4({ apiKey: input.apiKey });
12990
- let lastCheckpoint = {
12991
- phase: "navigating",
12992
- attempt: 1,
12993
- observedAt: (/* @__PURE__ */ new Date()).toISOString(),
12994
- records: [],
12995
- clickedQuestions: [],
12996
- finalUrl: null,
12997
- captchaObserved: false,
12998
- solverCleared: false,
12999
- error: null
13000
- };
13001
- const persist = async (patch) => {
13002
- lastCheckpoint = { ...lastCheckpoint, ...patch, observedAt: (/* @__PURE__ */ new Date()).toISOString() };
13003
- await input.onCheckpoint?.(lastCheckpoint);
13004
- };
13005
- for (let attempt = 1; attempt <= 2; attempt += 1) {
13006
- if (Date.now() >= workDeadline) break;
13007
- let sessionId = null;
13008
- const records = /* @__PURE__ */ new Map();
13009
- const clicked = /* @__PURE__ */ new Set();
13010
- const parents = {};
13011
- let navigation = null;
13012
- try {
13013
- await persist({ phase: "navigating", attempt, records: [], clickedQuestions: [], error: null });
13014
- const browser = await kernel.browsers.create({
13015
- stealth: true,
13016
- timeout_seconds: Math.max(60, Math.min(280, Math.ceil((workDeadline - Date.now()) / 1e3) + 20)),
13017
- ...input.proxyId ? { proxy_id: input.proxyId } : {},
13018
- ...input.profileName ? { profile: { name: input.profileName, save_changes: false } } : {}
13019
- });
13020
- sessionId = browser.session_id;
13021
- navigation = await execute(kernel, sessionId, navigationCode(searchUrl), workDeadline, 110);
13022
- await persist({
13023
- phase: navigation.outcome === "ready" ? "expanding" : "blocked",
13024
- finalUrl: navigation.finalUrl,
13025
- captchaObserved: navigation.captchaObserved,
13026
- solverCleared: navigation.solverCleared
13027
- });
13028
- if (navigation.outcome === "captcha" || navigation.outcome === "soft_block") {
13029
- if (attempt === 1 && records.size === 0) continue;
13030
- throw new Error(navigation.outcome);
13031
- }
13032
- let stagnant = 0;
13033
- for (let step = 0; step < input.maxQuestions * 2 && Date.now() < workDeadline - 35e3; step += 1) {
13034
- const observed = await execute(
13035
- kernel,
13036
- sessionId,
13037
- expansionCode(Array.from(clicked)),
13038
- workDeadline,
13039
- 25
13040
- );
13041
- const beforeSize = records.size;
13042
- for (const record of observed.records) {
13043
- const existing = records.get(record.question);
13044
- records.set(record.question, {
13045
- ...record,
13046
- answer: record.answer || existing?.answer || "",
13047
- sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
13048
- sourceSite: record.sourceSite || existing?.sourceSite || "",
13049
- sourceCite: record.sourceCite || existing?.sourceCite || "",
13050
- sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
13051
- parentQuestion: parents[record.question] ?? null,
13052
- depth: parents[record.question] ? 2 : 1
13053
- });
13054
- }
13055
- if (observed.outcome === "clicked" && observed.targetQuestion) {
13056
- clicked.add(observed.targetQuestion);
13057
- for (const question of observed.newQuestions) {
13058
- if (!parents[question]) parents[question] = observed.targetQuestion;
13059
- }
13060
- }
13061
- const selected2 = Array.from(records.values()).slice(0, input.maxQuestions);
13062
- await persist({
13063
- phase: "expanding",
13064
- records: selected2,
13065
- clickedQuestions: Array.from(clicked),
13066
- finalUrl: observed.finalUrl,
13067
- captchaObserved: lastCheckpoint.captchaObserved || observed.outcome === "captcha"
13068
- });
13069
- if (observed.outcome === "captcha" || observed.outcome === "soft_block") break;
13070
- if (selected2.length >= input.maxQuestions && selected2.every((record) => clicked.has(record.question))) break;
13071
- stagnant = records.size === beforeSize || observed.outcome === "exhausted" ? stagnant + 1 : 0;
13072
- if (stagnant >= 4) break;
13073
- }
13074
- const finalCapture = await execute(
13075
- kernel,
13076
- sessionId,
13077
- finalCaptureCode(input.maxQuestions, parents),
13078
- workDeadline,
13079
- 45
13080
- );
13081
- for (const record of finalCapture.paa) {
13082
- const existing = records.get(record.question);
13083
- records.set(record.question, {
13084
- ...record,
13085
- answer: record.answer || existing?.answer || "",
13086
- sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
13087
- sourceSite: record.sourceSite || existing?.sourceSite || "",
13088
- sourceCite: record.sourceCite || existing?.sourceCite || "",
13089
- sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
13090
- parentQuestion: parents[record.question] ?? record.parentQuestion,
13091
- depth: record.depth
13092
- });
13093
- }
13094
- let rawDomGzip = null;
13095
- let rawDomSha256 = null;
13096
- if (Date.now() < workDeadline - 12e3) {
13097
- const prepared = await execute(
13098
- kernel,
13099
- sessionId,
13100
- RAW_PREPARE_CODE,
13101
- workDeadline,
13102
- 35
13103
- );
13104
- const chunks = [];
13105
- for (let offset = 0; offset < prepared.chars && Date.now() < workDeadline - 5e3; offset += 18e4) {
13106
- const part = await execute(
13107
- kernel,
13108
- sessionId,
13109
- rawChunkCode(offset, 18e4),
13110
- workDeadline,
13111
- 15
13112
- );
13113
- if (part.offset !== offset) throw new Error("raw_dom_chunk_offset_mismatch");
13114
- chunks.push(part.chunk);
13115
- }
13116
- if (chunks.join("").length === prepared.chars) {
13117
- const transported = Buffer.from(chunks.join(""), "base64");
13118
- if (prepared.encoding === "gzip") rawDomGzip = transported;
13119
- else {
13120
- const { gzipSync } = await import("zlib");
13121
- rawDomGzip = gzipSync(transported);
13122
- }
13123
- rawDomSha256 = createHash2("sha256").update(rawDomGzip).digest("hex");
13124
- }
13125
- }
13126
- const selected = Array.from(records.values()).slice(0, input.maxQuestions);
13127
- const partial = selected.length < input.maxQuestions || selected.some((record) => !record.answer || !record.sourceCite);
13128
- await persist({ phase: "captured", records: selected, clickedQuestions: Array.from(clicked) });
13129
- return {
13130
- result: buildPublicResult({
13131
- query: input.query,
13132
- location: location2,
13133
- capture: finalCapture,
13134
- records: selected,
13135
- navigation,
13136
- durationMs: Date.now() - startedAt,
13137
- partial,
13138
- attempt,
13139
- rawDomSha256,
13140
- persistentIdentity: Boolean(input.proxyId && input.profileName)
13141
- }),
13142
- rawDomGzip,
13143
- rawDomSha256,
13144
- attempt,
13145
- partial,
13146
- checkpoint: lastCheckpoint
13147
- };
13148
- } catch (error) {
13149
- const message = error instanceof Error ? error.message : String(error);
13150
- await persist({
13151
- phase: records.size > 0 ? "captured" : "failed",
13152
- records: Array.from(records.values()).slice(0, input.maxQuestions),
13153
- clickedQuestions: Array.from(clicked),
13154
- error: message.slice(0, 500)
13155
- });
13156
- if (records.size > 0) {
13157
- const emptyCapture = {
13158
- paa: [],
13159
- organicResults: [],
13160
- localPack: [],
13161
- videos: [],
13162
- forums: [],
13163
- whatPeopleSaying: [],
13164
- aiOverview: { detected: false, text: null, citations: [] },
13165
- aiMode: { detected: false, text: null, citations: [] },
13166
- surface: "unknown",
13167
- entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
13168
- serpFeatures: { sectionHeadings: [], detected: {}, counts: {} },
13169
- relatedSearches: []
13170
- };
13171
- return {
13172
- result: buildPublicResult({
13173
- query: input.query,
13174
- location: location2,
13175
- capture: emptyCapture,
13176
- records: Array.from(records.values()).slice(0, input.maxQuestions),
13177
- navigation: navigation ?? {
13178
- outcome: "soft_block",
13179
- finalUrl: lastCheckpoint.finalUrl ?? "",
13180
- title: "",
13181
- captchaObserved: lastCheckpoint.captchaObserved,
13182
- solverCleared: lastCheckpoint.solverCleared,
13183
- environment: null
13184
- },
13185
- durationMs: Date.now() - startedAt,
13186
- partial: true,
13187
- attempt,
13188
- rawDomSha256: null,
13189
- persistentIdentity: Boolean(input.proxyId && input.profileName)
13190
- }),
13191
- rawDomGzip: null,
13192
- rawDomSha256: null,
13193
- attempt,
13194
- partial: true,
13195
- checkpoint: lastCheckpoint
13196
- };
13197
- }
13198
- if (attempt === 2 || Date.now() >= workDeadline - 35e3) throw error;
13199
- } finally {
13200
- if (sessionId) {
13201
- await kernel.browsers.deleteByID(sessionId).catch(() => void 0);
13202
- }
13203
- }
13204
- }
13205
- throw new Error("paa_work_deadline_exhausted_before_capture");
13206
- }
13207
-
13208
- // src/inngest/functions/paa-harvest.ts
13209
- var BRIGHTDATA_INVOCATION_BUDGET_MS = 76e4;
13210
- var BRIGHTDATA_WORK_BUDGET_MS = 72e4;
13211
- function durableRecordsFromResult(result) {
13212
- const flat = Array.isArray(result.flat) ? result.flat : [];
13213
- return flat.flatMap((value) => {
13214
- if (!value || typeof value !== "object") return [];
13215
- const row = value;
13216
- const question = typeof row.question === "string" ? row.question : "";
13217
- if (!question) return [];
13218
- const sourceCite = typeof row.source_cite === "string" ? row.source_cite : "";
13219
- const sourceTitle = typeof row.source_title === "string" ? row.source_title : "";
13220
- const sourceSite = typeof row.source_site === "string" ? row.source_site : "";
13221
- return [{
13222
- question,
13223
- answer: typeof row.answer === "string" ? row.answer : "",
13224
- sourceTitle,
13225
- sourceSite,
13226
- sourceCite,
13227
- sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
13228
- depth: typeof row.depth === "number" ? row.depth : 1,
13229
- parentQuestion: typeof row.parent_question === "string" && row.parent_question ? row.parent_question : null
13230
- }];
13231
- });
13232
- }
13233
- function questionCount(result) {
13234
- if (!result || typeof result !== "object") return 0;
13235
- const value = result;
13236
- if (typeof value.totalQuestions === "number") return value.totalQuestions;
13237
- return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
13238
- }
13239
- var paaHarvestFn = inngest.createFunction(
13240
- {
13241
- id: "paa-harvest",
13242
- retries: 0,
13243
- triggers: [{ event: "mcp-scraper/paa.requested" }],
13244
- onFailure: async ({ event: event2 }) => {
13245
- const jobId2 = event2?.data?.event?.data?.jobId;
13246
- if (!jobId2) return;
13247
- const current = await getJob(jobId2);
13248
- if (!current) return;
13249
- if (current.status === "done" || current.status === "failed" || current.status === "cancelled") {
13250
- await settlePaaHarvestJob(jobId2).catch((error) => {
13251
- console.error("[paa-harvest/onFailure] terminal settlement pending:", error instanceof Error ? error.message : String(error));
13252
- });
13253
- return;
13254
- }
13255
- const rawError = String(event2?.data?.error?.message ?? "durable PAA invocation failed").slice(0, 1e3);
13256
- const problem = classifyHarvestProblem(new Error(rawError));
13257
- await failJob(jobId2, rawError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
13258
- await settlePaaHarvestJob(jobId2).then(async (settled) => {
13259
- if (!settled) return;
13260
- const settledJob = await getJob(jobId2);
13261
- const billedMc = Number(settledJob?.options.billedMc ?? 0);
13262
- await failJob(jobId2, rawError, harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
13263
- }).catch((error) => {
13264
- console.error("[paa-harvest/onFailure] settlement pending:", error instanceof Error ? error.message : String(error));
13265
- });
13266
- }
13267
- },
13268
- async ({ event: event2 }) => {
13269
- const invocationStartedAt = Date.now();
13270
- const job = await claimInngestPaaJob(event2.data.jobId);
13271
- if (!job) return { jobId: event2.data.jobId, status: "missing_or_not_paa" };
13272
- if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
13273
- await settlePaaHarvestJob(job.id);
13274
- return { jobId: job.id, status: job.status };
13275
- }
13276
- const options = job.options;
13277
- const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
13278
- const serpIdentity = options.serpIdentity ? await getSerpIdentityRow(job.user_id, options.serpIdentity) : null;
13279
- if (options.serpIdentity && !serpIdentity) throw new Error("SERP identity not found");
13280
- if (serpIdentity && serpIdentity.status !== "ready") throw new Error("SERP identity is not ready");
13281
- const useBrightData = brightDataSerpEnabled() && !serpIdentity;
13282
- const apiKey = browserServiceApiKey();
13283
- if (!useBrightData && !apiKey) throw new Error("browser service is not configured");
13284
- const attemptStartedAt = (/* @__PURE__ */ new Date()).toISOString();
13285
- await startHarvestAttempt({
13286
- jobId: job.id,
13287
- userId: job.user_id,
13288
- attemptNumber: 1,
13289
- maxAttempts: 2,
13290
- query: options.query ?? job.query,
13291
- location: options.location ?? null,
13292
- maxQuestions,
13293
- startedAt: attemptStartedAt
13294
- });
13295
- const checkpointState = { value: null };
13296
- try {
13297
- const persistCheckpoint = async (checkpoint, brightData = false) => {
13298
- checkpointState.value = checkpoint;
13299
- const persisted = await checkpointJob(job.id, {
13300
- progress: checkpoint,
13301
- durable: {
13302
- invocationBudgetMs: brightData ? BRIGHTDATA_INVOCATION_BUDGET_MS : PAA_INVOCATION_BUDGET_MS,
13303
- workBudgetMs: brightData ? BRIGHTDATA_WORK_BUDGET_MS : 25e4,
13304
- cleanupReserveMs: brightData ? 4e4 : 3e4
13305
- }
13306
- });
13307
- if (!persisted) throw new Error("job_left_running_state_during_capture");
13308
- };
13309
- const query = options.query ?? job.query;
13310
- const capture = useBrightData ? await (async () => {
13311
- const navigating = {
13312
- phase: "navigating",
13313
- attempt: 1,
13314
- observedAt: (/* @__PURE__ */ new Date()).toISOString(),
13315
- records: [],
13316
- clickedQuestions: [],
13317
- finalUrl: null,
13318
- captchaObserved: false,
13319
- solverCleared: false,
13320
- error: null
13321
- };
13322
- await persistCheckpoint(navigating, true);
13323
- const result2 = await harvest({
13324
- query,
13325
- location: options.location ?? void 0,
13326
- gl: options.gl ?? "us",
13327
- hl: options.hl ?? "en",
13328
- maxQuestions,
13329
- serpOnly: options.serpOnly ?? false,
13330
- maxAttempts: 1,
13331
- headless: true,
13332
- format: "json",
13333
- outputDir: "/tmp/paa-output-inngest",
13334
- includeAllSerpFeatures: options.includeAllSerpFeatures ?? false,
13335
- includeLocalPack: options.includeLocalPack ?? false,
13336
- includeForums: options.includeForums ?? false,
13337
- includeVideos: options.includeVideos ?? false,
13338
- includeAiOverview: options.includeAiOverview ?? false,
13339
- includeWhatPeopleSaying: options.includeWhatPeopleSaying ?? false,
13340
- softDeadlineMs: invocationStartedAt + BRIGHTDATA_WORK_BUDGET_MS
13341
- });
13342
- const records = durableRecordsFromResult(result2);
13343
- const captured = {
13344
- ...navigating,
13345
- phase: "captured",
13346
- observedAt: (/* @__PURE__ */ new Date()).toISOString(),
13347
- records,
13348
- clickedQuestions: records.map((record) => record.question)
13349
- };
13350
- await persistCheckpoint(captured, true);
13351
- return {
13352
- result: {
13353
- ...result2,
13354
- capture: {
13355
- execution: "brightdata_browser_api_controller_fulfilled",
13356
- provider: "bright_data_browser_api",
13357
- sameQueryClickGraph: true
13358
- }
13359
- },
13360
- rawDomGzip: null,
13361
- rawDomSha256: null,
13362
- attempt: 1,
13363
- partial: records.length < maxQuestions || records.some((record) => !record.answer || !record.sourceCite),
13364
- checkpoint: captured
13365
- };
13366
- })() : await runDurablePaaCapture({
13367
- apiKey,
13368
- query,
13369
- location: options.location ?? null,
13370
- gl: options.gl ?? "us",
13371
- hl: options.hl ?? "en",
13372
- maxQuestions,
13373
- ...serpIdentity ? {
13374
- proxyId: serpIdentity.kernel_proxy_id,
13375
- profileName: serpIdentity.kernel_profile_name
13376
- } : {},
13377
- startedAtMs: invocationStartedAt,
13378
- onCheckpoint: (checkpoint) => persistCheckpoint(checkpoint, false)
13379
- });
13380
- const artifactResult = capture.rawDomGzip ? await tryCreatePaaRawDomArtifact({
13381
- userId: job.user_id,
13382
- jobId: job.id,
13383
- attempt: capture.attempt,
13384
- gzip: capture.rawDomGzip
13385
- }) : { artifact: null, status: "unavailable" };
13386
- const artifact = artifactResult.artifact;
13387
- const artifactMissing = capture.rawDomGzip !== null && artifactResult.status !== "stored";
13388
- const captureResult = capture.result;
13389
- const result = {
13390
- ...captureResult,
13391
- diagnostics: {
13392
- ...captureResult.diagnostics,
13393
- ...artifactMissing ? {
13394
- resultQuality: "partial",
13395
- degradedResult: true,
13396
- retryRecommended: false
13397
- } : {}
13398
- },
13399
- capture: {
13400
- ...captureResult.capture,
13401
- rawDomArtifact: artifact,
13402
- rawDomArtifactStatus: artifactResult.status
13403
- }
13404
- };
13405
- await completeJob(job.id, result);
13406
- if (options.serpIdentity) await touchSerpIdentity(job.user_id, options.serpIdentity).catch(() => void 0);
13407
- await finishHarvestAttempt({
13408
- jobId: job.id,
13409
- attemptNumber: 1,
13410
- outcome: capture.partial ? "partial" : "paa_found",
13411
- kernelSessionId: null,
13412
- questionCount: questionCount(result),
13413
- durationMs: Date.now() - invocationStartedAt,
13414
- error: capture.checkpoint.error,
13415
- willRetry: false,
13416
- kernelDeleteStarted: true,
13417
- kernelDeleteSucceeded: true,
13418
- kernelDeleteError: null,
13419
- browserCloseSucceeded: true,
13420
- browserCloseError: null,
13421
- debug: {
13422
- execution: useBrightData ? "brightdata_browser_api_controller_fulfilled" : "kernel_colocated_playwright",
13423
- stealth: true,
13424
- attemptUsed: capture.attempt,
13425
- rawDomStored: Boolean(artifact)
13426
- },
13427
- completedAt: (/* @__PURE__ */ new Date()).toISOString()
13428
- });
13429
- await settlePaaHarvestJob(job.id);
13430
- if (job.callback_url) {
13431
- await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
13432
- }
13433
- return {
13434
- jobId: job.id,
13435
- status: "done",
13436
- totalQuestions: questionCount(result),
13437
- partial: capture.partial,
13438
- artifactStored: Boolean(artifact)
13439
- };
13440
- } catch (error) {
13441
- const message = error instanceof Error ? error.message : String(error);
13442
- const problem = classifyHarvestProblem(error);
13443
- const latestCheckpoint = checkpointState.value;
13444
- await finishHarvestAttempt({
13445
- jobId: job.id,
13446
- attemptNumber: 1,
13447
- outcome: latestCheckpoint?.records.length ? "partial_session_lost" : "failed_before_capture",
13448
- kernelSessionId: null,
13449
- questionCount: latestCheckpoint?.records.length ?? 0,
13450
- durationMs: Date.now() - invocationStartedAt,
13451
- error: message.slice(0, 1e3),
13452
- willRetry: false,
13453
- kernelDeleteStarted: true,
13454
- kernelDeleteSucceeded: null,
13455
- kernelDeleteError: null,
13456
- browserCloseSucceeded: null,
13457
- browserCloseError: null,
13458
- debug: latestCheckpoint,
13459
- completedAt: (/* @__PURE__ */ new Date()).toISOString()
13460
- });
13461
- await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
13462
- await settlePaaHarvestJob(job.id).then(async (settled) => {
13463
- if (!settled) return;
13464
- const settledJob = await getJob(job.id);
13465
- const billedMc = Number(settledJob?.options.billedMc ?? 0);
13466
- await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
13467
- }).catch((settlementError) => {
13468
- console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
13469
- });
13470
- if (job.callback_url) {
13471
- await deliverWebhook(job.callback_url, {
13472
- job_id: job.id,
13473
- status: "failed",
13474
- ...harvestProblemResponse(problem),
13475
- progress: latestCheckpoint
13476
- });
13477
- }
13478
- throw error;
13479
- }
13480
- }
13481
- );
13482
-
13483
12415
  // src/api/local-sourcebook-repository.ts
13484
- import { randomUUID as randomUUID4 } from "crypto";
12416
+ import { randomUUID as randomUUID3 } from "crypto";
13485
12417
  var LOCAL_SOURCEBOOK_CATEGORIES = ["home", "professional", "restaurants", "financial", "realestate", "auto", "wellness"];
13486
12418
  var schemaPromise2 = null;
13487
12419
  var schemaDb2 = null;
@@ -13596,7 +12528,7 @@ function mapSubmission(row) {
13596
12528
  async function event(submissionId, eventType, actorKind, actorId, metadata = {}) {
13597
12529
  await getDb().execute({
13598
12530
  sql: `INSERT INTO local_sourcebook_events (id, submission_id, event_type, actor_kind, actor_id, metadata_json) VALUES (?, ?, ?, ?, ?, ?)`,
13599
- args: [randomUUID4(), submissionId, eventType, actorKind, actorId, JSON.stringify(metadata)]
12531
+ args: [randomUUID3(), submissionId, eventType, actorKind, actorId, JSON.stringify(metadata)]
13600
12532
  });
13601
12533
  }
13602
12534
  async function createLocalSourcebookSubmission(input) {
@@ -13610,7 +12542,7 @@ async function createLocalSourcebookSubmission(input) {
13610
12542
  return submission;
13611
12543
  }
13612
12544
  }
13613
- const id = `lsb_${randomUUID4().replace(/-/g, "")}`;
12545
+ const id = `lsb_${randomUUID3().replace(/-/g, "")}`;
13614
12546
  const coverage = {
13615
12547
  requested: ["website_crawl", "structured_data", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
13616
12548
  crawl: { state: "queued", pagesDiscovered: 0, pagesCaptured: 0 },
@@ -14700,7 +13632,7 @@ async function refundPersistedListingWrite(input) {
14700
13632
  }
14701
13633
 
14702
13634
  // src/api/local-sourcebook-compiler.ts
14703
- import { createHash as createHash3 } from "crypto";
13635
+ import { createHash as createHash2 } from "crypto";
14704
13636
 
14705
13637
  // src/api/local-sourcebook-public-urls.ts
14706
13638
  var DEFAULT_ROOT_DOMAIN = "localsourcebook.com";
@@ -14812,7 +13744,7 @@ var CREDENTIAL_PATH = /\/(certifications?|credentials?|licenses?|accreditation|a
14812
13744
  var BLOG_PATH = /\/(blog|articles?|resources?)\//i;
14813
13745
  var LOCATION_PATH = /\/(?:locations?|service-areas?|areas-we-serve|communities|offices?)(?:\/|$|-)|(?:company|in|near|serving)-[a-z0-9-]+-[a-z]{2}$/i;
14814
13746
  function digest(value) {
14815
- return createHash3("sha256").update(value).digest("hex").slice(0, 20);
13747
+ return createHash2("sha256").update(value).digest("hex").slice(0, 20);
14816
13748
  }
14817
13749
  function unique(values, limit = 100) {
14818
13750
  const seen = /* @__PURE__ */ new Set();
@@ -16129,7 +15061,7 @@ import { Hono as Hono3 } from "hono";
16129
15061
  import { z as z13 } from "zod";
16130
15062
 
16131
15063
  // src/api/unified-billing.ts
16132
- import { createHash as createHash4, createHmac, randomUUID as randomUUID5, timingSafeEqual } from "crypto";
15064
+ import { createHash as createHash3, createHmac, randomUUID as randomUUID4, timingSafeEqual } from "crypto";
16133
15065
 
16134
15066
  // src/api/scheduling-access.ts
16135
15067
  var PAID_PLAN_REQUIRED_CODE = "paid_plan_required";
@@ -16230,7 +15162,7 @@ function tokenFor(id, idempotencyKey4) {
16230
15162
  return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey4).digest("base64url");
16231
15163
  }
16232
15164
  function tokenHash(token6) {
16233
- return createHash4("sha256").update(token6).digest("hex");
15165
+ return createHash3("sha256").update(token6).digest("hex");
16234
15166
  }
16235
15167
  function verifyToken(row, token6) {
16236
15168
  const expected = Buffer.from(row.token_hash, "hex");
@@ -16350,7 +15282,7 @@ async function authorizeScheduledRun(args) {
16350
15282
  { balanceMc, requiredMc: SCHEDULED_RUN_BASE_MC }
16351
15283
  );
16352
15284
  }
16353
- const id = randomUUID5();
15285
+ const id = randomUUID4();
16354
15286
  const token6 = tokenFor(id, idempotencyKey4);
16355
15287
  const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
16356
15288
  const inserted = await getDb().execute({
@@ -16418,7 +15350,7 @@ async function startScheduledRun(args) {
16418
15350
  if (!authorization || !["starting", "started"].includes(authorization.status)) {
16419
15351
  throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer startable", 409);
16420
15352
  }
16421
- const eventId = existing?.id ?? randomUUID5();
15353
+ const eventId = existing?.id ?? randomUUID4();
16422
15354
  if (!existing) {
16423
15355
  await getDb().execute({
16424
15356
  sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, multiplier_bps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'capturing', ?, ?, ?)",
@@ -16557,7 +15489,7 @@ async function settleScheduledRun(args) {
16557
15489
  ...modelCostUnreported ? { modelCostUnreported: true } : {},
16558
15490
  ...pendingReason ? { reason: pendingReason } : {}
16559
15491
  });
16560
- const eventId = existing?.id ?? randomUUID5();
15492
+ const eventId = existing?.id ?? randomUUID4();
16561
15493
  const status = pendingReason ? "cost_pending" : "settling";
16562
15494
  const eventValues = [
16563
15495
  modelMc,
@@ -16828,7 +15760,7 @@ async function syncScheduledActionCredentials(user) {
16828
15760
  }
16829
15761
 
16830
15762
  // src/api/connected-usage-billing.ts
16831
- import { createHash as createHash5, randomUUID as randomUUID6 } from "crypto";
15763
+ import { createHash as createHash4, randomUUID as randomUUID5 } from "crypto";
16832
15764
  import { z as z12 } from "zod";
16833
15765
 
16834
15766
  // src/api/connected-cost-telemetry.ts
@@ -16903,7 +15835,7 @@ var ConnectedUsageBillingError = class extends Error {
16903
15835
  details;
16904
15836
  };
16905
15837
  function hash(value) {
16906
- return createHash5("sha256").update(value).digest("hex");
15838
+ return createHash4("sha256").update(value).digest("hex");
16907
15839
  }
16908
15840
  function eventKey(idempotencyKey4) {
16909
15841
  return `connected-usage:${hash(idempotencyKey4)}`;
@@ -17175,7 +16107,7 @@ async function settleConnectedUsage(rawInput) {
17175
16107
  sql: `INSERT OR IGNORE INTO billing_events
17176
16108
  (id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
17177
16109
  VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
17178
- args: [randomUUID6(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
16110
+ args: [randomUUID5(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
17179
16111
  });
17180
16112
  let event2 = await readEvent(key);
17181
16113
  if (!event2) throw new Error("connected usage receipt insert completed without a readable event");
@@ -20793,7 +19725,7 @@ var adminAuth = createMiddleware3(async (c, next) => {
20793
19725
  import { Hono as Hono9 } from "hono";
20794
19726
 
20795
19727
  // src/api/admin-credits.ts
20796
- import { randomUUID as randomUUID7 } from "crypto";
19728
+ import { randomUUID as randomUUID6 } from "crypto";
20797
19729
  var MIN_ADJUSTMENT_CREDITS = 1;
20798
19730
  var LARGE_ADJUSTMENT_CREDITS = 1e5;
20799
19731
  var MAX_ADJUSTMENT_CREDITS = 1e7;
@@ -20943,7 +19875,7 @@ async function applyAdjustment(input) {
20943
19875
  const credits = validateCredits(input.credits, input.confirmLarge === true);
20944
19876
  const reason = validateReason(input.reason);
20945
19877
  const actor = validateActor(input.actor);
20946
- const reference = input.reference?.trim() || `adj_${randomUUID7()}`;
19878
+ const reference = input.reference?.trim() || `adj_${randomUUID6()}`;
20947
19879
  if (reference.length > 120) throw new AdminCreditError(400, "reference must be at most 120 characters");
20948
19880
  const db = getDb();
20949
19881
  const existing = await db.execute({
@@ -23550,17 +22482,17 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
23550
22482
  });
23551
22483
 
23552
22484
  // src/api/directory-routes.ts
23553
- import { createHash as createHash6, randomUUID as randomUUID9 } from "crypto";
22485
+ import { createHash as createHash5, randomUUID as randomUUID8 } from "crypto";
23554
22486
  import { Hono as Hono16 } from "hono";
23555
22487
 
23556
22488
  // src/api/directory-workflow-dispatch.ts
23557
- import { randomUUID as randomUUID8 } from "crypto";
22489
+ import { randomUUID as randomUUID7 } from "crypto";
23558
22490
  function retryDelaySeconds(attempts) {
23559
22491
  return Math.min(60 * 60, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
23560
22492
  }
23561
22493
  async function dispatchPendingDirectoryWorkflows(limit = 25) {
23562
22494
  const rows = await claimDirectoryWorkflowOutbox({
23563
- workerId: `directory-dispatch-${process.pid}-${randomUUID8().slice(0, 8)}`,
22495
+ workerId: `directory-dispatch-${process.pid}-${randomUUID7().slice(0, 8)}`,
23564
22496
  limit
23565
22497
  });
23566
22498
  const result = { claimed: rows.length, dispatched: 0, failed: 0 };
@@ -23590,7 +22522,7 @@ function safeOptions(options) {
23590
22522
  return safe;
23591
22523
  }
23592
22524
  function requestFingerprint(options) {
23593
- return createHash6("sha256").update(JSON.stringify(safeOptions(options))).digest("hex");
22525
+ return createHash5("sha256").update(JSON.stringify(safeOptions(options))).digest("hex");
23594
22526
  }
23595
22527
  function idempotencyKey(raw) {
23596
22528
  if (raw !== void 0) {
@@ -23599,14 +22531,14 @@ function idempotencyKey(raw) {
23599
22531
  if (trimmed.length > 500) return { ok: false, message: "Idempotency-Key must be 500 characters or fewer." };
23600
22532
  return { ok: true, key: trimmed };
23601
22533
  }
23602
- return { ok: true, key: `directory-${randomUUID9()}` };
22534
+ return { ok: true, key: `directory-${randomUUID8()}` };
23603
22535
  }
23604
22536
  function debitKeyFor(userId, responseKey) {
23605
- const digest2 = createHash6("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
22537
+ const digest2 = createHash5("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
23606
22538
  return `directory-workflow:${userId}:${digest2}`;
23607
22539
  }
23608
22540
  function jobId() {
23609
- return `dir_${randomUUID9().replace(/-/g, "")}`;
22541
+ return `dir_${randomUUID8().replace(/-/g, "")}`;
23610
22542
  }
23611
22543
  function publicStatus(job, result) {
23612
22544
  if (job.status === "failed") return "failed";
@@ -23715,7 +22647,7 @@ async function runSynchronously(c, user, options, plan, responseKey) {
23715
22647
  const csv = renderDirectoryWorkflowCsv(result);
23716
22648
  const artifact = await createDirectoryCsvArtifact({
23717
22649
  ownerId: String(user.id),
23718
- jobId: createHash6("sha256").update(debitKey2).digest("hex").slice(0, 32),
22650
+ jobId: createHash5("sha256").update(debitKey2).digest("hex").slice(0, 32),
23719
22651
  createdAt: result.extractedAt,
23720
22652
  filename: `${options.state}-${options.query}-directory.csv`,
23721
22653
  csv,
@@ -23910,17 +22842,17 @@ directoryApp.get("/jobs/:id", createApiKeyAuth(), async (c) => {
23910
22842
  });
23911
22843
 
23912
22844
  // src/api/lead-list-input-routes.ts
23913
- import { createHash as createHash8 } from "crypto";
22845
+ import { createHash as createHash7 } from "crypto";
23914
22846
  import { Hono as Hono17 } from "hono";
23915
22847
 
23916
22848
  // src/api/lead-list-input-artifacts.ts
23917
- import { createHash as createHash7 } from "crypto";
22849
+ import { createHash as createHash6 } from "crypto";
23918
22850
  import { rm } from "fs/promises";
23919
22851
  import { homedir as homedir3 } from "os";
23920
22852
  import { join as join7 } from "path";
23921
22853
 
23922
22854
  // src/api/lead-list-import-repository.ts
23923
- import { randomUUID as randomUUID10 } from "crypto";
22855
+ import { randomUUID as randomUUID9 } from "crypto";
23924
22856
  var schemaDb3 = null;
23925
22857
  var schemaPromise3 = null;
23926
22858
  function parseJson3(value, fallback) {
@@ -23986,10 +22918,10 @@ function bounded(value, field, min, max) {
23986
22918
  return normalized;
23987
22919
  }
23988
22920
  function newLeadListUploadId() {
23989
- return `upl_${randomUUID10().replace(/-/g, "")}`;
22921
+ return `upl_${randomUUID9().replace(/-/g, "")}`;
23990
22922
  }
23991
22923
  function newImportedLeadListId() {
23992
- return `lst_${randomUUID10().replace(/-/g, "")}`;
22924
+ return `lst_${randomUUID9().replace(/-/g, "")}`;
23993
22925
  }
23994
22926
  async function ensureLeadListImportRepositorySchema() {
23995
22927
  const db = getDb();
@@ -24237,7 +23169,7 @@ function safeFilenameHint(value) {
24237
23169
  return cleaned || null;
24238
23170
  }
24239
23171
  function requestFingerprint2(filenameHint) {
24240
- return createHash7("sha256").update(JSON.stringify({ filenameHint })).digest("hex");
23172
+ return createHash6("sha256").update(JSON.stringify({ filenameHint })).digest("hex");
24241
23173
  }
24242
23174
  function publicBaseUrl() {
24243
23175
  const configured = process.env.MCP_SCRAPER_PUBLIC_BASE_URL?.trim() || process.env.PUBLIC_BASE_URL?.trim() || (process.env.VERCEL_PROJECT_PRODUCTION_URL ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}` : "") || "https://mcpscraper.dev";
@@ -24393,7 +23325,7 @@ async function inspectLeadListUpload(uploadId, ownerId2) {
24393
23325
  maxBytes: LEAD_LIST_UPLOAD_MAX_BYTES
24394
23326
  });
24395
23327
  if (!buffer) return null;
24396
- const sha2564 = createHash7("sha256").update(buffer).digest("hex");
23328
+ const sha2564 = createHash6("sha256").update(buffer).digest("hex");
24397
23329
  const completed = await completeLeadListUpload({
24398
23330
  id: record.id,
24399
23331
  ownerId: record.ownerId,
@@ -24876,7 +23808,7 @@ function canonicalize(value) {
24876
23808
  return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalize(object[key])}`).join(",")}}`;
24877
23809
  }
24878
23810
  function fingerprint(value) {
24879
- return createHash8("sha256").update(canonicalize(value)).digest("hex");
23811
+ return createHash7("sha256").update(canonicalize(value)).digest("hex");
24880
23812
  }
24881
23813
  async function safeJson(c) {
24882
23814
  try {
@@ -25131,17 +24063,17 @@ leadListInputApp.post("/imports", async (c) => {
25131
24063
  });
25132
24064
 
25133
24065
  // src/api/lead-list-enrichment-routes.ts
25134
- import { createHash as createHash9, randomUUID as randomUUID12 } from "crypto";
24066
+ import { createHash as createHash8, randomUUID as randomUUID11 } from "crypto";
25135
24067
  import { Hono as Hono18 } from "hono";
25136
24068
  import { z as z25 } from "zod";
25137
24069
 
25138
24070
  // src/api/lead-list-enrichment-dispatch.ts
25139
- import { randomUUID as randomUUID11 } from "crypto";
24071
+ import { randomUUID as randomUUID10 } from "crypto";
25140
24072
  function retryDelaySeconds2(attempts) {
25141
24073
  return Math.min(3600, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
25142
24074
  }
25143
24075
  async function dispatchPendingLeadListEnrichments(limit = 25) {
25144
- const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${randomUUID11().slice(0, 8)}`, limit });
24076
+ const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${randomUUID10().slice(0, 8)}`, limit });
25145
24077
  const result = { claimed: rows.length, dispatched: 0, failed: 0 };
25146
24078
  for (const row of rows) {
25147
24079
  if (!row.claimToken) continue;
@@ -25189,15 +24121,15 @@ function idempotencyKey2(raw) {
25189
24121
  return { ok: true, value };
25190
24122
  }
25191
24123
  function newJobId() {
25192
- return `lle_${randomUUID12().replace(/-/g, "")}`;
24124
+ return `lle_${randomUUID11().replace(/-/g, "")}`;
25193
24125
  }
25194
24126
  function debitKeyFor2(userId, key) {
25195
- const digest2 = createHash9("sha256").update(String(userId)).update("\0").update(key).digest("hex");
24127
+ const digest2 = createHash8("sha256").update(String(userId)).update("\0").update(key).digest("hex");
25196
24128
  return `lead-list-enrichment:${userId}:${digest2}`;
25197
24129
  }
25198
24130
  function inlineSourceDigest(headers, rows) {
25199
24131
  const values = rows.map((row) => headers.map((header) => row[header] ?? null));
25200
- return createHash9("sha256").update(stableCanonicalJson({ headers, values })).digest("hex");
24132
+ return createHash8("sha256").update(stableCanonicalJson({ headers, values })).digest("hex");
25201
24133
  }
25202
24134
  function issueMessage(error) {
25203
24135
  const issue = error.issues[0];
@@ -26277,43 +25209,230 @@ locationDataApp.post("/admin/sync-census", async (c) => {
26277
25209
  }
26278
25210
  });
26279
25211
 
26280
- // src/api/paa-harvest-dispatch.ts
26281
- async function dispatchPaaHarvest(jobId2) {
26282
- await inngest.send({
26283
- id: `paa-harvest:${jobId2}`,
26284
- name: "mcp-scraper/paa.requested",
26285
- data: { jobId: jobId2 }
25212
+ // src/api/paa-harvest-direct.ts
25213
+ import { waitUntil } from "@vercel/functions";
25214
+
25215
+ // src/api/paa-harvest-settlement.ts
25216
+ function capturedQuestionCount(result) {
25217
+ if (!result || typeof result !== "object") return 0;
25218
+ const value = result;
25219
+ if (typeof value.totalQuestions === "number") return value.totalQuestions;
25220
+ return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
25221
+ }
25222
+ function finalCost(result, heldMc) {
25223
+ const questions = capturedQuestionCount(result);
25224
+ if (questions <= 0) return 0;
25225
+ return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
25226
+ }
25227
+ async function settlePaaHarvestJob(jobOrId) {
25228
+ const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
25229
+ if (!job) return false;
25230
+ const options = job.options;
25231
+ const debitKey2 = options.billingDebitKey;
25232
+ const heldMc = Number(options.billingHoldMc ?? 0);
25233
+ if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
25234
+ const settlement = await settleDebitMcIdempotent(
25235
+ job.user_id,
25236
+ debitKey2,
25237
+ finalCost(job.result, heldMc),
25238
+ LedgerOperation.PAA_REFUND,
25239
+ "durable PAA harvest settlement",
25240
+ "paa_harvest"
25241
+ );
25242
+ await markJobBillingSettled(job.id, settlement.final_amount_mc);
25243
+ return true;
25244
+ }
25245
+
25246
+ // src/api/paa-harvest-runner.ts
25247
+ var CLEANUP_RESERVE_MS = 4e4;
25248
+ var DIRECT_PAA_MAX_ATTEMPTS = 3;
25249
+ function durableRecordsFromResult(result) {
25250
+ const flat = Array.isArray(result.flat) ? result.flat : [];
25251
+ return flat.flatMap((value) => {
25252
+ if (!value || typeof value !== "object") return [];
25253
+ const row = value;
25254
+ const question = typeof row.question === "string" ? row.question : "";
25255
+ if (!question) return [];
25256
+ const sourceCite = typeof row.source_cite === "string" ? row.source_cite : "";
25257
+ const sourceTitle = typeof row.source_title === "string" ? row.source_title : "";
25258
+ const sourceSite = typeof row.source_site === "string" ? row.source_site : "";
25259
+ return [{
25260
+ question,
25261
+ answer: typeof row.answer === "string" ? row.answer : "",
25262
+ sourceTitle,
25263
+ sourceSite,
25264
+ sourceCite,
25265
+ sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
25266
+ depth: typeof row.depth === "number" ? row.depth : 1,
25267
+ parentQuestion: typeof row.parent_question === "string" && row.parent_question ? row.parent_question : null
25268
+ }];
26286
25269
  });
26287
25270
  }
26288
- async function redispatchPendingPaaHarvests(limit = 25) {
26289
- const jobs = await listPendingInngestPaaJobs(limit);
26290
- let dispatched = 0;
26291
- let failed = 0;
26292
- for (const job of jobs) {
26293
- try {
26294
- await dispatchPaaHarvest(job.id);
26295
- dispatched += 1;
26296
- } catch {
26297
- failed += 1;
25271
+ function questionCount(result) {
25272
+ if (!result || typeof result !== "object") return 0;
25273
+ const value = result;
25274
+ if (typeof value.totalQuestions === "number") return value.totalQuestions;
25275
+ return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
25276
+ }
25277
+ async function runPaaHarvestJob(jobId2) {
25278
+ const invocationStartedAt = Date.now();
25279
+ const job = await claimDurablePaaJob(jobId2);
25280
+ if (!job) return { jobId: jobId2, status: "missing_or_not_paa" };
25281
+ if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
25282
+ await settlePaaHarvestJob(job.id);
25283
+ return { jobId: job.id, status: job.status };
25284
+ }
25285
+ const options = job.options;
25286
+ const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
25287
+ const timeoutBudget = harvestTimeoutBudget(maxQuestions);
25288
+ const workBudgetMs = Math.max(1, timeoutBudget.serverMs - CLEANUP_RESERVE_MS);
25289
+ const checkpointState = { value: null };
25290
+ try {
25291
+ if (!brightDataSerpEnabled()) throw new Error("SERP browser service is not configured");
25292
+ const navigating = {
25293
+ phase: "navigating",
25294
+ attempt: 1,
25295
+ observedAt: (/* @__PURE__ */ new Date()).toISOString(),
25296
+ records: [],
25297
+ clickedQuestions: [],
25298
+ finalUrl: null,
25299
+ captchaObserved: false,
25300
+ solverCleared: false,
25301
+ error: null
25302
+ };
25303
+ checkpointState.value = navigating;
25304
+ if (!await checkpointJob(job.id, {
25305
+ progress: navigating,
25306
+ durable: {
25307
+ invocationBudgetMs: timeoutBudget.serverMs,
25308
+ workBudgetMs,
25309
+ cleanupReserveMs: CLEANUP_RESERVE_MS
25310
+ }
25311
+ })) {
25312
+ throw new Error("job_left_running_state_during_capture");
25313
+ }
25314
+ const persistAttempt = createHarvestAttemptRecorder(job.id, job.user_id);
25315
+ let lastAttempt = 1;
25316
+ const onAttemptEvent = async (event2) => {
25317
+ lastAttempt = event2.attemptNumber;
25318
+ await persistAttempt(event2);
25319
+ };
25320
+ const harvestCtx = currentCostContext();
25321
+ const harvested = await runWithCostContext(
25322
+ { ...harvestCtx, op: "paa", userId: job.user_id },
25323
+ () => harvest({
25324
+ query: options.query ?? job.query,
25325
+ location: options.location ?? void 0,
25326
+ gl: options.gl ?? "us",
25327
+ hl: options.hl ?? "en",
25328
+ maxQuestions,
25329
+ maxAttempts: DIRECT_PAA_MAX_ATTEMPTS,
25330
+ proxyMode: "none",
25331
+ forceManagedSerp: true,
25332
+ headless: true,
25333
+ format: "json",
25334
+ outputDir: "/tmp/paa-output-direct",
25335
+ includeAllSerpFeatures: options.includeAllSerpFeatures ?? false,
25336
+ includeLocalPack: options.includeLocalPack ?? false,
25337
+ includeForums: options.includeForums ?? false,
25338
+ includeVideos: options.includeVideos ?? false,
25339
+ includeAiOverview: options.includeAiOverview ?? false,
25340
+ includeWhatPeopleSaying: options.includeWhatPeopleSaying ?? false,
25341
+ softDeadlineMs: invocationStartedAt + workBudgetMs,
25342
+ onAttemptEvent
25343
+ })
25344
+ );
25345
+ const harvestResult = harvested;
25346
+ const records = durableRecordsFromResult(harvestResult);
25347
+ const captured = {
25348
+ ...navigating,
25349
+ phase: "captured",
25350
+ attempt: lastAttempt,
25351
+ observedAt: (/* @__PURE__ */ new Date()).toISOString(),
25352
+ records,
25353
+ clickedQuestions: records.map((record) => record.question)
25354
+ };
25355
+ checkpointState.value = captured;
25356
+ if (!await checkpointJob(job.id, {
25357
+ progress: captured,
25358
+ durable: {
25359
+ invocationBudgetMs: timeoutBudget.serverMs,
25360
+ workBudgetMs,
25361
+ cleanupReserveMs: CLEANUP_RESERVE_MS
25362
+ }
25363
+ })) {
25364
+ throw new Error("job_left_running_state_during_capture");
25365
+ }
25366
+ const result = {
25367
+ ...harvestResult,
25368
+ capture: {
25369
+ ...harvestResult.capture,
25370
+ execution: "brightdata_browser_api_controller_fulfilled",
25371
+ provider: "bright_data_browser_api",
25372
+ sameQueryClickGraph: true,
25373
+ rawDomArtifact: null,
25374
+ rawDomArtifactStatus: "unavailable"
25375
+ }
25376
+ };
25377
+ await completeJob(job.id, result);
25378
+ await settlePaaHarvestJob(job.id);
25379
+ if (job.callback_url) {
25380
+ await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
26298
25381
  }
25382
+ return {
25383
+ jobId: job.id,
25384
+ status: "done",
25385
+ totalQuestions: questionCount(result),
25386
+ partial: records.length < maxQuestions || records.some((record) => !record.answer || !record.sourceCite),
25387
+ artifactStored: false
25388
+ };
25389
+ } catch (error) {
25390
+ const problem = classifyHarvestProblem(error);
25391
+ const latestCheckpoint = checkpointState.value;
25392
+ await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
25393
+ await settlePaaHarvestJob(job.id).then(async (settled) => {
25394
+ if (!settled) return;
25395
+ const settledJob = await getJob(job.id);
25396
+ const billedMc = Number(settledJob?.options.billedMc ?? 0);
25397
+ await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
25398
+ }).catch((settlementError) => {
25399
+ console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
25400
+ });
25401
+ if (job.callback_url) {
25402
+ await deliverWebhook(job.callback_url, {
25403
+ job_id: job.id,
25404
+ status: "failed",
25405
+ ...harvestProblemResponse(problem),
25406
+ progress: latestCheckpoint
25407
+ });
25408
+ }
25409
+ throw error;
26299
25410
  }
26300
- return { checked: jobs.length, dispatched, failed };
25411
+ }
25412
+
25413
+ // src/api/paa-harvest-direct.ts
25414
+ function scheduleDirectPaaHarvest(jobId2) {
25415
+ const work = runPaaHarvestJob(jobId2).catch((error) => {
25416
+ console.error("[paa-harvest/direct] run failed:", error instanceof Error ? error.message : String(error));
25417
+ });
25418
+ waitUntil(work);
26301
25419
  }
26302
25420
 
26303
25421
  // src/api/paa-harvest-reconciliation.ts
26304
25422
  async function reconcilePaaHarvestSettlements(limit = 25) {
26305
- const stale = await listStaleRunningInngestPaaJobs(limit);
25423
+ const stale = await listStaleRunningDurablePaaJobs(limit);
26306
25424
  let staleFailed = 0;
26307
25425
  for (const job of stale) {
26308
25426
  try {
26309
- const privateError = "durable PAA invocation exceeded its recovery window; preserved progress remains in result";
25427
+ const privateError = "direct PAA invocation exceeded its platform safety window; preserved progress remains in result";
26310
25428
  const problem = classifyHarvestProblem(new Error("harvest timeout"));
26311
- await failJob(job.id, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
26312
- staleFailed += 1;
25429
+ if (await failRunningJob(job.id, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }))) {
25430
+ staleFailed += 1;
25431
+ }
26313
25432
  } catch {
26314
25433
  }
26315
25434
  }
26316
- const jobs = await listRecentTerminalInngestPaaJobs(limit);
25435
+ const jobs = await listRecentTerminalDurablePaaJobs(limit);
26317
25436
  let settled = 0;
26318
25437
  let failed = 0;
26319
25438
  for (const job of jobs) {
@@ -27012,17 +26131,17 @@ workflowApp.post("/cron/dispatch", async (c) => {
27012
26131
 
27013
26132
  // src/api/serp-intelligence-routes.ts
27014
26133
  import { Hono as Hono22 } from "hono";
27015
- import { createHash as createHash11, randomUUID as randomUUID13 } from "crypto";
26134
+ import { createHash as createHash10, randomUUID as randomUUID12 } from "crypto";
27016
26135
 
27017
26136
  // src/serp-intelligence/page-snapshot-extractor.ts
27018
- import { createHash as createHash10 } from "crypto";
26137
+ import { createHash as createHash9 } from "crypto";
27019
26138
  import pLimit5 from "p-limit";
27020
26139
  var DEFAULT_TIMEOUT_MS = 15e3;
27021
26140
  var DEFAULT_MAX_CONCURRENCY = 2;
27022
26141
  var DEFAULT_MAX_CONTENT_CHARS = 25e4;
27023
26142
  function sha256(value) {
27024
26143
  if (!value) return null;
27025
- return createHash10("sha256").update(value).digest("hex");
26144
+ return createHash9("sha256").update(value).digest("hex");
27026
26145
  }
27027
26146
  function countWords(markdown) {
27028
26147
  const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
@@ -27798,9 +26917,9 @@ async function logRequestEventBestEffort(input) {
27798
26917
  }
27799
26918
  }
27800
26919
  function captureBillingKeys(userId, suppliedKey, body) {
27801
- const responseKey = suppliedKey?.trim() || randomUUID13();
27802
- const requestFingerprint3 = createHash11("sha256").update(JSON.stringify(body)).digest("hex");
27803
- const keyDigest = createHash11("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
26920
+ const responseKey = suppliedKey?.trim() || randomUUID12();
26921
+ const requestFingerprint3 = createHash10("sha256").update(JSON.stringify(body)).digest("hex");
26922
+ const keyDigest = createHash10("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
27804
26923
  return {
27805
26924
  responseKey,
27806
26925
  debitKey: `serp-capture:${userId}:${keyDigest}`,
@@ -28128,7 +27247,7 @@ var MemoryMcpToolExecutor = class {
28128
27247
  };
28129
27248
 
28130
27249
  // src/api/image-source-artifacts.ts
28131
- import { randomUUID as randomUUID14 } from "crypto";
27250
+ import { randomUUID as randomUUID13 } from "crypto";
28132
27251
  var IMAGE_SOURCE_ARTIFACT_PREFIX = "image-sources/";
28133
27252
  var IMAGE_SOURCE_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
28134
27253
  var IMAGE_SOURCE_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
@@ -28159,7 +27278,7 @@ async function createImageSourceArtifact(args) {
28159
27278
  if (args.content.length === 0 || args.content.length > IMAGE_SOURCE_MAX_BYTES) {
28160
27279
  throw new Error("image_source_size_invalid");
28161
27280
  }
28162
- const id = randomUUID14().replaceAll("-", "");
27281
+ const id = randomUUID13().replaceAll("-", "");
28163
27282
  return createPrivateArtifact({
28164
27283
  policy: policy3(),
28165
27284
  ownerId: args.ownerId,
@@ -28908,7 +28027,7 @@ import { randomUUID as randomUUID17 } from "crypto";
28908
28027
  import { Hono as Hono24 } from "hono";
28909
28028
 
28910
28029
  // src/api/browser-agent-db.ts
28911
- import { randomUUID as randomUUID15 } from "crypto";
28030
+ import { randomUUID as randomUUID14 } from "crypto";
28912
28031
  var _ready = false;
28913
28032
  var _migrationPromise = null;
28914
28033
  async function migrateBrowserAgent() {
@@ -29039,7 +28158,7 @@ async function runBrowserAgentMigration() {
29039
28158
  }
29040
28159
  async function createExtensionRow(input) {
29041
28160
  const db = getDb();
29042
- const id = `bext_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
28161
+ const id = `bext_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
29043
28162
  await db.execute({
29044
28163
  sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
29045
28164
  VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
@@ -29074,7 +28193,7 @@ async function deleteExtensionRow(userId, name) {
29074
28193
  }
29075
28194
  async function createAuthConnectionRow(input) {
29076
28195
  const db = getDb();
29077
- const connectionId = `authc_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
28196
+ const connectionId = `authc_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
29078
28197
  await db.execute({
29079
28198
  sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
29080
28199
  VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
@@ -29159,7 +28278,7 @@ async function deleteProfileLabel(userId, profile) {
29159
28278
  }
29160
28279
  async function createSessionRow(input) {
29161
28280
  const db = getDb();
29162
- const id = `bas_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
28281
+ const id = `bas_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
29163
28282
  await db.execute({
29164
28283
  sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
29165
28284
  VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
@@ -29237,7 +28356,7 @@ async function recordAction(input) {
29237
28356
  sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
29238
28357
  VALUES (?, ?, ?, ?, ?, ?)`,
29239
28358
  args: [
29240
- `baa_${randomUUID15().replace(/-/g, "").slice(0, 20)}`,
28359
+ `baa_${randomUUID14().replace(/-/g, "").slice(0, 20)}`,
29241
28360
  input.sessionId,
29242
28361
  input.type,
29243
28362
  input.params == null ? null : JSON.stringify(input.params),
@@ -29279,7 +28398,7 @@ async function listReplayRows(sessionId) {
29279
28398
  }
29280
28399
 
29281
28400
  // src/services/browser-agent/browser-agent-service.ts
29282
- import Kernel5 from "@onkernel/sdk";
28401
+ import Kernel4 from "@onkernel/sdk";
29283
28402
  import { chromium as playwrightChromium } from "playwright";
29284
28403
 
29285
28404
  // src/services/fanout/cdp-capture.ts
@@ -29988,7 +29107,7 @@ var DEFAULT_TIMEOUT_SECONDS = 600;
29988
29107
  function client() {
29989
29108
  const apiKey = browserServiceApiKey();
29990
29109
  if (!apiKey) throw new Error("Browser backend API key is required");
29991
- return new Kernel5({ apiKey });
29110
+ return new Kernel4({ apiKey });
29992
29111
  }
29993
29112
  function isProfileConflict(err) {
29994
29113
  const message = err instanceof Error ? err.message : String(err);
@@ -30347,17 +29466,66 @@ async function replayList(runtimeSessionId) {
30347
29466
  }));
30348
29467
  }
30349
29468
 
29469
+ // src/api/serp-identity-db.ts
29470
+ import { randomUUID as randomUUID15 } from "crypto";
29471
+ async function createSerpIdentityRow(input) {
29472
+ const id = `serpi_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
29473
+ await getDb().execute({
29474
+ sql: `INSERT INTO serp_identities
29475
+ (id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
29476
+ VALUES (?, ?, ?, ?, ?, 'isp', ?, 'ready')`,
29477
+ args: [id, input.userId, input.name, input.kernelProfileName, input.kernelProxyId, input.country]
29478
+ });
29479
+ const row = await getSerpIdentityRow(input.userId, input.name);
29480
+ if (!row) throw new Error("SERP identity insert failed");
29481
+ return row;
29482
+ }
29483
+ async function getSerpIdentityRow(userId, name) {
29484
+ const result = await getDb().execute({
29485
+ sql: `SELECT * FROM serp_identities WHERE user_id = ? AND name = ? LIMIT 1`,
29486
+ args: [userId, name]
29487
+ });
29488
+ return result.rows[0] ?? null;
29489
+ }
29490
+ async function listSerpIdentityRows(userId) {
29491
+ const result = await getDb().execute({
29492
+ sql: `SELECT * FROM serp_identities WHERE user_id = ? ORDER BY created_at DESC`,
29493
+ args: [userId]
29494
+ });
29495
+ return result.rows;
29496
+ }
29497
+ async function touchSerpIdentity(userId, name) {
29498
+ await getDb().execute({
29499
+ sql: `UPDATE serp_identities
29500
+ SET last_used_at = datetime('now'), updated_at = datetime('now')
29501
+ WHERE user_id = ? AND name = ?`,
29502
+ args: [userId, name]
29503
+ });
29504
+ }
29505
+ async function setSerpIdentityStatus(userId, name, status) {
29506
+ await getDb().execute({
29507
+ sql: `UPDATE serp_identities SET status = ?, updated_at = datetime('now') WHERE user_id = ? AND name = ?`,
29508
+ args: [status, userId, name]
29509
+ });
29510
+ }
29511
+ async function deleteSerpIdentityRow(userId, name) {
29512
+ await getDb().execute({
29513
+ sql: `DELETE FROM serp_identities WHERE user_id = ? AND name = ?`,
29514
+ args: [userId, name]
29515
+ });
29516
+ }
29517
+
30350
29518
  // src/api/serp-identity-service.ts
30351
- import { createHash as createHash12, randomUUID as randomUUID16 } from "crypto";
30352
- import Kernel6 from "@onkernel/sdk";
29519
+ import { createHash as createHash11, randomUUID as randomUUID16 } from "crypto";
29520
+ import Kernel5 from "@onkernel/sdk";
30353
29521
  var MAX_SERP_IDENTITIES_PER_USER = 3;
30354
29522
  function kernelClient() {
30355
29523
  const apiKey = browserServiceApiKey();
30356
29524
  if (!apiKey) throw new Error("Browser backend API key is required");
30357
- return new Kernel6({ apiKey });
29525
+ return new Kernel5({ apiKey });
30358
29526
  }
30359
29527
  function backendName(userId, name, resource) {
30360
- const digest2 = createHash12("sha256").update(`${userId}:${name}`).digest("hex").slice(0, 16);
29528
+ const digest2 = createHash11("sha256").update(`${userId}:${name}`).digest("hex").slice(0, 16);
30361
29529
  const nonce = randomUUID16().replace(/-/g, "").slice(0, 8);
30362
29530
  return `mcp-serp-${resource}-${digest2}-${nonce}`;
30363
29531
  }
@@ -31697,7 +30865,7 @@ async function resolveUser3(customerId, emailFallback) {
31697
30865
  // src/api/oauth-routes.ts
31698
30866
  import { Hono as Hono26 } from "hono";
31699
30867
  import { getCookie, setCookie } from "hono/cookie";
31700
- import { createHash as createHash13, randomBytes as randomBytes3, randomUUID as randomUUID18 } from "crypto";
30868
+ import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as randomUUID18 } from "crypto";
31701
30869
  import { importPKCS8, exportJWK, SignJWT } from "jose";
31702
30870
  var ISSUER = "https://mcpscraper.dev";
31703
30871
  var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
@@ -31755,7 +30923,7 @@ async function getKeys() {
31755
30923
  const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
31756
30924
  const full = await exportJWK(privateKey);
31757
30925
  const publicJwk = { kty: full.kty, n: full.n, e: full.e };
31758
- const kid = createHash13("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
30926
+ const kid = createHash12("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
31759
30927
  publicJwk.kid = kid;
31760
30928
  publicJwk.alg = "RS256";
31761
30929
  publicJwk.use = "sig";
@@ -32077,7 +31245,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
32077
31245
  });
32078
31246
  function pkceMatches(verifier, challenge) {
32079
31247
  if (!verifier) return false;
32080
- const computed = createHash13("sha256").update(verifier).digest("base64url");
31248
+ const computed = createHash12("sha256").update(verifier).digest("base64url");
32081
31249
  return computed === challenge;
32082
31250
  }
32083
31251
  async function mintAccessToken(identity, scope, plan, audience) {
@@ -33530,7 +32698,7 @@ import { Hono as Hono30 } from "hono";
33530
32698
  import { Resend } from "resend";
33531
32699
 
33532
32700
  // src/api/resend-support-thread.ts
33533
- import { createHash as createHash14 } from "crypto";
32701
+ import { createHash as createHash13 } from "crypto";
33534
32702
  var SUPPORT_TAGS = [
33535
32703
  {
33536
32704
  tag: "support",
@@ -33570,7 +32738,7 @@ function quoteUntrusted(value) {
33570
32738
  return clean2.split("\n").map((line) => `> ${line}`).join("\n");
33571
32739
  }
33572
32740
  function shortHash(value, length = 24) {
33573
- return createHash14("sha256").update(value).digest("hex").slice(0, length);
32741
+ return createHash13("sha256").update(value).digest("hex").slice(0, length);
33574
32742
  }
33575
32743
  function safePathPart(value) {
33576
32744
  return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "contact";
@@ -35069,7 +34237,7 @@ import { Hono as Hono32 } from "hono";
35069
34237
  import { z as z32 } from "zod";
35070
34238
 
35071
34239
  // src/editorial-reading-room/render.ts
35072
- import { createHash as createHash15 } from "crypto";
34240
+ import { createHash as createHash14 } from "crypto";
35073
34241
  import { readFileSync as readFileSync3 } from "fs";
35074
34242
  import { dirname as dirname3, join as join8, resolve } from "path";
35075
34243
  import { Marked, Renderer } from "marked";
@@ -35230,7 +34398,7 @@ function renderEditorialReadingRoom(input, now = /* @__PURE__ */ new Date()) {
35230
34398
  articleCount: articles.length,
35231
34399
  wordCount: totalWordCount,
35232
34400
  bytes,
35233
- sha256: createHash15("sha256").update(html).digest("hex"),
34401
+ sha256: createHash14("sha256").update(html).digest("hex"),
35234
34402
  warnings
35235
34403
  };
35236
34404
  }
@@ -35313,7 +34481,7 @@ ${provenance}`;
35313
34481
  ...rendered,
35314
34482
  html,
35315
34483
  bytes: Buffer.byteLength(html),
35316
- sha256: createHash15("sha256").update(html).digest("hex")
34484
+ sha256: createHash14("sha256").update(html).digest("hex")
35317
34485
  };
35318
34486
  }
35319
34487
 
@@ -35436,10 +34604,10 @@ import { Hono as Hono33 } from "hono";
35436
34604
  import { z as z33 } from "zod";
35437
34605
 
35438
34606
  // src/api/commons-repository.ts
35439
- import { createHash as createHash18, randomUUID as randomUUID20 } from "crypto";
34607
+ import { createHash as createHash17, randomUUID as randomUUID20 } from "crypto";
35440
34608
 
35441
34609
  // src/api/commons-image-store.ts
35442
- import { createHash as createHash16 } from "crypto";
34610
+ import { createHash as createHash15 } from "crypto";
35443
34611
  import { lookup as dnsLookup } from "dns/promises";
35444
34612
  import { isIP } from "net";
35445
34613
  var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
@@ -35578,7 +34746,7 @@ async function hostCommonsImage(input) {
35578
34746
  415
35579
34747
  );
35580
34748
  }
35581
- const digest2 = createHash16("sha256").update(bytes).digest("hex");
34749
+ const digest2 = createHash15("sha256").update(bytes).digest("hex");
35582
34750
  const existing = await getDb().execute({
35583
34751
  sql: "SELECT id, url, content_type, bytes, source_url FROM commons_images WHERE digest = ? LIMIT 1",
35584
34752
  args: [digest2]
@@ -35693,7 +34861,7 @@ async function schemaObjectsPresent(names) {
35693
34861
  }
35694
34862
 
35695
34863
  // src/api/commons-linksets.ts
35696
- import { createHash as createHash17 } from "crypto";
34864
+ import { createHash as createHash16 } from "crypto";
35697
34865
  var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
35698
34866
  var COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
35699
34867
  var REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
@@ -35800,7 +34968,7 @@ function buildCommonsLinkset(entity, claims) {
35800
34968
  context[claim.predicate] = targets;
35801
34969
  }
35802
34970
  const document2 = { linkset: [context] };
35803
- const etag = `"${createHash17("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
34971
+ const etag = `"${createHash16("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
35804
34972
  return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
35805
34973
  }
35806
34974
  function commonsLinksetDiscoveryHeader(idOrSlug) {
@@ -37582,7 +36750,7 @@ function commonsIndexDocuments(entity, now) {
37582
36750
  return documents;
37583
36751
  }
37584
36752
  function commonsIndexDocumentId(entityId, documentType, documentKey) {
37585
- return `commons-index-${createHash18("sha256").update(`${entityId}
36753
+ return `commons-index-${createHash17("sha256").update(`${entityId}
37586
36754
  ${documentType}
37587
36755
  ${documentKey}`).digest("hex").slice(0, 32)}`;
37588
36756
  }
@@ -38160,7 +37328,7 @@ var CommonsRepositoryError = class extends Error {
38160
37328
  };
38161
37329
 
38162
37330
  // src/api/commons-publication-repository.ts
38163
- import { createHash as createHash19, randomUUID as randomUUID21 } from "crypto";
37331
+ import { createHash as createHash18, randomUUID as randomUUID21 } from "crypto";
38164
37332
 
38165
37333
  // src/editorial-reading-room/article-page.ts
38166
37334
  function escapeHtml6(value) {
@@ -38449,7 +37617,7 @@ async function publishCommonsEditorial(input, user) {
38449
37617
  JSON.stringify(editionInput.articles),
38450
37618
  html,
38451
37619
  rendered.filename,
38452
- createHash19("sha256").update(html).digest("hex"),
37620
+ createHash18("sha256").update(html).digest("hex"),
38453
37621
  rendered.articleCount,
38454
37622
  rendered.wordCount,
38455
37623
  Buffer.byteLength(html),
@@ -38531,7 +37699,7 @@ async function updateCommonsEditorialArticle(input, user) {
38531
37699
  JSON.stringify(nextArticles),
38532
37700
  html,
38533
37701
  rendered.filename,
38534
- createHash19("sha256").update(html).digest("hex"),
37702
+ createHash18("sha256").update(html).digest("hex"),
38535
37703
  rendered.articleCount,
38536
37704
  rendered.wordCount,
38537
37705
  Buffer.byteLength(html),
@@ -39266,12 +38434,12 @@ function xmlEscape(value) {
39266
38434
  import { Hono as Hono34 } from "hono";
39267
38435
  import { createMiddleware as createMiddleware4 } from "hono/factory";
39268
38436
  import { z as z35 } from "zod";
39269
- import { createHash as createHash21 } from "crypto";
38437
+ import { createHash as createHash20 } from "crypto";
39270
38438
  import Papa4 from "papaparse";
39271
38439
 
39272
38440
  // src/api/analytics-entitlement.ts
39273
38441
  import {
39274
- createHash as createHash20,
38442
+ createHash as createHash19,
39275
38443
  createHmac as createHmac3,
39276
38444
  randomBytes as randomBytes4,
39277
38445
  timingSafeEqual as timingSafeEqual2
@@ -39486,7 +38654,7 @@ function dashboardCallbackUrl() {
39486
38654
  async function createThorbitConnectUrl(userId) {
39487
38655
  if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
39488
38656
  const state = randomBytes4(32).toString("base64url");
39489
- const stateHash = createHash20("sha256").update(state).digest("hex");
38657
+ const stateHash = createHash19("sha256").update(state).digest("hex");
39490
38658
  await getAnalyticsPool().query(
39491
38659
  `INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
39492
38660
  VALUES ($1,$2,now()+interval '10 minutes')
@@ -39515,7 +38683,7 @@ function verifyThorbitAssertion(token6) {
39515
38683
  return parsed.data.entitlement;
39516
38684
  }
39517
38685
  async function consumeThorbitConnectCallback(input) {
39518
- const stateHash = createHash20("sha256").update(input.state).digest("hex");
38686
+ const stateHash = createHash19("sha256").update(input.state).digest("hex");
39519
38687
  const client2 = await getAnalyticsPool().connect();
39520
38688
  try {
39521
38689
  await client2.query("BEGIN");
@@ -39991,7 +39159,7 @@ analyticsApp.post("/forms/:publicId/submissions", async (c) => {
39991
39159
  const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
39992
39160
  const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
39993
39161
  const identitySeed = email || `${fullName}:${Date.now()}`;
39994
- const suffix2 = createHash21("sha256").update(identitySeed).digest("hex").slice(0, 12);
39162
+ const suffix2 = createHash20("sha256").update(identitySeed).digest("hex").slice(0, 12);
39995
39163
  const path5 = `Leads/person-${suffix2}`;
39996
39164
  const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
39997
39165
  const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path5 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
@@ -40090,9 +39258,9 @@ Submitted the published form.
40090
39258
  occurredAt: (/* @__PURE__ */ new Date()).toISOString()
40091
39259
  });
40092
39260
  const match = {
40093
- ...email ? { emailSha256: createHash21("sha256").update(email).digest("hex") } : {},
39261
+ ...email ? { emailSha256: createHash20("sha256").update(email).digest("hex") } : {},
40094
39262
  ...phone ? {
40095
- phoneSha256: createHash21("sha256").update(phone.replace(/\D/g, "")).digest("hex")
39263
+ phoneSha256: createHash20("sha256").update(phone.replace(/\D/g, "")).digest("hex")
40096
39264
  } : {}
40097
39265
  };
40098
39266
  const activationQueued = await queueAnalyticsActivation({
@@ -40520,7 +39688,7 @@ analyticsApp.post("/sites/:siteId/crm-imports", auth3, async (c) => {
40520
39688
  rejectedCount += 1;
40521
39689
  continue;
40522
39690
  }
40523
- const digest2 = createHash21("sha256").update(
39691
+ const digest2 = createHash20("sha256").update(
40524
39692
  `${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
40525
39693
  ).digest("hex").slice(0, 16);
40526
39694
  const path5 = `Leads/person-${digest2}`;
@@ -40971,13 +40139,13 @@ import { Hono as Hono35 } from "hono";
40971
40139
  import { z as z36 } from "zod";
40972
40140
 
40973
40141
  // src/api/scheduled-artifact-owner.ts
40974
- import { createHash as createHash22 } from "crypto";
40142
+ import { createHash as createHash21 } from "crypto";
40975
40143
  function scheduledArtifactOwnerIdForApiKey(apiKey) {
40976
- return createHash22("sha256").update(apiKey).digest("hex").slice(0, 24);
40144
+ return createHash21("sha256").update(apiKey).digest("hex").slice(0, 24);
40977
40145
  }
40978
40146
 
40979
40147
  // src/api/scheduled-run-view-links.ts
40980
- import { createHash as createHash23, randomBytes as randomBytes5, randomUUID as randomUUID22 } from "crypto";
40148
+ import { createHash as createHash22, randomBytes as randomBytes5, randomUUID as randomUUID22 } from "crypto";
40981
40149
  var schemaReady2 = false;
40982
40150
  async function ensureScheduledRunViewLinksSchema() {
40983
40151
  if (schemaReady2) return;
@@ -41031,7 +40199,7 @@ async function ensureScheduledRunViewLinksSchema() {
41031
40199
  schemaReady2 = true;
41032
40200
  }
41033
40201
  function tokenHash2(token6) {
41034
- return createHash23("sha256").update(token6).digest("hex");
40202
+ return createHash22("sha256").update(token6).digest("hex");
41035
40203
  }
41036
40204
  function mapRow(row) {
41037
40205
  return {
@@ -41118,7 +40286,7 @@ async function revokeScheduledRunViewLink(ownerId2, runId, shareId, now = /* @__
41118
40286
  }
41119
40287
 
41120
40288
  // src/personal-authority/render.ts
41121
- import { createHash as createHash24 } from "crypto";
40289
+ import { createHash as createHash23 } from "crypto";
41122
40290
  function escapeHtml8(value) {
41123
40291
  return value.replace(/[&<>"']/g, (character) => ({
41124
40292
  "&": "&amp;",
@@ -41178,13 +40346,13 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
41178
40346
  html,
41179
40347
  filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
41180
40348
  bytes,
41181
- sha256: createHash24("sha256").update(html).digest("hex"),
40349
+ sha256: createHash23("sha256").update(html).digest("hex"),
41182
40350
  generatedAt: generatedAtIso
41183
40351
  };
41184
40352
  }
41185
40353
 
41186
40354
  // src/personal-authority/render-v2.ts
41187
- import { createHash as createHash25 } from "crypto";
40355
+ import { createHash as createHash24 } from "crypto";
41188
40356
  var FONT_STACKS = Object.freeze({
41189
40357
  "editorial-serif": "Iowan Old Style, Baskerville, Times New Roman, serif",
41190
40358
  "modern-sans": "Avenir Next, Avenir, Helvetica Neue, Arial, sans-serif",
@@ -41288,13 +40456,13 @@ ${section2(flags.finalCta && Boolean(finalCta), `<section class="section cta" id
41288
40456
  html,
41289
40457
  filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
41290
40458
  bytes,
41291
- sha256: createHash25("sha256").update(html).digest("hex"),
40459
+ sha256: createHash24("sha256").update(html).digest("hex"),
41292
40460
  generatedAt: generatedAtIso
41293
40461
  };
41294
40462
  }
41295
40463
 
41296
40464
  // src/newsroom-publisher/render.ts
41297
- import { createHash as createHash26 } from "crypto";
40465
+ import { createHash as createHash25 } from "crypto";
41298
40466
  function escapeHtml10(value) {
41299
40467
  return value.replace(/[&<>"']/g, (character) => ({
41300
40468
  "&": "&amp;",
@@ -41400,13 +40568,13 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
41400
40568
  @media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
41401
40569
  </style></head><body data-theme="${escapeHtml10(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
41402
40570
  const bytes = Buffer.byteLength(html);
41403
- const sha2564 = createHash26("sha256").update(html).digest("hex");
40571
+ const sha2564 = createHash25("sha256").update(html).digest("hex");
41404
40572
  const filename2 = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
41405
40573
  return { html, filename: filename2, bytes, sha256: sha2564, generatedAt: generatedAtIso };
41406
40574
  }
41407
40575
 
41408
40576
  // src/blog-article/render.ts
41409
- import { createHash as createHash27 } from "crypto";
40577
+ import { createHash as createHash26 } from "crypto";
41410
40578
  function escapeHtml11(value) {
41411
40579
  return value.replace(
41412
40580
  /[&<>"']/g,
@@ -41494,7 +40662,7 @@ function renderBlogArticleV1(input, config, generatedAt) {
41494
40662
  :root{--ink:#10213d;--ink2:#1c355a;--blue:#2563eb;--blue2:#1749b6;--mint:#45d5aa;--paper:#f7f5f0;--line:#dce4ef;--card:#fff;--body:#475569;--serif:Iowan Old Style,Palatino Linotype,Palatino,Georgia,serif;--sans:Inter,Avenir Next,ui-sans-serif,system-ui,sans-serif}body[data-theme="slate"]{--ink:#1e293b;--ink2:#334155;--blue:#475569;--blue2:#1e293b;--mint:#94a3b8;--paper:#f8fafc}body[data-theme="forest"]{--ink:#12352e;--ink2:#23584c;--blue:#147d64;--blue2:#0c5b48;--mint:#79d8bd;--paper:#f4f8f3}*{box-sizing:border-box}html{scroll-behavior:smooth;background:var(--paper)}body{margin:0;color:#252a33;background:var(--paper);font-family:var(--sans);line-height:1.6}a{color:inherit}button{font:inherit}button:focus-visible,a:focus-visible{outline:3px solid var(--mint);outline-offset:3px}.progress{position:fixed;z-index:50;top:0;left:0;height:4px;width:0;background:var(--mint)}.hero{position:relative;overflow:hidden;color:#fff;background:var(--ink)}.hero:after{position:absolute;right:-110px;top:-150px;width:420px;height:420px;border:1px solid #ffffff1a;border-radius:50%;content:""}.hero-inner{position:relative;z-index:1;max-width:1240px;margin:auto;padding:58px 42px 126px}.breadcrumb{display:flex;gap:8px;margin-bottom:34px;color:#dbeafecc;font-size:14px}.eyebrow{display:block;color:var(--blue);font-size:12px;font-weight:800;letter-spacing:.16em;text-transform:uppercase}.hero .eyebrow{color:var(--mint)}h1,h2,h3,p{margin-top:0}.hero h1{max-width:1000px;margin:14px 0 24px;font:700 clamp(44px,6vw,78px)/.98 var(--serif);letter-spacing:-.035em}.hero .dek{max-width:780px;color:#dbeafe;font-size:20px}.contributors{margin-top:30px;color:#eff6ff;font-size:14px}.contributors a{font-weight:800}.meta{display:flex;flex-wrap:wrap;gap:12px;margin-top:6px;color:#dbeafecc}.layout{position:relative;z-index:2;display:grid;grid-template-columns:minmax(0,760px) 290px;gap:64px;max-width:1240px;margin:-78px auto 0;padding:0 42px 80px}.main{min-width:0}.takeaways{padding:38px 44px;border-radius:24px;background:#fff;box-shadow:0 24px 70px #10213d24}.takeaways h2{margin:5px 0 20px;font:700 32px/1.1 var(--serif);color:var(--ink)}.takeaways ul{display:grid;gap:16px;margin:0;padding:0;list-style:none}.takeaways li{position:relative;padding-left:26px;color:var(--body);font-size:17px}.takeaways li:before{position:absolute;left:2px;top:.7em;width:8px;height:8px;border-radius:50%;background:var(--blue);content:""}.intro{padding:40px 0 4px}.intro p,.article-section>p{margin-bottom:22px;color:var(--body);font:400 18px/1.75 var(--sans)}.article-section{scroll-margin-top:28px;padding-top:45px}.article-section h2,.faq h2{margin:6px 0 20px;color:var(--ink);font:700 38px/1.08 var(--serif);letter-spacing:-.02em}.article-section ul{display:grid;gap:10px;margin:22px 0;padding-left:24px;color:var(--body);font-size:17px}.article-section li::marker{color:var(--blue)}.callout{margin:34px 0;padding:24px 28px;border-left:4px solid var(--blue);border-radius:0 16px 16px 0;background:#fff}.callout span{color:var(--blue);font-size:11px;font-weight:800;letter-spacing:.13em;text-transform:uppercase}.callout h3{margin:5px 0 8px;color:var(--ink);font:700 24px/1.15 var(--serif)}.callout p{margin:0;color:var(--body)}.rail{padding-top:0}.sidebar-media{overflow:hidden;margin:0 0 18px;border:1px solid #dbeafe;border-radius:16px;background:#fff}.sidebar-media img,.sidebar-media svg{display:block;width:100%;aspect-ratio:16/9;object-fit:cover}.sidebar-media figcaption{padding:10px 13px;color:#64748b;font-size:11px}.disclosure{margin-bottom:28px;border-top:1px solid #dbeafe;border-bottom:1px solid #dbeafe}.disclosure button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:14px;padding:15px 0;border:0;color:#64748b;background:transparent;text-align:left;font-size:12px;cursor:pointer}.disclosure button b{color:var(--blue);font-size:18px;transition:transform .2s}.disclosure button[aria-expanded="true"] b{transform:rotate(180deg)}.disclosure p{padding:0 0 16px;margin:0;color:#64748b;font-size:12px}.toc{position:sticky;top:30px;padding:22px;border:1px solid var(--line);border-radius:16px;background:#fff;box-shadow:0 2px 5px #10213d12}.toc header{display:flex;align-items:center;justify-content:space-between}.toc header strong{color:var(--ink);font:700 21px/1 var(--serif)}.toc header button{width:34px;height:34px;border:0;border-radius:50%;color:var(--blue);background:transparent;cursor:pointer}.toc nav{display:grid;margin-top:14px}.toc nav a{display:flex;align-items:flex-start;gap:11px;padding:8px 0;color:#64748b;text-decoration:none;font-size:13px;line-height:1.35}.toc nav i{width:8px;height:8px;margin-top:5px;border:2px solid #bfdbfe;border-radius:50%}.toc nav a[aria-current="location"]{color:var(--ink);font-weight:800}.toc nav a[aria-current="location"] i{border-color:var(--blue);background:var(--blue)}.back-top{width:100%;margin-top:18px;padding:16px 0 0;border:0;border-top:1px solid var(--line);color:var(--blue);background:transparent;text-align:right;font-size:13px;font-weight:800;cursor:pointer}.faq{scroll-margin-top:28px;padding-top:64px}.faq>div{border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.faq article+article{border-top:1px solid var(--line)}.faq h3{margin:0}.faq h3 button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:18px;padding:19px 0;border:0;color:var(--ink);background:transparent;text-align:left;font-weight:800;cursor:pointer}.faq h3 b{color:var(--blue);font-size:22px}.faq article>p{padding:0 0 20px;margin:0;color:var(--body);font-size:16px}.author-card{margin-top:64px;padding:32px 36px;border-radius:18px;background:#fff}.article-actions{display:flex;flex-wrap:wrap;justify-content:space-between;gap:14px}.article-actions button{display:inline-flex;min-height:46px;align-items:center;justify-content:center;gap:8px;padding:8px 18px;border:2px solid var(--blue);border-radius:999px;color:var(--ink);background:#fff;font-weight:800;cursor:pointer}.article-actions button:hover{color:#fff;background:var(--blue)}.action-status{min-height:20px;margin:8px 0 0;color:var(--blue);text-align:right;font-size:13px;font-weight:700}.author-grid{display:grid;grid-template-columns:auto 1fr auto;gap:22px;align-items:center;margin-top:24px}.author-portrait{width:88px;height:88px;border-radius:50%;object-fit:cover}.author-initials{display:grid;place-items:center;color:var(--ink);background:#fecdd3;font-size:22px;font-weight:900}.author-grid span{color:#64748b;font-size:13px}.author-grid h2{margin:2px 0;color:var(--ink);font:700 27px/1 var(--serif)}.author-grid h2 a{text-decoration:none}.author-grid p{margin:8px 0;color:#64748b;font-size:14px}.author-grid nav{display:flex;flex-wrap:wrap;gap:14px}.author-grid nav a{color:#64748b;font-size:12px}.author-more{display:inline-flex;min-height:48px;align-items:center;justify-content:center;padding:10px 24px;border:2px solid var(--blue);color:var(--blue);text-align:center;text-decoration:none;font-size:13px;font-weight:800}.author-bio{margin:24px 0 0!important;padding-top:20px;border-top:1px solid var(--line);color:var(--body);font-size:16px}dialog{width:min(720px,calc(100% - 30px));max-height:calc(100dvh - 30px);padding:0;border:0;border-radius:18px;color:var(--ink);background:#fff;box-shadow:0 24px 70px #10213d38}dialog::backdrop{background:#0f172a7a;backdrop-filter:blur(2px)}dialog>div{position:relative;padding:30px}dialog h2{margin:0 50px 2px 0;font:700 30px/1.1 var(--serif)}dialog>div>p{color:#64748b}.dialog-close{position:absolute;top:13px;right:13px;width:44px;height:44px;border:0;border-radius:50%;color:var(--blue);background:transparent;font-size:29px;cursor:pointer}dialog section{border-top:1px solid var(--line)}dialog section button{display:block;width:100%;padding:17px 5px;border:0;border-bottom:1px solid var(--line);color:var(--body);background:#fff;text-align:left;line-height:1.65;cursor:pointer}.citation-status{min-height:22px;margin:14px 0 0!important;color:var(--blue)!important;font-size:13px;font-weight:800}.generated{max-width:1156px;margin:-52px auto 50px;padding:0 42px;color:#94a3b8;font-size:11px}@media(max-width:900px){.layout{display:block}.rail{display:none}.hero-inner{padding-bottom:116px}}@media(max-width:620px){.hero-inner{padding:36px 20px 104px}.hero h1{font-size:46px}.hero .dek{font-size:17px}.layout{margin-top:-70px;padding:0 16px 54px}.takeaways{padding:28px 24px}.article-section h2,.faq h2{font-size:32px}.author-card{padding:24px}.article-actions{display:grid;justify-content:start}.author-grid{grid-template-columns:1fr;gap:12px}.author-more{width:100%}dialog>div{padding:24px}.generated{padding:0 20px}}
41495
40663
  </style></head><body data-theme="${escapeHtml11(config.theme)}" data-scheduled-renderer="blog_article_v1"><div class="progress" aria-hidden="true"></div><header class="hero"><div class="hero-inner"><nav class="breadcrumb" aria-label="Breadcrumb"><a href="${safeHref4(input.canonicalUrl)}">${escapeHtml11(input.category)}</a><span>/</span><span>Article</span></nav><span class="eyebrow">${escapeHtml11(input.eyebrow)}</span><h1>${escapeHtml11(input.title)}</h1>${input.dek ? `<p class="dek">${escapeHtml11(input.dek)}</p>` : ""}<div class="contributors">${contributorText}<div class="meta"><time datetime="${escapeHtml11(articleDate)}">Updated ${formatDate2(articleDate)}</time><span>\u2022</span><span>${input.readTimeMinutes} min read</span>${input.reviewer ? "<span>\u2022</span><strong>Expert reviewed</strong>" : ""}</div></div></div></header><main class="layout"><article class="main"><section class="takeaways"><span class="eyebrow">The short answer</span><h2>Key takeaways</h2><ul>${input.keyTakeaways.map((item) => `<li>${escapeHtml11(item)}</li>`).join("")}</ul></section><section class="intro">${input.introduction.map((paragraph) => `<p>${escapeHtml11(paragraph)}</p>`).join("")}</section>${sections}${faq}${author}</article><div class="rail">${media}${disclosure}${toc}</div></main>${config.showGeneratedAt ? `<p class="generated">Rendered ${formatDate2(generatedAtIso)} by MCP Scraper.</p>` : ""}${citationDialog}<script>(()=>{const citations=${jsonForScript(citationValues)};const canonical=${jsonForScript(input.canonicalUrl)};const title=${jsonForScript(input.title)};const progress=document.querySelector('.progress');const tocLinks=[...document.querySelectorAll('.toc nav a')];const targets=tocLinks.map(link=>document.querySelector(link.hash)).filter(Boolean);const update=()=>{const max=document.documentElement.scrollHeight-innerHeight;if(progress)progress.style.width=(max>0?scrollY/max*100:0)+'%';let active=targets[0]?.id;for(const target of targets){if(target.getBoundingClientRect().top<=innerHeight*.55)active=target.id}tocLinks.forEach(link=>link.toggleAttribute('aria-current',link.hash==='#'+active));tocLinks.forEach(link=>{if(link.hash==='#'+active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current')})};addEventListener('scroll',update,{passive:true});addEventListener('resize',update);update();document.querySelectorAll('.faq h3 button').forEach(button=>button.addEventListener('click',()=>{const panel=document.getElementById(button.getAttribute('aria-controls'));const open=button.getAttribute('aria-expanded')==='true';button.setAttribute('aria-expanded',String(!open));panel.hidden=open;button.querySelector('b').textContent=open?'+':'\u2212'}));const disclosure=document.querySelector('.disclosure button');disclosure?.addEventListener('click',()=>{const panel=document.getElementById(disclosure.getAttribute('aria-controls'));const open=disclosure.getAttribute('aria-expanded')==='true';disclosure.setAttribute('aria-expanded',String(!open));panel.hidden=open});const tocToggle=document.querySelector('.toc header button');tocToggle?.addEventListener('click',()=>{const nav=document.getElementById('toc-links');const open=tocToggle.getAttribute('aria-expanded')==='true';tocToggle.setAttribute('aria-expanded',String(!open));nav.hidden=open;tocToggle.textContent=open?'\u2304':'\u2303'});document.querySelector('.back-top')?.addEventListener('click',()=>scrollTo({top:0,behavior:'smooth'}));const copy=async text=>{try{await navigator.clipboard.writeText(text)}catch{const area=document.createElement('textarea');area.value=text;area.style.position='fixed';area.style.opacity='0';document.body.append(area);area.select();document.execCommand('copy');area.remove()}};const dialog=document.getElementById('citation-dialog');document.querySelector('[data-cite]')?.addEventListener('click',()=>dialog.showModal());document.querySelector('.dialog-close')?.addEventListener('click',()=>dialog.close());dialog?.addEventListener('click',event=>{if(event.target===dialog)dialog.close()});document.querySelectorAll('[data-citation]').forEach(button=>button.addEventListener('click',async()=>{const style=button.dataset.citation;await copy(citations[style]);document.querySelector('.citation-status').textContent=style+' citation copied to clipboard.'}));document.querySelector('[data-share]')?.addEventListener('click',async()=>{const status=document.querySelector('.action-status');if(navigator.share){try{await navigator.share({title,url:canonical});status.textContent='Sharing options opened.';return}catch(error){if(error.name==='AbortError')return}}await copy(canonical);status.textContent='Article link copied to clipboard.'})})()</script></body></html>`;
41496
40664
  const bytes = Buffer.byteLength(html);
41497
- const sha2564 = createHash27("sha256").update(html).digest("hex");
40665
+ const sha2564 = createHash26("sha256").update(html).digest("hex");
41498
40666
  return {
41499
40667
  html,
41500
40668
  filename: "blog-article.html",
@@ -41797,7 +40965,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
41797
40965
  import { Hono as Hono36 } from "hono";
41798
40966
 
41799
40967
  // src/scheduled-artifacts/scheduled-run-artifact-store.ts
41800
- import { createHash as createHash28 } from "crypto";
40968
+ import { createHash as createHash27 } from "crypto";
41801
40969
  var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
41802
40970
  var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
41803
40971
  var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
@@ -41813,7 +40981,7 @@ function policy5() {
41813
40981
  };
41814
40982
  }
41815
40983
  function runStorageSegment(runId) {
41816
- return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash28("sha256").update(runId).digest("hex").slice(0, 32)}`;
40984
+ return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash27("sha256").update(runId).digest("hex").slice(0, 32)}`;
41817
40985
  }
41818
40986
  async function createScheduledRunArtifact(args) {
41819
40987
  if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
@@ -41920,7 +41088,7 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
41920
41088
  import { Hono as Hono37 } from "hono";
41921
41089
 
41922
41090
  // src/api/scheduler-integration-auth.ts
41923
- import { createHash as createHash29, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
41091
+ import { createHash as createHash28, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
41924
41092
 
41925
41093
  // src/api/service-connections.ts
41926
41094
  import { randomUUID as randomUUID23 } from "crypto";
@@ -42218,7 +41386,7 @@ function signingSecret() {
42218
41386
  return secret;
42219
41387
  }
42220
41388
  function schedulerIntegrationSignature(args) {
42221
- const bodyHash = createHash29("sha256").update(args.body).digest("hex");
41389
+ const bodyHash = createHash28("sha256").update(args.body).digest("hex");
42222
41390
  return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
42223
41391
  ${args.path}
42224
41392
  ${args.timestamp}
@@ -42410,7 +41578,7 @@ function shouldRunSiteExtractInBackground(input) {
42410
41578
  }
42411
41579
 
42412
41580
  // src/api/page-media-artifacts.ts
42413
- import { createHash as createHash30, randomBytes as randomBytes6 } from "crypto";
41581
+ import { createHash as createHash29, randomBytes as randomBytes6 } from "crypto";
42414
41582
  import { homedir as homedir4 } from "os";
42415
41583
  import { basename, extname, join as join9 } from "path";
42416
41584
  import pLimit6 from "p-limit";
@@ -42557,7 +41725,7 @@ async function fetchMedia(rawUrl, expectedType) {
42557
41725
  finalUrl: checked.parsed.href,
42558
41726
  width: dimensions?.width ?? null,
42559
41727
  height: dimensions?.height ?? null,
42560
- sha256: createHash30("sha256").update(bytes).digest("hex")
41728
+ sha256: createHash29("sha256").update(bytes).digest("hex")
42561
41729
  };
42562
41730
  }
42563
41731
  throw new Error("media_redirect_rejected");
@@ -42763,13 +41931,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
42763
41931
  }
42764
41932
 
42765
41933
  // src/api/page-diff.ts
42766
- import { createHash as createHash31 } from "crypto";
41934
+ import { createHash as createHash30 } from "crypto";
42767
41935
  import { diffLines } from "diff";
42768
41936
  var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
42769
41937
  var MAX_DIFF_HUNKS = 200;
42770
41938
  var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
42771
41939
  function sha256Hex(value) {
42772
- return createHash31("sha256").update(value).digest("hex");
41940
+ return createHash30("sha256").update(value).digest("hex");
42773
41941
  }
42774
41942
  function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
42775
41943
  if (value.length <= maxChars) return { value, truncated: false };
@@ -42883,10 +42051,10 @@ async function persistScrapeBody(user, opts) {
42883
42051
  }
42884
42052
 
42885
42053
  // src/api/scrape-image-sink.ts
42886
- import { createHash as createHash32 } from "crypto";
42054
+ import { createHash as createHash31 } from "crypto";
42887
42055
  var MAX_IMAGES_PER_SCRAPE = 25;
42888
42056
  function idempotencyKey3(userId, vault, input) {
42889
- return `scrape-image-${createHash32("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
42057
+ return `scrape-image-${createHash31("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
42890
42058
  }
42891
42059
  async function persistScrapeImagesToMemory(user, inputs, vault) {
42892
42060
  const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
@@ -43358,7 +42526,7 @@ function inferredWaybackCapturedAt(content) {
43358
42526
  }
43359
42527
 
43360
42528
  // src/api/connection-memory-import.ts
43361
- import { createHash as createHash33 } from "crypto";
42529
+ import { createHash as createHash32 } from "crypto";
43362
42530
  var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
43363
42531
  var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
43364
42532
  var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
@@ -43388,7 +42556,7 @@ function canonicalJson(value) {
43388
42556
  return JSON.stringify(value);
43389
42557
  }
43390
42558
  function sha2562(value) {
43391
- return createHash33("sha256").update(value).digest("hex");
42559
+ return createHash32("sha256").update(value).digest("hex");
43392
42560
  }
43393
42561
  function sensitiveKey(key) {
43394
42562
  const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
@@ -43657,7 +42825,7 @@ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
43657
42825
  }
43658
42826
 
43659
42827
  // src/api/site-export-reader.ts
43660
- import { createHash as createHash34 } from "crypto";
42828
+ import { createHash as createHash33 } from "crypto";
43661
42829
  var SiteExportFormatUnavailableError = class extends Error {
43662
42830
  constructor(format) {
43663
42831
  super(`The ${format} representation was not retained for this export page.`);
@@ -43667,7 +42835,7 @@ var SiteExportFormatUnavailableError = class extends Error {
43667
42835
  format;
43668
42836
  };
43669
42837
  function sha2563(value) {
43670
- return createHash34("sha256").update(value).digest("hex");
42838
+ return createHash33("sha256").update(value).digest("hex");
43671
42839
  }
43672
42840
  function publicPageRecord(page) {
43673
42841
  const { bodyMarkdown: _body, contentRef: _contentRef, discoveryLinks: _discovery, ...metadata } = page;
@@ -44519,7 +43687,7 @@ async function dbUsage(identity, plan) {
44519
43687
  }
44520
43688
 
44521
43689
  // src/api/nango-control.ts
44522
- import { createHash as createHash36, randomUUID as randomUUID27 } from "crypto";
43690
+ import { createHash as createHash35, randomUUID as randomUUID27 } from "crypto";
44523
43691
 
44524
43692
  // src/api/slack-archive-analysis.ts
44525
43693
  function isRecord2(value) {
@@ -44893,7 +44061,7 @@ async function exportSlackChannelPage(input, dependencies) {
44893
44061
  }
44894
44062
 
44895
44063
  // src/api/main-nango-transport.ts
44896
- import { createHash as createHash35, randomUUID as randomUUID26 } from "crypto";
44064
+ import { createHash as createHash34, randomUUID as randomUUID26 } from "crypto";
44897
44065
  import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
44898
44066
 
44899
44067
  // src/api/connected-tool-policy.ts
@@ -45286,7 +44454,7 @@ async function listNangoToolsDirect(identity, connectionId) {
45286
44454
  });
45287
44455
  const readTools = [...policies.values()].filter((policy7) => policy7.classification === "read").map((policy7) => policy7.name);
45288
44456
  const actionTools = [...policies.values()].filter((policy7) => policy7.classification === "action").map((policy7) => policy7.name);
45289
- const revision = createHash35("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
44457
+ const revision = createHash34("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
45290
44458
  await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
45291
44459
  const refreshed = await getOwnedServiceConnection(identity, connection.id);
45292
44460
  return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
@@ -45314,7 +44482,7 @@ async function callNangoToolDirect(args) {
45314
44482
  ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
45315
44483
  });
45316
44484
  const requestId = args.requestId?.trim() || randomUUID26();
45317
- const idempotencyKey4 = `main-nango:${createHash35("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
44485
+ const idempotencyKey4 = `main-nango:${createHash34("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
45318
44486
  const startedAt = /* @__PURE__ */ new Date();
45319
44487
  const started = performance.now();
45320
44488
  let result;
@@ -45339,7 +44507,7 @@ async function callNangoToolDirect(args) {
45339
44507
  toolName: args.tool,
45340
44508
  operationKind: args.operationKind ?? args.classification,
45341
44509
  outcome: providerError ? "error" : "partial",
45342
- requestId: requestId.length <= 200 ? requestId : createHash35("sha256").update(requestId).digest("hex"),
44510
+ requestId: requestId.length <= 200 ? requestId : createHash34("sha256").update(requestId).digest("hex"),
45343
44511
  startedAt: startedAt.toISOString(),
45344
44512
  completedAt: completedAt.toISOString()
45345
44513
  }
@@ -45381,7 +44549,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
45381
44549
  providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
45382
44550
  protocolVersion: null,
45383
44551
  schemaSource: "live_tools_list",
45384
- schemaHash: createHash35("sha256").update(JSON.stringify(projected)).digest("hex"),
44552
+ schemaHash: createHash34("sha256").update(JSON.stringify(projected)).digest("hex"),
45385
44553
  fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
45386
44554
  };
45387
44555
  }
@@ -46105,8 +45273,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
46105
45273
  return data.connection.actionsEnabled === true;
46106
45274
  }
46107
45275
  async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey4) {
46108
- const requestId = `main-connected-action:${createHash36("sha256").update(identity).update("\0").update(idempotencyKey4?.trim() || randomUUID27()).digest("hex")}`;
46109
- const requestDigest = createHash36("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
45276
+ const requestId = `main-connected-action:${createHash35("sha256").update(identity).update("\0").update(idempotencyKey4?.trim() || randomUUID27()).digest("hex")}`;
45277
+ const requestDigest = createHash35("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
46110
45278
  if (mainOwnsIntegrations()) {
46111
45279
  const selectedTool = tool?.trim();
46112
45280
  if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
@@ -46257,7 +45425,7 @@ function canonicalJson2(value) {
46257
45425
  return JSON.stringify(value);
46258
45426
  }
46259
45427
  function projectedToolSchemaHash(tool) {
46260
- return createHash36("sha256").update(canonicalJson2(tool)).digest("hex");
45428
+ return createHash35("sha256").update(canonicalJson2(tool)).digest("hex");
46261
45429
  }
46262
45430
  async function describeNangoTool(identity, connectionId, tool, fresh) {
46263
45431
  if (mainOwnsIntegrations()) {
@@ -46536,7 +45704,7 @@ async function callMainOwnedExportPage(identity, input) {
46536
45704
  }
46537
45705
 
46538
45706
  // src/api/resend-control.ts
46539
- import { createHash as createHash37 } from "crypto";
45707
+ import { createHash as createHash36 } from "crypto";
46540
45708
  var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
46541
45709
  var RESEND_PROVIDER_CONFIG_KEY = "resend";
46542
45710
  var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
@@ -46795,7 +45963,7 @@ async function callResendRead(identity, connectionId, tool, args) {
46795
45963
  return isRecord5(data) ? data.result ?? data : data;
46796
45964
  }
46797
45965
  async function callResendAction(identity, connectionId, tool, input, idempotencyKey4) {
46798
- const requestId = `main-resend-action:${createHash37("sha256").update(identity).update("\0").update(idempotencyKey4.trim()).digest("hex")}`;
45966
+ const requestId = `main-resend-action:${createHash36("sha256").update(identity).update("\0").update(idempotencyKey4.trim()).digest("hex")}`;
46799
45967
  const body = await controlRequest2("/api/internal/resend/actions/call", {
46800
45968
  method: "POST",
46801
45969
  headers: { "x-request-id": requestId },
@@ -48651,8 +47819,6 @@ app.post("/harvest", auth4, async (c) => {
48651
47819
  if (checked.error) return c.json({ error: checked.error }, 400);
48652
47820
  body.callback_url = checked.parsed?.href;
48653
47821
  }
48654
- const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
48655
- if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
48656
47822
  const options = {
48657
47823
  query: body.query.trim(),
48658
47824
  location: body.location,
@@ -48674,6 +47840,26 @@ app.post("/harvest", auth4, async (c) => {
48674
47840
  includeAiOverview: body.includeAiOverview ?? false,
48675
47841
  includeWhatPeopleSaying: body.includeWhatPeopleSaying ?? false
48676
47842
  };
47843
+ const suppliedIdempotencyKey = c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
47844
+ let derivedJobId = null;
47845
+ let requestFingerprint3 = null;
47846
+ if (!options.serpOnly && suppliedIdempotencyKey != null) {
47847
+ const trimmedKey = suppliedIdempotencyKey.trim();
47848
+ if (!trimmedKey || trimmedKey.length > 200) {
47849
+ return c.json({ error: "Idempotency-Key must contain 1-200 characters.", errorCode: "invalid_idempotency_key", retryable: false }, 400);
47850
+ }
47851
+ derivedJobId = `paa-${sha256Hex(`${user.id}\0${trimmedKey}`).slice(0, 24)}`;
47852
+ requestFingerprint3 = sha256Hex(JSON.stringify({ options, callbackUrl: body.callback_url ?? null }));
47853
+ const existing = await getJob(derivedJobId, user.id);
47854
+ if (existing) {
47855
+ if (existing.options.requestFingerprint !== requestFingerprint3) {
47856
+ return c.json({ error: "Idempotency-Key was already used for a different asynchronous harvest_paa call.", errorCode: "idempotency_conflict", retryable: false }, 409);
47857
+ }
47858
+ return c.json({ job_id: existing.id, status: existing.status, replayed: true }, ["done", "failed", "cancelled"].includes(existing.status) ? 200 : 202);
47859
+ }
47860
+ }
47861
+ const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
47862
+ if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
48677
47863
  const harvestCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount(options.maxQuestions);
48678
47864
  let jobId2;
48679
47865
  if (options.serpOnly) {
@@ -48686,7 +47872,7 @@ app.post("/harvest", auth4, async (c) => {
48686
47872
  if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
48687
47873
  jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
48688
47874
  } else {
48689
- jobId2 = randomUUID28();
47875
+ jobId2 = derivedJobId ?? randomUUID28();
48690
47876
  const billingDebitKey = `paa-harvest:${jobId2}:hold`;
48691
47877
  const description = `PAA harvest: ${options.query}`.slice(0, 500);
48692
47878
  const hold = await debitMcIdempotent(
@@ -48700,11 +47886,16 @@ app.post("/harvest", auth4, async (c) => {
48700
47886
  try {
48701
47887
  await createJobWithId(jobId2, user.id, options.query, {
48702
47888
  ...options,
48703
- executionOwner: "inngest",
47889
+ executionOwner: "direct",
48704
47890
  billingHoldMc: harvestCost,
48705
- billingDebitKey
47891
+ billingDebitKey,
47892
+ ...requestFingerprint3 ? { requestFingerprint: requestFingerprint3 } : {}
48706
47893
  }, body.callback_url);
48707
47894
  } catch (error) {
47895
+ const existing = derivedJobId ? await getJob(derivedJobId, user.id).catch(() => null) : null;
47896
+ if (existing && existing.options.requestFingerprint === requestFingerprint3) {
47897
+ return c.json({ job_id: existing.id, status: existing.status, replayed: true }, 202);
47898
+ }
48708
47899
  await settleDebitMcIdempotent(
48709
47900
  user.id,
48710
47901
  billingDebitKey,
@@ -48715,11 +47906,9 @@ app.post("/harvest", auth4, async (c) => {
48715
47906
  ).catch(() => void 0);
48716
47907
  throw error;
48717
47908
  }
48718
- await dispatchPaaHarvest(jobId2).catch((error) => {
48719
- console.error("[harvest] durable PAA dispatch pending reconciliation:", error instanceof Error ? error.message : String(error));
48720
- });
47909
+ scheduleDirectPaaHarvest(jobId2);
48721
47910
  }
48722
- if (process.env.CRON_SECRET) {
47911
+ if (options.serpOnly && process.env.CRON_SECRET) {
48723
47912
  const url = new URL(c.req.url);
48724
47913
  void fetch(`${url.origin}/cron/tick`, {
48725
47914
  headers: { Authorization: `Bearer ${process.env.CRON_SECRET}` }
@@ -50237,11 +49426,11 @@ app.get("/cron/tick", async (c) => {
50237
49426
  return c.json({ error: "Unauthorized" }, 401);
50238
49427
  }
50239
49428
  const startedAt = Date.now();
50240
- const { drainQueue } = await import("./worker-K7MPPQ7I.js");
49429
+ const { drainQueue } = await import("./worker-UTBUWEXQ.js");
50241
49430
  const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
50242
49431
  const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
50243
49432
  const unfinished = [];
50244
- const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, leadListDispatch, leadListReconciliation, leadListArtifactCleanup, leadListInputCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation, commonsEmbeddingDrain, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
49433
+ const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, leadListDispatch, leadListReconciliation, leadListArtifactCleanup, leadListInputCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestReconciliation, commonsEmbeddingDrain, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
50245
49434
  settleWithinTickBudget("drainQueue", unfinished, drainQueue(budget), []),
50246
49435
  settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
50247
49436
  settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
@@ -50260,7 +49449,6 @@ app.get("/cron/tick", async (c) => {
50260
49449
  settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
50261
49450
  settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
50262
49451
  settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
50263
- settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
50264
49452
  settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
50265
49453
  settleWithinTickBudget("commonsEmbeddingDrain", unfinished, embedQueuedCommonsDocuments(50), { claimed: 0, embedded: 0, failed: 0, remaining: 0 }),
50266
49454
  settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
@@ -50295,7 +49483,6 @@ app.get("/cron/tick", async (c) => {
50295
49483
  siteExtractArtifactCleanup,
50296
49484
  siteExtractRedispatch,
50297
49485
  siteExtractReconciliation,
50298
- paaHarvestRedispatch,
50299
49486
  paaHarvestReconciliation,
50300
49487
  commonsEmbeddingDrain,
50301
49488
  localSourcebookRedispatch,
@@ -50317,8 +49504,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
50317
49504
  finishExtractJob: finishExtractJob2,
50318
49505
  terminalExtractJobStatus: terminalExtractJobStatus2,
50319
49506
  extractJobLimitInfo: extractJobLimitInfo2
50320
- } = await import("./site-extract-repository-J3RH3MOS.js");
50321
- const { assembleExtractArtifacts } = await import("./extract-bundle-QAI7JYV2.js");
49507
+ } = await import("./site-extract-repository-NTM4MDQN.js");
49508
+ const { assembleExtractArtifacts } = await import("./extract-bundle-EA5URLXN.js");
50322
49509
  const existing = await getExtractJob2(jobId2);
50323
49510
  if (!existing) return c.json({ error: "job not found" }, 404);
50324
49511
  if (existing.status !== "failed") {
@@ -50345,7 +49532,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
50345
49532
  if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
50346
49533
  let settlement = "already_settled_or_refunded";
50347
49534
  if (job.billedMc == null && job.userId != null) {
50348
- const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-J3RH3MOS.js");
49535
+ const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-NTM4MDQN.js");
50349
49536
  const heldMc = Number(job.options.heldMc ?? 0);
50350
49537
  const successful = await countSuccessfulPages2(jobId2);
50351
49538
  const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
@@ -50362,7 +49549,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
50362
49549
  });
50363
49550
  app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({
50364
49551
  client: inngest,
50365
- functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, leadListEnrichmentFn, paaHarvestFn, localSourcebookFn]
49552
+ functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, leadListEnrichmentFn, localSourcebookFn]
50366
49553
  }));
50367
49554
  app.route("/admin/credits", adminCreditsApp);
50368
49555
  app.route("/api/internal/site-architecture-auditor", siteAuditApp);
@@ -50532,4 +49719,3 @@ app.get("/blog/:slug/", (c) => {
50532
49719
  export {
50533
49720
  app
50534
49721
  };
50535
- //# sourceMappingURL=server-MTKWBSAR.js.map