mcp-scraper 0.40.2 → 0.40.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/api-server.cjs +1517 -236
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs +155 -87
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +6 -5
  14. package/dist/bin/mcp-stdio-server.js.map +1 -1
  15. package/dist/bin/paa-harvest.cjs +31 -9
  16. package/dist/bin/paa-harvest.cjs.map +1 -1
  17. package/dist/bin/paa-harvest.js +4 -4
  18. package/dist/{chunk-XORPNO3Z.js → chunk-27DUCRAZ.js} +2 -2
  19. package/dist/{chunk-44HZLHDV.js → chunk-5UN33CGU.js} +38 -1
  20. package/dist/chunk-5UN33CGU.js.map +1 -0
  21. package/dist/{chunk-X2LKCX6H.js → chunk-6J57U6HA.js} +3 -3
  22. package/dist/{chunk-LP6E462I.js → chunk-6SZ52BQ5.js} +49 -101
  23. package/dist/chunk-6SZ52BQ5.js.map +1 -0
  24. package/dist/chunk-BBI7RGOT.js +99 -0
  25. package/dist/chunk-BBI7RGOT.js.map +1 -0
  26. package/dist/{chunk-NVXNEOUQ.js → chunk-CS4HE6QY.js} +47 -11
  27. package/dist/chunk-CS4HE6QY.js.map +1 -0
  28. package/dist/{chunk-3PIWJS6Y.js → chunk-EHES33KB.js} +2 -2
  29. package/dist/{chunk-GUVKHCKE.js → chunk-IQTT7CAB.js} +94 -3
  30. package/dist/chunk-IQTT7CAB.js.map +1 -0
  31. package/dist/{chunk-7XBBFBYY.js → chunk-JVIW4GK2.js} +44 -12
  32. package/dist/chunk-JVIW4GK2.js.map +1 -0
  33. package/dist/chunk-XU4ZLIB2.js +7 -0
  34. package/dist/chunk-XU4ZLIB2.js.map +1 -0
  35. package/dist/{db-W3CP562I.js → db-LVVU6NOK.js} +16 -2
  36. package/dist/{extract-bundle-K4PG3RZJ.js → extract-bundle-SKXEDG4Z.js} +4 -4
  37. package/dist/index.cjs +31 -9
  38. package/dist/index.cjs.map +1 -1
  39. package/dist/index.js +4 -4
  40. package/dist/{location-data-repository-RLQX6SNM.js → location-data-repository-NBTCOA7B.js} +3 -3
  41. package/dist/{server-KM3CFWCF.js → server-W76RUYD3.js} +1108 -66
  42. package/dist/server-W76RUYD3.js.map +1 -0
  43. package/dist/{site-extract-repository-2SMMFKKL.js → site-extract-repository-6GX72U4N.js} +4 -4
  44. package/dist/{worker-FXAGFYOE.js → worker-5RWLFSAJ.js} +27 -28
  45. package/dist/worker-5RWLFSAJ.js.map +1 -0
  46. package/package.json +1 -1
  47. package/dist/chunk-44HZLHDV.js.map +0 -1
  48. package/dist/chunk-7XBBFBYY.js.map +0 -1
  49. package/dist/chunk-GUVKHCKE.js.map +0 -1
  50. package/dist/chunk-LP6E462I.js.map +0 -1
  51. package/dist/chunk-NVXNEOUQ.js.map +0 -1
  52. package/dist/chunk-ZUJLSICT.js +0 -7
  53. package/dist/chunk-ZUJLSICT.js.map +0 -1
  54. package/dist/server-KM3CFWCF.js.map +0 -1
  55. package/dist/worker-FXAGFYOE.js.map +0 -1
  56. /package/dist/{chunk-XORPNO3Z.js.map → chunk-27DUCRAZ.js.map} +0 -0
  57. /package/dist/{chunk-X2LKCX6H.js.map → chunk-6J57U6HA.js.map} +0 -0
  58. /package/dist/{chunk-3PIWJS6Y.js.map → chunk-EHES33KB.js.map} +0 -0
  59. /package/dist/{db-W3CP562I.js.map → db-LVVU6NOK.js.map} +0 -0
  60. /package/dist/{extract-bundle-K4PG3RZJ.js.map → extract-bundle-SKXEDG4Z.js.map} +0 -0
  61. /package/dist/{location-data-repository-RLQX6SNM.js.map → location-data-repository-NBTCOA7B.js.map} +0 -0
  62. /package/dist/{site-extract-repository-2SMMFKKL.js.map → site-extract-repository-6GX72U4N.js.map} +0 -0
@@ -76,14 +76,48 @@ var init_browser_service_env = __esm({
76
76
  });
77
77
 
78
78
  // src/errors.ts
79
+ function normalizePublicErrorCode(code) {
80
+ return typeof code === "string" && PUBLIC_ERROR_CODES.has(code) ? code : "service_unavailable";
81
+ }
82
+ function publicErrorMessage(code) {
83
+ switch (normalizePublicErrorCode(code)) {
84
+ case "request_aborted":
85
+ return "The request was cancelled before it completed.";
86
+ case "captcha_exhausted":
87
+ case "captcha_or_blocked":
88
+ return "The target site returned a verification challenge. Please retry later.";
89
+ case "location_mismatch":
90
+ return "The target site returned results for a different location. Please retry.";
91
+ case "proxy_tunnel_failed":
92
+ case "proxy_unavailable":
93
+ return "The web request could not establish a working connection. Please retry later.";
94
+ case "harvest_timeout":
95
+ case "mcp_request_timeout":
96
+ return "The request timed out before it completed. Please retry with a smaller request.";
97
+ default:
98
+ return PUBLIC_SERVICE_UNAVAILABLE_MESSAGE;
99
+ }
100
+ }
79
101
  function sanitizeVendorName(message) {
80
102
  return message.replace(/kernel\.sh\s+sessions?/gi, "sessions").replace(/kernel\.sh\s+session/gi, "this session").replace(/kernel\.sh/gi, "the service").replace(/kernel\s+sessions?/gi, "sessions").replace(/kernel\s+session/gi, "this session").replace(/\bkernel\b/gi, "the service").replace(/ +/g, " ").trim();
81
103
  }
82
- var RECAPTCHA_INSTRUCTIONS, CaptchaError, ExtractionError, MapsSearchIncompletePageError, RequestAbortedError, LocationMismatchError;
104
+ var RECAPTCHA_INSTRUCTIONS, PUBLIC_SERVICE_UNAVAILABLE_MESSAGE, PUBLIC_ERROR_CODES, CaptchaError, ExtractionError, MapsSearchIncompletePageError, RequestAbortedError, LocationMismatchError;
83
105
  var init_errors = __esm({
84
106
  "src/errors.ts"() {
85
107
  "use strict";
86
108
  RECAPTCHA_INSTRUCTIONS = "Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.";
109
+ PUBLIC_SERVICE_UNAVAILABLE_MESSAGE = "This operation is temporarily unavailable. Please retry later.";
110
+ PUBLIC_ERROR_CODES = /* @__PURE__ */ new Set([
111
+ "request_aborted",
112
+ "captcha_exhausted",
113
+ "captcha_or_blocked",
114
+ "location_mismatch",
115
+ "proxy_tunnel_failed",
116
+ "proxy_unavailable",
117
+ "harvest_timeout",
118
+ "mcp_request_timeout",
119
+ "service_unavailable"
120
+ ]);
87
121
  CaptchaError = class extends Error {
88
122
  constructor(instructions) {
89
123
  super(`CAPTCHA detected. ${instructions}`);
@@ -130,8 +164,8 @@ function redactVendorUrls(value) {
130
164
  function sanitizeOutboundDiagnostics(value, parentKey = "") {
131
165
  if (typeof value === "string") {
132
166
  let out = redactVendorUrls(value);
133
- if (SANITIZED_VALUE_KEYS.test(parentKey) && /kernel/i.test(out)) {
134
- out = sanitizeVendorName(out);
167
+ if (SANITIZED_VALUE_KEYS.test(parentKey)) {
168
+ out = publicErrorMessage("service_unavailable");
135
169
  }
136
170
  return out;
137
171
  }
@@ -139,6 +173,7 @@ function sanitizeOutboundDiagnostics(value, parentKey = "") {
139
173
  if (value !== null && typeof value === "object") {
140
174
  const out = {};
141
175
  for (const [key, val] of Object.entries(value)) {
176
+ if (INTERNAL_DIAGNOSTIC_KEYS.has(key)) continue;
142
177
  const renamed = KEY_RENAMES[key] ?? key;
143
178
  if (VENDOR_URL_KEYS.has(key)) {
144
179
  out[renamed] = null;
@@ -164,24 +199,47 @@ function sanitizeHarvestResult(result) {
164
199
  }
165
200
  };
166
201
  }
167
- var KEY_RENAMES, SANITIZED_VALUE_KEYS, VENDOR_URL_KEYS, VENDOR_URL_RE;
202
+ var KEY_RENAMES, SANITIZED_VALUE_KEYS, INTERNAL_DIAGNOSTIC_KEYS, VENDOR_URL_KEYS, VENDOR_URL_RE;
168
203
  var init_outbound_sanitize = __esm({
169
204
  "src/api/outbound-sanitize.ts"() {
170
205
  "use strict";
171
206
  init_errors();
172
207
  KEY_RENAMES = {
173
- kernel: "browserRuntime",
174
- kernel_session_id: "browser_session_id",
175
- kernel_delete_started: "session_cleanup_started",
176
- kernel_delete_succeeded: "session_cleanup_succeeded",
177
- kernel_delete_error: "session_cleanup_error",
178
- kernelSessionId: "browserSessionId",
179
- kernelDeleteStarted: "sessionCleanupStarted",
180
- kernelDeleteSucceeded: "sessionCleanupSucceeded",
181
- kernelDeleteError: "sessionCleanupError",
182
- kernelProxyId: "proxyId"
208
+ kernel: "browserRuntime"
183
209
  };
184
210
  SANITIZED_VALUE_KEYS = /error|message/i;
211
+ INTERNAL_DIAGNOSTIC_KEYS = /* @__PURE__ */ new Set([
212
+ "kernel_session_id",
213
+ "browser_session_id",
214
+ "kernelSessionId",
215
+ "browserSessionId",
216
+ "sessionId",
217
+ "kernel_delete_started",
218
+ "session_cleanup_started",
219
+ "kernelDeleteStarted",
220
+ "sessionCleanupStarted",
221
+ "kernel_delete_succeeded",
222
+ "session_cleanup_succeeded",
223
+ "kernelDeleteSucceeded",
224
+ "sessionCleanupSucceeded",
225
+ "kernel_delete_error",
226
+ "session_cleanup_error",
227
+ "kernelDeleteError",
228
+ "sessionCleanupError",
229
+ "kernelProxyId",
230
+ "proxyId",
231
+ "requestedProxyIdSuffix",
232
+ "retrievedProxyIdSuffix",
233
+ "provider",
234
+ "providerName",
235
+ "service",
236
+ "serviceName",
237
+ "vendor",
238
+ "vendorName",
239
+ "organization",
240
+ "organizationId",
241
+ "accountId"
242
+ ]);
185
243
  VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
186
244
  "hosted_url",
187
245
  "hostedUrl",
@@ -3619,6 +3677,8 @@ __export(db_exports, {
3619
3677
  appendWorkflowArtifacts: () => appendWorkflowArtifacts,
3620
3678
  cancelJob: () => cancelJob,
3621
3679
  checkRateLimit: () => checkRateLimit,
3680
+ checkpointJob: () => checkpointJob,
3681
+ claimInngestPaaJob: () => claimInngestPaaJob,
3622
3682
  claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
3623
3683
  claimPendingJob: () => claimPendingJob,
3624
3684
  claimPendingKpoJob: () => claimPendingKpoJob,
@@ -3632,6 +3692,7 @@ __export(db_exports, {
3632
3692
  countActiveJobsForUser: () => countActiveJobsForUser,
3633
3693
  countActiveUsers: () => countActiveUsers,
3634
3694
  createJob: () => createJob,
3695
+ createJobWithId: () => createJobWithId,
3635
3696
  createKpoJob: () => createKpoJob,
3636
3697
  createPasswordResetToken: () => createPasswordResetToken,
3637
3698
  createRunningJob: () => createRunningJob,
@@ -3685,7 +3746,10 @@ __export(db_exports, {
3685
3746
  listInboxMessages: () => listInboxMessages,
3686
3747
  listJobs: () => listJobs,
3687
3748
  listKpoJobs: () => listKpoJobs,
3749
+ listPendingInngestPaaJobs: () => listPendingInngestPaaJobs,
3750
+ listRecentTerminalInngestPaaJobs: () => listRecentTerminalInngestPaaJobs,
3688
3751
  listRequestEvents: () => listRequestEvents,
3752
+ listStaleRunningInngestPaaJobs: () => listStaleRunningInngestPaaJobs,
3689
3753
  listUsers: () => listUsers,
3690
3754
  listUsersDueForConnectedAccountBillingReconciliation: () => listUsersDueForConnectedAccountBillingReconciliation,
3691
3755
  listWorkflowArtifacts: () => listWorkflowArtifacts,
@@ -3694,6 +3758,7 @@ __export(db_exports, {
3694
3758
  logKpoPhaseComplete: () => logKpoPhaseComplete,
3695
3759
  logRequestEvent: () => logRequestEvent,
3696
3760
  markInboxMessageOpened: () => markInboxMessageOpened,
3761
+ markJobBillingSettled: () => markJobBillingSettled,
3697
3762
  markWorkflowRunRunning: () => markWorkflowRunRunning,
3698
3763
  markWorkflowScheduleRan: () => markWorkflowScheduleRan,
3699
3764
  migrate: () => migrate,
@@ -4977,11 +5042,15 @@ function deserialize(raw) {
4977
5042
  }
4978
5043
  async function createJob(userId, query, options, callbackUrl) {
4979
5044
  const id = (0, import_node_crypto.randomUUID)();
5045
+ await createJobWithId(id, userId, query, options, callbackUrl);
5046
+ return id;
5047
+ }
5048
+ async function createJobWithId(id, userId, query, options, callbackUrl) {
5049
+ if (!/^[a-zA-Z0-9-]{1,100}$/.test(id)) throw new Error("job id is invalid");
4980
5050
  await getDb().execute({
4981
5051
  sql: "INSERT INTO jobs (id, user_id, query, options, callback_url) VALUES (?, ?, ?, ?, ?)",
4982
5052
  args: [id, userId, query, JSON.stringify(options), callbackUrl ?? null]
4983
5053
  });
4984
- return id;
4985
5054
  }
4986
5055
  async function createRunningJob(userId, query, options) {
4987
5056
  const id = (0, import_node_crypto.randomUUID)();
@@ -5511,7 +5580,13 @@ async function countActiveJobsForUser(userId) {
5511
5580
  }
5512
5581
  async function claimPendingJob() {
5513
5582
  const db = getDb();
5514
- const res = await db.execute(`SELECT * FROM jobs WHERE status = 'pending' ORDER BY created_at LIMIT 1`);
5583
+ const res = await db.execute(`
5584
+ SELECT * FROM jobs
5585
+ WHERE status = 'pending'
5586
+ AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') != 'inngest'
5587
+ ORDER BY created_at
5588
+ LIMIT 1
5589
+ `);
5515
5590
  const job = res.rows[0] ? rowToRawJob(res.rows[0]) : void 0;
5516
5591
  if (!job) return void 0;
5517
5592
  const upd = await db.execute({
@@ -5520,6 +5595,80 @@ async function claimPendingJob() {
5520
5595
  });
5521
5596
  return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
5522
5597
  }
5598
+ async function claimInngestPaaJob(id) {
5599
+ const db = getDb();
5600
+ const current = await getJob(id);
5601
+ if (!current || current.options.executionOwner !== "inngest" || current.options.serpOnly === true) return void 0;
5602
+ if (current.status === "running") return void 0;
5603
+ if (current.status === "pending") {
5604
+ const updated = await db.execute({
5605
+ sql: `UPDATE jobs
5606
+ SET status = 'running', started_at = COALESCE(started_at, datetime('now')), error = NULL
5607
+ WHERE id = ? AND status = 'pending'`,
5608
+ args: [id]
5609
+ });
5610
+ if (updated.rowsAffected === 0) return getJob(id);
5611
+ return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
5612
+ }
5613
+ return current;
5614
+ }
5615
+ async function checkpointJob(id, result) {
5616
+ const updated = await getDb().execute({
5617
+ sql: `UPDATE jobs SET result = ? WHERE id = ? AND status = 'running'`,
5618
+ args: [JSON.stringify(result), id]
5619
+ });
5620
+ return updated.rowsAffected === 1;
5621
+ }
5622
+ async function listPendingInngestPaaJobs(limit = 25) {
5623
+ const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
5624
+ const rows = await getDb().execute({
5625
+ sql: `SELECT * FROM jobs
5626
+ WHERE status = 'pending'
5627
+ AND json_extract(options, '$.executionOwner') = 'inngest'
5628
+ AND COALESCE(json_extract(options, '$.serpOnly'), 0) = 0
5629
+ ORDER BY created_at
5630
+ LIMIT ?`,
5631
+ args: [safeLimit]
5632
+ });
5633
+ return rows.rows.map((row) => deserialize(rowToRawJob(row)));
5634
+ }
5635
+ async function listRecentTerminalInngestPaaJobs(limit = 25) {
5636
+ const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
5637
+ const rows = await getDb().execute({
5638
+ sql: `SELECT * FROM jobs
5639
+ WHERE status IN ('done', 'failed', 'cancelled')
5640
+ AND json_extract(options, '$.executionOwner') = 'inngest'
5641
+ AND json_extract(options, '$.billingDebitKey') IS NOT NULL
5642
+ AND json_extract(options, '$.billingSettledAt') IS NULL
5643
+ ORDER BY completed_at DESC
5644
+ LIMIT ?`,
5645
+ args: [safeLimit]
5646
+ });
5647
+ return rows.rows.map((row) => deserialize(rowToRawJob(row)));
5648
+ }
5649
+ async function listStaleRunningInngestPaaJobs(limit = 25) {
5650
+ const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
5651
+ const rows = await getDb().execute({
5652
+ sql: `SELECT * FROM jobs
5653
+ WHERE status = 'running'
5654
+ AND started_at <= datetime('now', '-10 minutes')
5655
+ AND json_extract(options, '$.executionOwner') = 'inngest'
5656
+ ORDER BY started_at
5657
+ LIMIT ?`,
5658
+ args: [safeLimit]
5659
+ });
5660
+ return rows.rows.map((row) => deserialize(rowToRawJob(row)));
5661
+ }
5662
+ async function markJobBillingSettled(id, billedMc, settledAt = (/* @__PURE__ */ new Date()).toISOString()) {
5663
+ if (!Number.isSafeInteger(billedMc) || billedMc < 0) throw new Error("job billed amount must be a non-negative integer");
5664
+ const updated = await getDb().execute({
5665
+ sql: `UPDATE jobs
5666
+ SET options = json_set(options, '$.billingSettledAt', ?, '$.billedMc', ?)
5667
+ WHERE id = ?`,
5668
+ args: [settledAt, billedMc, id]
5669
+ });
5670
+ return updated.rowsAffected === 1;
5671
+ }
5523
5672
  async function completeJob(id, result) {
5524
5673
  await getDb().execute({ sql: `UPDATE jobs SET status = 'done', result = ?, completed_at = datetime('now') WHERE id = ?`, args: [JSON.stringify(result), id] });
5525
5674
  }
@@ -15931,15 +16080,15 @@ function formatStructuredError(body, fallback) {
15931
16080
  return `Insufficient credits. Balance: ${body.balance_credits} credits. This call requires ${body.required_credits} credits. Top up at ${body.topup_url}`;
15932
16081
  }
15933
16082
  if (body.error === "mcp_request_timeout") {
15934
- return typeof body.message === "string" ? body.message : "MCP Scraper request timed out and was cancelled.";
16083
+ return publicErrorMessage("mcp_request_timeout");
15935
16084
  }
15936
16085
  if (typeof body.error_code === "string") {
15937
- const message = typeof body.error === "string" ? body.error : typeof body.message === "string" ? body.message : fallback;
16086
+ const code = normalizePublicErrorCode(body.error_code);
16087
+ const message = publicErrorMessage(code);
15938
16088
  const retryable = body.retryable === true ? " Retryable: yes." : "";
15939
- return `${body.error_code}: ${message}${retryable}${errorAttemptsSection(body)}`;
16089
+ return `${code}: ${message}${retryable}${errorAttemptsSection(body)}`;
15940
16090
  }
15941
- if (typeof body.error === "string") return body.error;
15942
- return fallback || "Tool error";
16091
+ return publicErrorMessage("service_unavailable");
15943
16092
  }
15944
16093
  function parseData(raw) {
15945
16094
  const first = raw.content.find((b) => b.type === "text");
@@ -15950,7 +16099,7 @@ function parseData(raw) {
15950
16099
  const data = parsed.result ?? parsed;
15951
16100
  return { data };
15952
16101
  } catch {
15953
- if (raw.isError) return { error: sanitizeVendorText(text2 || "Tool error") };
16102
+ if (raw.isError) return { error: publicErrorMessage("service_unavailable") };
15954
16103
  return { error: "Failed to parse tool response" };
15955
16104
  }
15956
16105
  }
@@ -15992,10 +16141,9 @@ function debugSection(debug) {
15992
16141
  const candidates = Array.isArray(locationEvidence?.candidates) ? locationEvidence.candidates.slice(0, 4).map((c) => `${c.city}, ${c.regionCode} (${c.count})`).join(", ") : "";
15993
16142
  const lines = [
15994
16143
  "\n## Debug",
15995
- `- Proxy mode: ${request.proxyMode ?? kernel.proxyMode ?? "unknown"} \xB7 requested proxy: ${kernel.requestedProxyIdPresent === true ? `yes (${kernel.requestedProxyIdSuffix ?? "redacted"})` : "no"}`,
15996
- `- Proxy resolution: ${proxyResolution.source ?? "unknown"}${proxyResolution.target ? ` \xB7 ${proxyResolution.target.level ?? "city"} ${proxyResolution.target.city}, ${proxyResolution.target.state}` : ""}${proxyResolution.error ? ` \xB7 ${truncate(proxyResolution.error, 180)}` : ""}`,
15997
- `- Browser session: ${kernel.sessionId ?? "unknown"} \xB7 retrieved proxy: ${kernel.retrievedProxyIdPresent === true ? `yes (${kernel.retrievedProxyIdSuffix ?? "redacted"})` : kernel.retrievedProxyIdPresent === false ? "no" : "unknown"}`,
15998
- `- Browser IP geo: ${[network.ip, network.city, network.region, network.country].filter(Boolean).join(" \xB7 ") || network.error || "unknown"}`,
16144
+ `- Proxy mode: ${request.proxyMode ?? kernel.proxyMode ?? "unknown"}`,
16145
+ `- Proxy resolution: ${proxyResolution.target ? `${proxyResolution.target.level ?? "city"} ${proxyResolution.target.city}, ${proxyResolution.target.state}` : proxyResolution.error ? publicErrorMessage("service_unavailable") : "unknown"}`,
16146
+ `- Browser IP geo: ${[network.ip, network.city, network.region, network.country].filter(Boolean).join(" \xB7 ") || (network.error ? publicErrorMessage("service_unavailable") : "unknown")}`,
15999
16147
  `- Google URL: ${truncate(nav.requestedUrl, 240) || "unknown"}`,
16000
16148
  `- Final URL: ${truncate(nav.finalUrl, 240) || "unknown"} \xB7 CAPTCHA: ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 redirected: ${nav.redirected === true ? "yes" : nav.redirected === false ? "no" : "unknown"}`
16001
16149
  ];
@@ -16011,7 +16159,6 @@ function errorAttemptsSection(body) {
16011
16159
  const debug = attempt.debug ?? {};
16012
16160
  const browser = debug.browser ?? {};
16013
16161
  const kernel = browser.browserRuntime ?? browser.kernel ?? {};
16014
- const proxyResolution = kernel.proxyResolution ?? {};
16015
16162
  const network = browser.networkLocation ?? {
16016
16163
  ip: attempt.observedIp ?? attempt.observed_ip,
16017
16164
  city: attempt.observedCity ?? attempt.observed_city,
@@ -16019,10 +16166,7 @@ function errorAttemptsSection(body) {
16019
16166
  };
16020
16167
  const nav = browser.serpNavigation ?? {};
16021
16168
  const geo = [network.ip, network.city, network.region].filter(Boolean).join(" / ") || "geo unknown";
16022
- const sessionId = attempt.browser_session_id ?? attempt.browserSessionIdSuffix ?? attempt.kernel_session_id ?? kernel.sessionId ?? "unknown";
16023
- const cleanupSucceeded2 = attempt.session_cleanup_succeeded ?? attempt.kernel_delete_succeeded;
16024
- const proxySource = proxyResolution.source ?? attempt.proxyResolutionSource;
16025
- return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 session ${sessionId} \xB7 proxy ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"}${proxySource ? `/${proxySource}` : ""} \xB7 ${geo} \xB7 CAPTCHA ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 cleanup ${cleanupSucceeded2 === true ? "yes" : cleanupSucceeded2 === false ? "no" : "unknown"}`;
16169
+ return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 proxy mode ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"} \xB7 ${geo} \xB7 verification challenge ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"}`;
16026
16170
  });
16027
16171
  return `
16028
16172
 
@@ -18533,6 +18677,9 @@ var init_selectors = __esm({
18533
18677
  itemDataInitQ: "data-initq",
18534
18678
  itemQuestionEl: ".JlqpRe",
18535
18679
  answerContainer: ".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",
18680
+ aiAnswerBody: ".pWvJNd",
18681
+ aiCitation: "a.PMDqCb[href]",
18682
+ aiSourceCard: "a.vIWmYe[href], a.Zbfntb[href]",
18536
18683
  sourceTitle: "h3",
18537
18684
  sourceSite: ".VuuXrf",
18538
18685
  sourceCite: "cite",
@@ -23182,6 +23329,968 @@ var init_directory_workflow2 = __esm({
23182
23329
  }
23183
23330
  });
23184
23331
 
23332
+ // src/api/paa-capture-artifacts.ts
23333
+ function paaCaptureArtifactPolicy() {
23334
+ return {
23335
+ prefix: PAA_CAPTURE_ARTIFACT_PREFIX,
23336
+ artifactTtlMs: 7 * DAY_MS,
23337
+ downloadTtlMs: 15 * 60 * 1e3,
23338
+ token: process.env.PAA_CAPTURE_BLOB_READ_WRITE_TOKEN ?? process.env.BLOB_READ_WRITE_TOKEN ?? null
23339
+ };
23340
+ }
23341
+ async function createPaaRawDomArtifact(input) {
23342
+ return createPrivateArtifact({
23343
+ policy: paaCaptureArtifactPolicy(),
23344
+ ownerId: String(input.userId),
23345
+ artifactKey: `${input.jobId}-attempt-${input.attempt}.html.gz`,
23346
+ createdAt: input.createdAt ?? /* @__PURE__ */ new Date(),
23347
+ filename: `${input.jobId}-serp.html.gz`,
23348
+ contentType: "application/gzip",
23349
+ content: input.gzip
23350
+ });
23351
+ }
23352
+ async function tryCreatePaaRawDomArtifact(input) {
23353
+ try {
23354
+ return {
23355
+ artifact: await createPaaRawDomArtifact(input),
23356
+ status: "stored"
23357
+ };
23358
+ } catch (error) {
23359
+ console.error("[paa-capture-artifact] private DOM preservation unavailable:", error instanceof Error ? error.message : String(error));
23360
+ return { artifact: null, status: "unavailable" };
23361
+ }
23362
+ }
23363
+ var DAY_MS, PAA_CAPTURE_ARTIFACT_PREFIX;
23364
+ var init_paa_capture_artifacts = __esm({
23365
+ "src/api/paa-capture-artifacts.ts"() {
23366
+ "use strict";
23367
+ init_private_artifacts();
23368
+ DAY_MS = 24 * 60 * 60 * 1e3;
23369
+ PAA_CAPTURE_ARTIFACT_PREFIX = "paa-captures";
23370
+ }
23371
+ });
23372
+
23373
+ // src/api/paa-harvest-settlement.ts
23374
+ function capturedQuestionCount(result) {
23375
+ if (!result || typeof result !== "object") return 0;
23376
+ const value = result;
23377
+ if (typeof value.totalQuestions === "number") return value.totalQuestions;
23378
+ return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
23379
+ }
23380
+ function finalCost(result, heldMc) {
23381
+ const questions = capturedQuestionCount(result);
23382
+ if (questions <= 0) return 0;
23383
+ return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
23384
+ }
23385
+ async function settlePaaHarvestJob(jobOrId) {
23386
+ const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
23387
+ if (!job) return false;
23388
+ const options = job.options;
23389
+ const debitKey2 = options.billingDebitKey;
23390
+ const heldMc = Number(options.billingHoldMc ?? 0);
23391
+ if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
23392
+ const settlement = await settleDebitMcIdempotent(
23393
+ job.user_id,
23394
+ debitKey2,
23395
+ finalCost(job.result, heldMc),
23396
+ LedgerOperation.PAA_REFUND,
23397
+ "durable PAA harvest settlement",
23398
+ "paa_harvest"
23399
+ );
23400
+ await markJobBillingSettled(job.id, settlement.final_amount_mc);
23401
+ return true;
23402
+ }
23403
+ var init_paa_harvest_settlement = __esm({
23404
+ "src/api/paa-harvest-settlement.ts"() {
23405
+ "use strict";
23406
+ init_db();
23407
+ init_rates();
23408
+ }
23409
+ });
23410
+
23411
+ // src/api/webhook.ts
23412
+ async function deliverWebhook(url, payload, retries = 3) {
23413
+ for (let attempt = 1; attempt <= retries; attempt++) {
23414
+ try {
23415
+ const res = await fetch(url, {
23416
+ method: "POST",
23417
+ headers: { "content-type": "application/json" },
23418
+ body: JSON.stringify(payload),
23419
+ signal: AbortSignal.timeout(1e4)
23420
+ });
23421
+ if (res.ok) return;
23422
+ console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
23423
+ } catch (err) {
23424
+ console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
23425
+ }
23426
+ if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
23427
+ }
23428
+ console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
23429
+ }
23430
+ var init_webhook = __esm({
23431
+ "src/api/webhook.ts"() {
23432
+ "use strict";
23433
+ }
23434
+ });
23435
+
23436
+ // src/paa/durable-capture.ts
23437
+ function remainingSeconds(deadlineMs, ceiling) {
23438
+ return Math.max(1, Math.min(ceiling, Math.floor((deadlineMs - Date.now()) / 1e3)));
23439
+ }
23440
+ async function execute(kernel, sessionId, code, deadlineMs, ceilingSeconds) {
23441
+ if (Date.now() >= deadlineMs) throw new Error("paa_work_deadline_exhausted");
23442
+ const response = await kernel.browsers.playwright.execute(sessionId, {
23443
+ code,
23444
+ timeout_sec: remainingSeconds(deadlineMs, ceilingSeconds)
23445
+ });
23446
+ if (!response.success || response.result === null || response.result === void 0) {
23447
+ throw new Error(response.error || response.stderr || "co-located browser execution failed");
23448
+ }
23449
+ return response.result;
23450
+ }
23451
+ function navigationCode(searchUrl) {
23452
+ return `
23453
+ const wait = ms => page.waitForTimeout(ms);
23454
+ const block = async () => {
23455
+ if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
23456
+ if (await page.locator(${JSON.stringify(PAASelectors.captchaMarker)}).count().catch(() => 0)) return 'captcha';
23457
+ const title = await page.title().catch(() => '');
23458
+ const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
23459
+ if (/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(text)) return 'captcha';
23460
+ if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(title + '\\n' + text.slice(0, 3000))) return 'soft_block';
23461
+ return null;
23462
+ };
23463
+ await page.goto(${JSON.stringify(searchUrl)}, { waitUntil: 'domcontentloaded', timeout: 45000 });
23464
+ let captchaObserved = false;
23465
+ let solverCleared = false;
23466
+ let blocked = await block();
23467
+ const solverDeadline = Date.now() + ${PAA_SOLVER_BUDGET_MS};
23468
+ while (blocked === 'captcha' && Date.now() < solverDeadline) {
23469
+ captchaObserved = true;
23470
+ await wait(4000 + Math.floor(Math.random() * 1500));
23471
+ blocked = await block();
23472
+ if (!blocked) solverCleared = true;
23473
+ }
23474
+ let count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
23475
+ for (let scan = 0; !blocked && count === 0 && scan < 7; scan += 1) {
23476
+ await page.mouse.wheel(0, 520 + Math.floor(Math.random() * 420));
23477
+ await wait(350 + Math.floor(Math.random() * 500));
23478
+ count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
23479
+ }
23480
+ await page.evaluate((itemSelector) => {
23481
+ const state = window;
23482
+ if (state.__mcpTrustedListenerInstalled) return;
23483
+ document.addEventListener('click', event => {
23484
+ if (event.target instanceof Element && event.target.closest(itemSelector)) {
23485
+ state.__mcpLastPaaClickTrusted = event.isTrusted;
23486
+ }
23487
+ }, { capture: true });
23488
+ state.__mcpTrustedListenerInstalled = true;
23489
+ }, ${JSON.stringify(PAASelectors.item)}).catch(() => {});
23490
+ return {
23491
+ outcome: blocked || (count > 0 ? 'ready' : 'no_paa'),
23492
+ finalUrl: page.url(),
23493
+ title: await page.title().catch(() => ''),
23494
+ captchaObserved,
23495
+ solverCleared,
23496
+ environment: await page.evaluate(() => ({
23497
+ userAgent: navigator.userAgent,
23498
+ language: navigator.language,
23499
+ platform: navigator.platform,
23500
+ timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
23501
+ innerWidth: window.innerWidth,
23502
+ innerHeight: window.innerHeight,
23503
+ outerWidth: window.outerWidth,
23504
+ outerHeight: window.outerHeight,
23505
+ screenWidth: window.screen.width,
23506
+ screenHeight: window.screen.height,
23507
+ devicePixelRatio: window.devicePixelRatio,
23508
+ })).catch(() => null),
23509
+ };
23510
+ `;
23511
+ }
23512
+ function recordReaderSource() {
23513
+ return `
23514
+ const clean = value => (value || '').replace(/\\u00a0/g, ' ').replace(/[ \\t]+/g, ' ').replace(/\\n{3,}/g, '\\n\\n').trim();
23515
+ const externalHref = anchor => {
23516
+ const href = anchor && anchor.href || '';
23517
+ if (!/^https?:\\/\\//i.test(href)) return '';
23518
+ try {
23519
+ const url = new URL(href);
23520
+ if (/(^|\\.)google(usercontent)?\\.[a-z.]+$/i.test(url.hostname)) {
23521
+ const redirected = url.searchParams.get('q') || url.searchParams.get('url');
23522
+ return redirected && redirected.startsWith('http') ? redirected : '';
23523
+ }
23524
+ } catch {}
23525
+ return href;
23526
+ };
23527
+ const sourceLabel = value => clean(value)
23528
+ .replace(/\\s*\\(\\+\\d+\\)\\s*-\\s*View related links.*$/i, '')
23529
+ .replace(/\\s*-\\s*Opens in new tab.*$/i, '').trim();
23530
+ const readRecords = () => Array.from(document.querySelectorAll(SEL.item)).map(item => {
23531
+ const question = clean(item.getAttribute(SEL.itemDataQ)
23532
+ || item.getAttribute(SEL.itemDataInitQ)
23533
+ || item.querySelector(SEL.itemQuestionEl)?.innerText);
23534
+ const expanded = item.classList.contains(SEL.expandedClass)
23535
+ || item.querySelector(SEL.clickTarget)?.getAttribute('aria-expanded') === 'true';
23536
+ const container = expanded ? item.querySelector(SEL.answerContainer) : null;
23537
+ const answerElement = container?.querySelector(SEL.aiAnswerBody) || container;
23538
+ const anchors = expanded ? Array.from(answerElement?.querySelectorAll(
23539
+ SEL.aiCitation + ',' + SEL.aiSourceCard + ',a[href]'
23540
+ ) || []) : [];
23541
+ const byUrl = new Map();
23542
+ for (const anchor of anchors) {
23543
+ const url = externalHref(anchor);
23544
+ if (!url || byUrl.has(url)) continue;
23545
+ let site = '';
23546
+ try { site = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
23547
+ byUrl.set(url, { title: sourceLabel(anchor.getAttribute('aria-label')) || clean(anchor.textContent), site, url });
23548
+ }
23549
+ const sources = Array.from(byUrl.values());
23550
+ let sourceTitle = expanded ? clean(item.querySelector(SEL.sourceTitle)?.textContent) : '';
23551
+ let sourceSite = expanded ? clean(item.querySelector(SEL.sourceSite)?.textContent) : '';
23552
+ let sourceCite = expanded ? externalHref(item.querySelector(SEL.sourceCite)) : '';
23553
+ const primary = sources.find(source => source.title.length > 2) || sources[0];
23554
+ if (!sourceCite && primary) {
23555
+ sourceTitle ||= primary.title;
23556
+ sourceSite ||= primary.site;
23557
+ sourceCite = primary.url;
23558
+ }
23559
+ return {
23560
+ question,
23561
+ answer: clean(answerElement?.innerText || answerElement?.textContent),
23562
+ sourceTitle,
23563
+ sourceSite,
23564
+ sourceCite,
23565
+ sources,
23566
+ depth: 1,
23567
+ parentQuestion: null,
23568
+ };
23569
+ }).filter(record => record.question);
23570
+ `;
23571
+ }
23572
+ function expansionCode(clickedQuestions) {
23573
+ return `
23574
+ const SEL = ${JSON.stringify(PAASelectors)};
23575
+ ${recordReaderSource()}
23576
+ const deliveryBlock = async () => {
23577
+ if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
23578
+ if (await page.locator(SEL.captchaMarker).count().catch(() => 0)) return 'captcha';
23579
+ const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
23580
+ if (/recaptcha|unusual traffic|are you a robot|about this page/i.test(text)) return 'captcha';
23581
+ if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(text.slice(0, 3000))) return 'soft_block';
23582
+ return null;
23583
+ };
23584
+ const blocked = await deliveryBlock();
23585
+ const records = await page.evaluate(({ SEL }) => {
23586
+ ${recordReaderSource()}
23587
+ return readRecords();
23588
+ }, { SEL });
23589
+ if (blocked) return {
23590
+ outcome: blocked, targetQuestion: null, records, newQuestions: [],
23591
+ finalUrl: page.url(), title: await page.title().catch(() => ''),
23592
+ interactionTrusted: null, clickError: null,
23593
+ };
23594
+ const clicked = new Set(${JSON.stringify(clickedQuestions)});
23595
+ const targetQuestion = records.map(record => record.question).find(question => !clicked.has(question)) || null;
23596
+ if (!targetQuestion) return {
23597
+ outcome: 'exhausted', targetQuestion: null, records, newQuestions: [],
23598
+ finalUrl: page.url(), title: await page.title().catch(() => ''),
23599
+ interactionTrusted: null, clickError: null,
23600
+ };
23601
+ const items = page.locator(SEL.item);
23602
+ let targetIndex = -1;
23603
+ for (let index = 0, count = await items.count().catch(() => 0); index < count; index += 1) {
23604
+ const question = await items.nth(index).evaluate((item, SEL) =>
23605
+ item.getAttribute(SEL.itemDataQ)
23606
+ || item.getAttribute(SEL.itemDataInitQ)
23607
+ || item.querySelector(SEL.itemQuestionEl)?.innerText?.trim()
23608
+ || '', SEL).catch(() => '');
23609
+ if (question.replace(/\\s+/g, ' ').trim() === targetQuestion) { targetIndex = index; break; }
23610
+ }
23611
+ if (targetIndex < 0) return {
23612
+ outcome: 'exhausted', targetQuestion, records, newQuestions: [],
23613
+ finalUrl: page.url(), title: await page.title().catch(() => ''),
23614
+ interactionTrusted: null, clickError: 'question disappeared before click',
23615
+ };
23616
+ const beforeQuestions = new Set(records.map(record => record.question));
23617
+ const item = items.nth(targetIndex);
23618
+ await item.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => {});
23619
+ await page.waitForTimeout(180 + Math.floor(Math.random() * 420));
23620
+ await page.evaluate(() => { window.__mcpLastPaaClickTrusted = null; }).catch(() => {});
23621
+ let clickError = null;
23622
+ try {
23623
+ await item.locator(SEL.clickTarget).first().click({
23624
+ timeout: 6000,
23625
+ delay: 55 + Math.floor(Math.random() * 120),
23626
+ });
23627
+ } catch (error) {
23628
+ clickError = String(error && error.message || error).slice(0, 240);
23629
+ }
23630
+ const interactionTrusted = await page.evaluate(() => window.__mcpLastPaaClickTrusted ?? null).catch(() => null);
23631
+ await page.waitForTimeout(650 + Math.floor(Math.random() * 750));
23632
+ const afterRecords = await page.evaluate(({ SEL }) => {
23633
+ ${recordReaderSource()}
23634
+ return readRecords();
23635
+ }, { SEL });
23636
+ return {
23637
+ outcome: clickError ? 'exhausted' : 'clicked',
23638
+ targetQuestion,
23639
+ records: afterRecords,
23640
+ newQuestions: afterRecords.map(record => record.question).filter(question => !beforeQuestions.has(question)),
23641
+ finalUrl: page.url(),
23642
+ title: await page.title().catch(() => ''),
23643
+ interactionTrusted,
23644
+ clickError,
23645
+ };
23646
+ `;
23647
+ }
23648
+ function finalCaptureCode(maxQuestions, parentByQuestion) {
23649
+ const surfaces = {
23650
+ aiOverview: AIOverviewSelectors,
23651
+ aiMode: AIModeSelectors,
23652
+ organic: OrganicSelectors,
23653
+ localPack: LocalPackSelectors,
23654
+ video: VideoSelectors,
23655
+ forum: ForumSelectors,
23656
+ whatPeopleSaying: WhatPeopleSayingSelectors
23657
+ };
23658
+ return `
23659
+ const SEL = ${JSON.stringify(PAASelectors)};
23660
+ const SURF = ${JSON.stringify(surfaces)};
23661
+ const PARENTS = ${JSON.stringify(parentByQuestion)};
23662
+ ${recordReaderSource()}
23663
+ const capture = await page.evaluate(({ SEL, SURF, PARENTS, maxQuestions }) => {
23664
+ ${recordReaderSource()}
23665
+ const unique = (items, key) => {
23666
+ const seen = new Set();
23667
+ return items.filter(item => { const value = key(item); if (!value || seen.has(value)) return false; seen.add(value); return true; });
23668
+ };
23669
+ const paa = readRecords().map(record => {
23670
+ const parentQuestion = PARENTS[record.question] || null;
23671
+ let depth = 1, parent = parentQuestion;
23672
+ const seen = new Set();
23673
+ while (parent && !seen.has(parent)) { seen.add(parent); depth += 1; parent = PARENTS[parent] || null; }
23674
+ return { ...record, depth, parentQuestion };
23675
+ }).slice(0, maxQuestions);
23676
+ const organicResults = [];
23677
+ document.querySelectorAll(SURF.organic.result).forEach(card => {
23678
+ const titleElement = card.querySelector(SURF.organic.title);
23679
+ const title = clean(titleElement?.textContent);
23680
+ const link = titleElement?.closest('a');
23681
+ const url = externalHref(link);
23682
+ if (!title || !url) return;
23683
+ let domain = '';
23684
+ try { domain = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
23685
+ const rating = card.querySelector(SURF.organic.ratingWrap);
23686
+ organicResults.push({
23687
+ position: organicResults.length + 1, title, url, domain,
23688
+ cite: clean(card.querySelector(SURF.organic.cite)?.textContent) || null,
23689
+ snippet: clean(card.querySelector(SURF.organic.snippet)?.textContent) || null,
23690
+ isRedditStyle: Boolean(card.querySelector(SURF.organic.redditCite)),
23691
+ inlineRating: rating ? {
23692
+ value: clean(rating.querySelector(SURF.organic.ratingValue)?.textContent),
23693
+ count: clean(rating.querySelector(SURF.organic.reviewCount)?.textContent),
23694
+ } : null,
23695
+ });
23696
+ });
23697
+ const localPack = [];
23698
+ document.querySelectorAll(SURF.localPack.card).forEach(card => {
23699
+ const name = clean(card.querySelector(SURF.localPack.name)?.textContent);
23700
+ if (!name) return;
23701
+ const links = Array.from(card.querySelectorAll('a[href]'));
23702
+ let cid = card.querySelector('a[data-cid]')?.getAttribute('data-cid') || null;
23703
+ for (const link of links) {
23704
+ if (cid) break;
23705
+ const match = link.href.match(/[?&]cid=(\\d+)/);
23706
+ if (match) cid = match[1];
23707
+ }
23708
+ localPack.push({
23709
+ position: localPack.length + 1, name, cid,
23710
+ rating: clean(card.querySelector(SURF.localPack.ratingValue)?.textContent) || null,
23711
+ reviewCount: clean(card.querySelector(SURF.localPack.reviewCount)?.textContent).replace(/[()]/g, '') || null,
23712
+ metadata: [], websiteUrl: links.map(externalHref).find(Boolean) || null,
23713
+ directionsUrl: links.find(link => link.href.includes('google.com/maps'))?.href || null,
23714
+ });
23715
+ });
23716
+ const videos = unique(Array.from(document.querySelectorAll(SURF.video.item)).map(link => {
23717
+ const url = externalHref(link);
23718
+ const raw = clean(link.textContent);
23719
+ const platform = /youtu/i.test(url) ? 'YouTube' : /tiktok/i.test(url) ? 'TikTok' : /instagram/i.test(url) ? 'Instagram' : /facebook/i.test(url) ? 'Facebook' : '';
23720
+ return { type: 'video', title: raw, channel: '', platform, duration: raw.match(/\\b\\d{1,2}:\\d{2}\\b/)?.[0] || '', url };
23721
+ }).filter(item => item.title && item.url), item => item.url);
23722
+ const forums = unique(Array.from(document.querySelectorAll(SURF.forum.item)).map(link => ({
23723
+ title: clean(link.querySelector(SURF.forum.title)?.textContent),
23724
+ source: clean(link.querySelector(SURF.forum.source)?.textContent),
23725
+ url: externalHref(link),
23726
+ })).filter(item => item.title && item.url), item => item.url);
23727
+ const socialCards = Array.from(document.querySelectorAll(SURF.whatPeopleSaying.card));
23728
+ const whatPeopleSaying = unique(socialCards.map(card => {
23729
+ const link = card.querySelector(SURF.whatPeopleSaying.cardLink);
23730
+ const url = externalHref(link);
23731
+ const title = clean(card.querySelector(SURF.whatPeopleSaying.titleH1)?.textContent || card.querySelector(SURF.whatPeopleSaying.titleDiv)?.textContent);
23732
+ const source = clean(card.querySelector(SURF.whatPeopleSaying.ytChannel)?.textContent || card.querySelector(SURF.whatPeopleSaying.source)?.textContent);
23733
+ const platform = clean(card.querySelector(SURF.whatPeopleSaying.platformBadge)?.textContent) || (/youtube/i.test(url) ? 'YouTube' : '');
23734
+ const identity = (platform + source + url).toLowerCase();
23735
+ const type = identity.includes('reddit') ? 'reddit' : identity.includes('facebook') ? 'facebook' : identity.includes('instagram') ? 'instagram' : identity.includes('tiktok') ? 'tiktok' : identity.includes('youtube') ? 'youtube' : 'unknown';
23736
+ return { type, title, url, source, platform, popularComment: null, engagement: '', date: '', duration: null, authorNote: null };
23737
+ }).filter(item => item.title || item.url), item => item.url || item.type + ':' + item.title);
23738
+ const aioRoot = document.querySelector(SURF.aiOverview.root) || document.querySelector(SURF.aiOverview.legacyRoot);
23739
+ const aimRoot = document.querySelector(SURF.aiMode.root);
23740
+ const citations = root => unique(Array.from(root?.querySelectorAll('a[href]') || []).map(link => ({
23741
+ text: clean(link.textContent || link.getAttribute('aria-label')), href: externalHref(link),
23742
+ })).filter(item => item.href), item => item.href);
23743
+ const aiOverview = { detected: Boolean(aioRoot), text: aioRoot ? clean(aioRoot.innerText || aioRoot.textContent) : null, citations: citations(aioRoot), expanded: true, fullyExpanded: true, sections: [] };
23744
+ const aiMode = { detected: Boolean(aimRoot), text: aimRoot ? clean(aimRoot.innerText || aimRoot.textContent) : null, citations: citations(aimRoot) };
23745
+ const html = document.documentElement.outerHTML;
23746
+ const kgIds = unique(Array.from(html.matchAll(/\\/g\\/[a-zA-Z0-9_-]{5,20}/g)).map(match => match[0]), value => value);
23747
+ const cids = unique([
23748
+ ...localPack.map(item => item.cid).filter(Boolean),
23749
+ ...Array.from(html.matchAll(/[?&]cid=(\\d+)/g)).map(match => match[1]),
23750
+ ], value => value);
23751
+ const gcids = unique(Array.from(html.matchAll(/gcid:[a-zA-Z0-9_]+/g)).map(match => match[0]), value => value);
23752
+ const entities = localPack.filter(item => item.cid).map(item => ({ name: item.name, kgId: null, cid: item.cid, gcid: null }));
23753
+ const headings = unique(Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"]')).map(node => clean(node.textContent)).filter(text => text.length > 1 && text.length < 160), value => value);
23754
+ const relatedSearches = unique(Array.from(document.querySelectorAll('a[href*="/search?"]')).map(link => ({
23755
+ text: clean(link.textContent), url: link.href,
23756
+ })).filter(item => item.text && /related/i.test(item.url + ' ' + item.text)), item => item.url);
23757
+ const counts = {
23758
+ peopleAlsoAsk: paa.length, organic: organicResults.length, localPack: localPack.length,
23759
+ videos: videos.length, discussions: forums.length, whatPeopleSaying: whatPeopleSaying.length,
23760
+ aiOverview: aiOverview.detected ? 1 : 0, aiMode: aiMode.detected ? 1 : 0,
23761
+ relatedSearches: relatedSearches.length,
23762
+ };
23763
+ return {
23764
+ paa, organicResults: unique(organicResults, item => item.url), localPack: unique(localPack, item => item.cid || item.name),
23765
+ videos, forums, whatPeopleSaying, aiOverview, aiMode,
23766
+ surface: window.google?.sn === 'aim' ? 'aim' : window.google?.sn === 'web' ? 'web' : 'unknown',
23767
+ entityIds: { entities, kgIds, cids, gcids },
23768
+ serpFeatures: { sectionHeadings: headings, detected: Object.fromEntries(Object.entries(counts).map(([key, count]) => [key, count > 0])), counts },
23769
+ relatedSearches,
23770
+ };
23771
+ }, { SEL, SURF, PARENTS, maxQuestions: ${maxQuestions} });
23772
+ return capture;
23773
+ `;
23774
+ }
23775
+ function rawChunkCode(offset, length) {
23776
+ return `return await page.evaluate(({ offset, length }) => ({ offset, chunk: window.__mcpPaaRawDom.slice(offset, offset + length) }), { offset: ${offset}, length: ${length} });`;
23777
+ }
23778
+ function buildPublicResult(input) {
23779
+ const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
23780
+ const selected = input.records;
23781
+ const flat = selected.map((record) => ({
23782
+ seed_query: input.query,
23783
+ question: record.question,
23784
+ answer: record.answer,
23785
+ source_title: record.sourceTitle,
23786
+ source_site: record.sourceSite,
23787
+ source_cite: record.sourceCite,
23788
+ depth: record.depth,
23789
+ parent_question: record.parentQuestion ?? "",
23790
+ extracted_at: extractedAt
23791
+ }));
23792
+ const tree = selected.map((record) => ({
23793
+ question: record.question,
23794
+ answer: record.answer || null,
23795
+ sourceTitle: record.sourceTitle || null,
23796
+ sourceSite: record.sourceSite || null,
23797
+ sourceCite: record.sourceCite || null,
23798
+ depth: record.depth,
23799
+ parentQuestion: record.parentQuestion,
23800
+ children: []
23801
+ }));
23802
+ return {
23803
+ seed: input.query,
23804
+ location: input.location,
23805
+ extractedAt,
23806
+ diagnostics: {
23807
+ completionStatus: selected.length > 0 ? "paa_found" : "no_paa",
23808
+ problem: null,
23809
+ resultQuality: input.partial ? "partial" : "complete",
23810
+ degradedResult: input.partial,
23811
+ retryRecommended: false,
23812
+ completeness: {
23813
+ paaWithoutAnswer: selected.filter((record) => !record.answer).length,
23814
+ paaWithoutSource: selected.filter((record) => !record.sourceCite).length,
23815
+ paaAnswersRecovered: 0,
23816
+ aioShareCaptured: null
23817
+ }
23818
+ },
23819
+ totalQuestions: selected.length,
23820
+ surface: input.capture.surface,
23821
+ aiOverview: input.capture.aiOverview,
23822
+ aiMode: input.capture.aiMode,
23823
+ whatPeopleSaying: input.capture.whatPeopleSaying,
23824
+ tree,
23825
+ flat,
23826
+ videos: input.capture.videos,
23827
+ forums: input.capture.forums,
23828
+ organicResults: input.capture.organicResults,
23829
+ localPack: input.capture.localPack,
23830
+ entityIds: input.capture.entityIds,
23831
+ serpFeatures: input.capture.serpFeatures,
23832
+ relatedSearches: input.capture.relatedSearches,
23833
+ stats: {
23834
+ seed: input.query,
23835
+ totalQuestions: selected.length,
23836
+ maxDepthReached: selected.reduce((max, item) => Math.max(max, item.depth), 0),
23837
+ durationMs: input.durationMs,
23838
+ errorCount: input.partial ? 1 : 0
23839
+ },
23840
+ capture: {
23841
+ execution: "kernel_colocated_playwright",
23842
+ stealth: true,
23843
+ managedProxy: "kernel_default",
23844
+ metadataOverrides: false,
23845
+ locationSignal: input.location ? "uule_only" : "none",
23846
+ attempt: input.attempt,
23847
+ finalUrl: input.navigation.finalUrl,
23848
+ captchaObserved: input.navigation.captchaObserved,
23849
+ solverCleared: input.navigation.solverCleared,
23850
+ environment: input.navigation.environment,
23851
+ rawDomSha256: input.rawDomSha256
23852
+ }
23853
+ };
23854
+ }
23855
+ async function runDurablePaaCapture(input) {
23856
+ const startedAt = input.startedAtMs ?? Date.now();
23857
+ const workDeadline = startedAt + PAA_BROWSER_WORK_BUDGET_MS;
23858
+ const location2 = input.location?.trim() || null;
23859
+ const params = new URLSearchParams({
23860
+ q: input.query,
23861
+ gl: input.gl ?? "us",
23862
+ hl: input.hl ?? "en",
23863
+ pws: "0"
23864
+ });
23865
+ if (location2) params.set("uule", encodeUule(normalizeLocation(location2)));
23866
+ const searchUrl = `https://www.google.com/search?${params.toString()}`;
23867
+ const kernel = new import_sdk8.default({ apiKey: input.apiKey });
23868
+ let lastCheckpoint = {
23869
+ phase: "navigating",
23870
+ attempt: 1,
23871
+ observedAt: (/* @__PURE__ */ new Date()).toISOString(),
23872
+ records: [],
23873
+ clickedQuestions: [],
23874
+ finalUrl: null,
23875
+ captchaObserved: false,
23876
+ solverCleared: false,
23877
+ error: null
23878
+ };
23879
+ const persist = async (patch) => {
23880
+ lastCheckpoint = { ...lastCheckpoint, ...patch, observedAt: (/* @__PURE__ */ new Date()).toISOString() };
23881
+ await input.onCheckpoint?.(lastCheckpoint);
23882
+ };
23883
+ for (let attempt = 1; attempt <= 2; attempt += 1) {
23884
+ if (Date.now() >= workDeadline) break;
23885
+ let sessionId = null;
23886
+ const records = /* @__PURE__ */ new Map();
23887
+ const clicked = /* @__PURE__ */ new Set();
23888
+ const parents = {};
23889
+ let navigation = null;
23890
+ try {
23891
+ await persist({ phase: "navigating", attempt, records: [], clickedQuestions: [], error: null });
23892
+ const browser = await kernel.browsers.create({
23893
+ stealth: true,
23894
+ timeout_seconds: Math.max(60, Math.min(280, Math.ceil((workDeadline - Date.now()) / 1e3) + 20))
23895
+ });
23896
+ sessionId = browser.session_id;
23897
+ navigation = await execute(kernel, sessionId, navigationCode(searchUrl), workDeadline, 110);
23898
+ await persist({
23899
+ phase: navigation.outcome === "ready" ? "expanding" : "blocked",
23900
+ finalUrl: navigation.finalUrl,
23901
+ captchaObserved: navigation.captchaObserved,
23902
+ solverCleared: navigation.solverCleared
23903
+ });
23904
+ if (navigation.outcome === "captcha" || navigation.outcome === "soft_block") {
23905
+ if (attempt === 1 && records.size === 0) continue;
23906
+ throw new Error(navigation.outcome);
23907
+ }
23908
+ let stagnant = 0;
23909
+ for (let step = 0; step < input.maxQuestions * 2 && Date.now() < workDeadline - 35e3; step += 1) {
23910
+ const observed = await execute(
23911
+ kernel,
23912
+ sessionId,
23913
+ expansionCode(Array.from(clicked)),
23914
+ workDeadline,
23915
+ 25
23916
+ );
23917
+ const beforeSize = records.size;
23918
+ for (const record of observed.records) {
23919
+ const existing = records.get(record.question);
23920
+ records.set(record.question, {
23921
+ ...record,
23922
+ answer: record.answer || existing?.answer || "",
23923
+ sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
23924
+ sourceSite: record.sourceSite || existing?.sourceSite || "",
23925
+ sourceCite: record.sourceCite || existing?.sourceCite || "",
23926
+ sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
23927
+ parentQuestion: parents[record.question] ?? null,
23928
+ depth: parents[record.question] ? 2 : 1
23929
+ });
23930
+ }
23931
+ if (observed.outcome === "clicked" && observed.targetQuestion) {
23932
+ clicked.add(observed.targetQuestion);
23933
+ for (const question of observed.newQuestions) {
23934
+ if (!parents[question]) parents[question] = observed.targetQuestion;
23935
+ }
23936
+ }
23937
+ const selected2 = Array.from(records.values()).slice(0, input.maxQuestions);
23938
+ await persist({
23939
+ phase: "expanding",
23940
+ records: selected2,
23941
+ clickedQuestions: Array.from(clicked),
23942
+ finalUrl: observed.finalUrl,
23943
+ captchaObserved: lastCheckpoint.captchaObserved || observed.outcome === "captcha"
23944
+ });
23945
+ if (observed.outcome === "captcha" || observed.outcome === "soft_block") break;
23946
+ if (selected2.length >= input.maxQuestions && selected2.every((record) => clicked.has(record.question))) break;
23947
+ stagnant = records.size === beforeSize || observed.outcome === "exhausted" ? stagnant + 1 : 0;
23948
+ if (stagnant >= 4) break;
23949
+ }
23950
+ const finalCapture = await execute(
23951
+ kernel,
23952
+ sessionId,
23953
+ finalCaptureCode(input.maxQuestions, parents),
23954
+ workDeadline,
23955
+ 45
23956
+ );
23957
+ for (const record of finalCapture.paa) {
23958
+ const existing = records.get(record.question);
23959
+ records.set(record.question, {
23960
+ ...record,
23961
+ answer: record.answer || existing?.answer || "",
23962
+ sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
23963
+ sourceSite: record.sourceSite || existing?.sourceSite || "",
23964
+ sourceCite: record.sourceCite || existing?.sourceCite || "",
23965
+ sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
23966
+ parentQuestion: parents[record.question] ?? record.parentQuestion,
23967
+ depth: record.depth
23968
+ });
23969
+ }
23970
+ let rawDomGzip = null;
23971
+ let rawDomSha256 = null;
23972
+ if (Date.now() < workDeadline - 12e3) {
23973
+ const prepared = await execute(
23974
+ kernel,
23975
+ sessionId,
23976
+ RAW_PREPARE_CODE,
23977
+ workDeadline,
23978
+ 35
23979
+ );
23980
+ const chunks = [];
23981
+ for (let offset = 0; offset < prepared.chars && Date.now() < workDeadline - 5e3; offset += 18e4) {
23982
+ const part = await execute(
23983
+ kernel,
23984
+ sessionId,
23985
+ rawChunkCode(offset, 18e4),
23986
+ workDeadline,
23987
+ 15
23988
+ );
23989
+ if (part.offset !== offset) throw new Error("raw_dom_chunk_offset_mismatch");
23990
+ chunks.push(part.chunk);
23991
+ }
23992
+ if (chunks.join("").length === prepared.chars) {
23993
+ const transported = Buffer.from(chunks.join(""), "base64");
23994
+ if (prepared.encoding === "gzip") rawDomGzip = transported;
23995
+ else {
23996
+ const { gzipSync } = await import("zlib");
23997
+ rawDomGzip = gzipSync(transported);
23998
+ }
23999
+ rawDomSha256 = (0, import_node_crypto11.createHash)("sha256").update(rawDomGzip).digest("hex");
24000
+ }
24001
+ }
24002
+ const selected = Array.from(records.values()).slice(0, input.maxQuestions);
24003
+ const partial = selected.length < input.maxQuestions || selected.some((record) => !record.answer || !record.sourceCite);
24004
+ await persist({ phase: "captured", records: selected, clickedQuestions: Array.from(clicked) });
24005
+ return {
24006
+ result: buildPublicResult({
24007
+ query: input.query,
24008
+ location: location2,
24009
+ capture: finalCapture,
24010
+ records: selected,
24011
+ navigation,
24012
+ durationMs: Date.now() - startedAt,
24013
+ partial,
24014
+ attempt,
24015
+ rawDomSha256
24016
+ }),
24017
+ rawDomGzip,
24018
+ rawDomSha256,
24019
+ attempt,
24020
+ partial,
24021
+ checkpoint: lastCheckpoint
24022
+ };
24023
+ } catch (error) {
24024
+ const message = error instanceof Error ? error.message : String(error);
24025
+ await persist({
24026
+ phase: records.size > 0 ? "captured" : "failed",
24027
+ records: Array.from(records.values()).slice(0, input.maxQuestions),
24028
+ clickedQuestions: Array.from(clicked),
24029
+ error: message.slice(0, 500)
24030
+ });
24031
+ if (records.size > 0) {
24032
+ const emptyCapture = {
24033
+ paa: [],
24034
+ organicResults: [],
24035
+ localPack: [],
24036
+ videos: [],
24037
+ forums: [],
24038
+ whatPeopleSaying: [],
24039
+ aiOverview: { detected: false, text: null, citations: [] },
24040
+ aiMode: { detected: false, text: null, citations: [] },
24041
+ surface: "unknown",
24042
+ entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
24043
+ serpFeatures: { sectionHeadings: [], detected: {}, counts: {} },
24044
+ relatedSearches: []
24045
+ };
24046
+ return {
24047
+ result: buildPublicResult({
24048
+ query: input.query,
24049
+ location: location2,
24050
+ capture: emptyCapture,
24051
+ records: Array.from(records.values()).slice(0, input.maxQuestions),
24052
+ navigation: navigation ?? {
24053
+ outcome: "soft_block",
24054
+ finalUrl: lastCheckpoint.finalUrl ?? "",
24055
+ title: "",
24056
+ captchaObserved: lastCheckpoint.captchaObserved,
24057
+ solverCleared: lastCheckpoint.solverCleared,
24058
+ environment: null
24059
+ },
24060
+ durationMs: Date.now() - startedAt,
24061
+ partial: true,
24062
+ attempt,
24063
+ rawDomSha256: null
24064
+ }),
24065
+ rawDomGzip: null,
24066
+ rawDomSha256: null,
24067
+ attempt,
24068
+ partial: true,
24069
+ checkpoint: lastCheckpoint
24070
+ };
24071
+ }
24072
+ if (attempt === 2 || Date.now() >= workDeadline - 35e3) throw error;
24073
+ } finally {
24074
+ if (sessionId) {
24075
+ await kernel.browsers.deleteByID(sessionId).catch(() => void 0);
24076
+ }
24077
+ }
24078
+ }
24079
+ throw new Error("paa_work_deadline_exhausted_before_capture");
24080
+ }
24081
+ var import_sdk8, import_node_crypto11, PAA_INVOCATION_BUDGET_MS, PAA_BROWSER_WORK_BUDGET_MS, PAA_SOLVER_BUDGET_MS, RAW_PREPARE_CODE;
24082
+ var init_durable_capture = __esm({
24083
+ "src/paa/durable-capture.ts"() {
24084
+ "use strict";
24085
+ import_sdk8 = __toESM(require("@onkernel/sdk"), 1);
24086
+ import_node_crypto11 = require("crypto");
24087
+ init_selectors();
24088
+ init_uule();
24089
+ PAA_INVOCATION_BUDGET_MS = 28e4;
24090
+ PAA_BROWSER_WORK_BUDGET_MS = 25e4;
24091
+ PAA_SOLVER_BUDGET_MS = 6e4;
24092
+ RAW_PREPARE_CODE = `
24093
+ const result = await page.evaluate(async () => {
24094
+ const source = new TextEncoder().encode(document.documentElement.outerHTML);
24095
+ const compressed = typeof CompressionStream === 'undefined'
24096
+ ? source
24097
+ : new Uint8Array(await new Response(new Blob([source]).stream().pipeThrough(new CompressionStream('gzip'))).arrayBuffer());
24098
+ const pieces = [];
24099
+ for (let offset = 0; offset < compressed.length; offset += 32768) {
24100
+ pieces.push(String.fromCharCode(...compressed.subarray(offset, offset + 32768)));
24101
+ }
24102
+ window.__mcpPaaRawDom = btoa(pieces.join(''));
24103
+ return { encoding: typeof CompressionStream === 'undefined' ? 'plain' : 'gzip', chars: window.__mcpPaaRawDom.length };
24104
+ });
24105
+ return result;
24106
+ `;
24107
+ }
24108
+ });
24109
+
24110
+ // src/inngest/functions/paa-harvest.ts
24111
+ function questionCount(result) {
24112
+ if (!result || typeof result !== "object") return 0;
24113
+ const value = result;
24114
+ if (typeof value.totalQuestions === "number") return value.totalQuestions;
24115
+ return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
24116
+ }
24117
+ var paaHarvestFn;
24118
+ var init_paa_harvest = __esm({
24119
+ "src/inngest/functions/paa-harvest.ts"() {
24120
+ "use strict";
24121
+ init_client();
24122
+ init_browser_service_env();
24123
+ init_db();
24124
+ init_paa_capture_artifacts();
24125
+ init_paa_harvest_settlement();
24126
+ init_webhook();
24127
+ init_durable_capture();
24128
+ paaHarvestFn = inngest.createFunction(
24129
+ {
24130
+ id: "paa-harvest",
24131
+ retries: 0,
24132
+ triggers: [{ event: "mcp-scraper/paa.requested" }],
24133
+ onFailure: async ({ event }) => {
24134
+ const jobId2 = event?.data?.event?.data?.jobId;
24135
+ if (!jobId2) return;
24136
+ const current = await getJob(jobId2);
24137
+ if (!current) return;
24138
+ if (current.status === "done" || current.status === "failed" || current.status === "cancelled") {
24139
+ await settlePaaHarvestJob(jobId2).catch((error) => {
24140
+ console.error("[paa-harvest/onFailure] terminal settlement pending:", error instanceof Error ? error.message : String(error));
24141
+ });
24142
+ return;
24143
+ }
24144
+ const message = String(event?.data?.error?.message ?? "durable PAA invocation failed").slice(0, 1e3);
24145
+ await failJob(jobId2, message);
24146
+ await settlePaaHarvestJob(jobId2).catch((error) => {
24147
+ console.error("[paa-harvest/onFailure] settlement pending:", error instanceof Error ? error.message : String(error));
24148
+ });
24149
+ }
24150
+ },
24151
+ async ({ event }) => {
24152
+ const invocationStartedAt = Date.now();
24153
+ const job = await claimInngestPaaJob(event.data.jobId);
24154
+ if (!job) return { jobId: event.data.jobId, status: "missing_or_not_paa" };
24155
+ if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
24156
+ await settlePaaHarvestJob(job.id);
24157
+ return { jobId: job.id, status: job.status };
24158
+ }
24159
+ const options = job.options;
24160
+ const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
24161
+ const apiKey = browserServiceApiKey();
24162
+ if (!apiKey) throw new Error("browser service is not configured");
24163
+ const attemptStartedAt = (/* @__PURE__ */ new Date()).toISOString();
24164
+ await startHarvestAttempt({
24165
+ jobId: job.id,
24166
+ userId: job.user_id,
24167
+ attemptNumber: 1,
24168
+ maxAttempts: 2,
24169
+ query: options.query ?? job.query,
24170
+ location: options.location ?? null,
24171
+ maxQuestions,
24172
+ startedAt: attemptStartedAt
24173
+ });
24174
+ const checkpointState = { value: null };
24175
+ try {
24176
+ const capture = await runDurablePaaCapture({
24177
+ apiKey,
24178
+ query: options.query ?? job.query,
24179
+ location: options.location ?? null,
24180
+ gl: options.gl ?? "us",
24181
+ hl: options.hl ?? "en",
24182
+ maxQuestions,
24183
+ startedAtMs: invocationStartedAt,
24184
+ onCheckpoint: async (checkpoint) => {
24185
+ checkpointState.value = checkpoint;
24186
+ const persisted = await checkpointJob(job.id, {
24187
+ progress: checkpoint,
24188
+ durable: {
24189
+ invocationBudgetMs: PAA_INVOCATION_BUDGET_MS,
24190
+ workBudgetMs: 25e4,
24191
+ cleanupReserveMs: 3e4
24192
+ }
24193
+ });
24194
+ if (!persisted) throw new Error("job_left_running_state_during_capture");
24195
+ }
24196
+ });
24197
+ const artifactResult = capture.rawDomGzip ? await tryCreatePaaRawDomArtifact({
24198
+ userId: job.user_id,
24199
+ jobId: job.id,
24200
+ attempt: capture.attempt,
24201
+ gzip: capture.rawDomGzip
24202
+ }) : { artifact: null, status: "unavailable" };
24203
+ const artifact = artifactResult.artifact;
24204
+ const artifactMissing = capture.rawDomGzip !== null && artifactResult.status !== "stored";
24205
+ const result = {
24206
+ ...capture.result,
24207
+ diagnostics: {
24208
+ ...capture.result.diagnostics,
24209
+ ...artifactMissing ? {
24210
+ resultQuality: "partial",
24211
+ degradedResult: true,
24212
+ retryRecommended: false
24213
+ } : {}
24214
+ },
24215
+ capture: {
24216
+ ...capture.result.capture,
24217
+ rawDomArtifact: artifact,
24218
+ rawDomArtifactStatus: artifactResult.status
24219
+ }
24220
+ };
24221
+ await completeJob(job.id, result);
24222
+ await finishHarvestAttempt({
24223
+ jobId: job.id,
24224
+ attemptNumber: 1,
24225
+ outcome: capture.partial ? "partial" : "paa_found",
24226
+ kernelSessionId: null,
24227
+ questionCount: questionCount(result),
24228
+ durationMs: Date.now() - invocationStartedAt,
24229
+ error: capture.checkpoint.error,
24230
+ willRetry: false,
24231
+ kernelDeleteStarted: true,
24232
+ kernelDeleteSucceeded: true,
24233
+ kernelDeleteError: null,
24234
+ browserCloseSucceeded: true,
24235
+ browserCloseError: null,
24236
+ debug: {
24237
+ execution: "kernel_colocated_playwright",
24238
+ stealth: true,
24239
+ attemptUsed: capture.attempt,
24240
+ rawDomStored: Boolean(artifact)
24241
+ },
24242
+ completedAt: (/* @__PURE__ */ new Date()).toISOString()
24243
+ });
24244
+ await settlePaaHarvestJob(job.id);
24245
+ if (job.callback_url) {
24246
+ await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
24247
+ }
24248
+ return {
24249
+ jobId: job.id,
24250
+ status: "done",
24251
+ totalQuestions: questionCount(result),
24252
+ partial: capture.partial,
24253
+ artifactStored: Boolean(artifact)
24254
+ };
24255
+ } catch (error) {
24256
+ const message = error instanceof Error ? error.message : String(error);
24257
+ const latestCheckpoint = checkpointState.value;
24258
+ await finishHarvestAttempt({
24259
+ jobId: job.id,
24260
+ attemptNumber: 1,
24261
+ outcome: latestCheckpoint?.records.length ? "partial_session_lost" : "failed_before_capture",
24262
+ kernelSessionId: null,
24263
+ questionCount: latestCheckpoint?.records.length ?? 0,
24264
+ durationMs: Date.now() - invocationStartedAt,
24265
+ error: message.slice(0, 1e3),
24266
+ willRetry: false,
24267
+ kernelDeleteStarted: true,
24268
+ kernelDeleteSucceeded: null,
24269
+ kernelDeleteError: null,
24270
+ browserCloseSucceeded: null,
24271
+ browserCloseError: null,
24272
+ debug: latestCheckpoint,
24273
+ completedAt: (/* @__PURE__ */ new Date()).toISOString()
24274
+ });
24275
+ await failJob(job.id, message.slice(0, 1e3));
24276
+ await settlePaaHarvestJob(job.id).catch((settlementError) => {
24277
+ console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
24278
+ });
24279
+ if (job.callback_url) {
24280
+ await deliverWebhook(job.callback_url, {
24281
+ job_id: job.id,
24282
+ status: "failed",
24283
+ error: message,
24284
+ progress: latestCheckpoint
24285
+ });
24286
+ }
24287
+ throw error;
24288
+ }
24289
+ }
24290
+ );
24291
+ }
24292
+ });
24293
+
23185
24294
  // src/api/site-extract-start.ts
23186
24295
  async function prepareSiteExtractStart(input) {
23187
24296
  const existing = await getExtractJobByIdempotencyKey(input.userId, input.idempotencyKey);
@@ -23953,15 +25062,15 @@ function normalizeIdempotencyKey(value) {
23953
25062
  return key;
23954
25063
  }
23955
25064
  function tokenFor(id, idempotencyKey2) {
23956
- return (0, import_node_crypto11.createHmac)("sha256", billingSecret()).update(id).update(":").update(idempotencyKey2).digest("base64url");
25065
+ return (0, import_node_crypto12.createHmac)("sha256", billingSecret()).update(id).update(":").update(idempotencyKey2).digest("base64url");
23957
25066
  }
23958
25067
  function tokenHash(token2) {
23959
- return (0, import_node_crypto11.createHash)("sha256").update(token2).digest("hex");
25068
+ return (0, import_node_crypto12.createHash)("sha256").update(token2).digest("hex");
23960
25069
  }
23961
25070
  function verifyToken(row, token2) {
23962
25071
  const expected = Buffer.from(row.token_hash, "hex");
23963
25072
  const actual = Buffer.from(tokenHash(token2), "hex");
23964
- if (expected.length !== actual.length || !(0, import_node_crypto11.timingSafeEqual)(expected, actual)) {
25073
+ if (expected.length !== actual.length || !(0, import_node_crypto12.timingSafeEqual)(expected, actual)) {
23965
25074
  throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
23966
25075
  }
23967
25076
  }
@@ -24076,7 +25185,7 @@ async function authorizeScheduledRun(args) {
24076
25185
  { balanceMc, requiredMc: SCHEDULED_RUN_BASE_MC }
24077
25186
  );
24078
25187
  }
24079
- const id = (0, import_node_crypto11.randomUUID)();
25188
+ const id = (0, import_node_crypto12.randomUUID)();
24080
25189
  const token2 = tokenFor(id, idempotencyKey2);
24081
25190
  const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
24082
25191
  const inserted = await getDb().execute({
@@ -24144,7 +25253,7 @@ async function startScheduledRun(args) {
24144
25253
  if (!authorization || !["starting", "started"].includes(authorization.status)) {
24145
25254
  throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer startable", 409);
24146
25255
  }
24147
- const eventId = existing?.id ?? (0, import_node_crypto11.randomUUID)();
25256
+ const eventId = existing?.id ?? (0, import_node_crypto12.randomUUID)();
24148
25257
  if (!existing) {
24149
25258
  await getDb().execute({
24150
25259
  sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, multiplier_bps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'capturing', ?, ?, ?)",
@@ -24281,7 +25390,7 @@ async function settleScheduledRun(args) {
24281
25390
  chargeMode: "model_delta",
24282
25391
  ...pendingReason ? { reason: pendingReason } : {}
24283
25392
  });
24284
- const eventId = existing?.id ?? (0, import_node_crypto11.randomUUID)();
25393
+ const eventId = existing?.id ?? (0, import_node_crypto12.randomUUID)();
24285
25394
  const status = pendingReason ? "cost_pending" : "settling";
24286
25395
  const eventValues = [
24287
25396
  modelMc,
@@ -24340,11 +25449,11 @@ async function voidScheduledRunAuthorization(args) {
24340
25449
  }
24341
25450
  return { ok: true, status: authorization.status };
24342
25451
  }
24343
- var import_node_crypto11, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_SOURCE_SURFACE, AUTHORIZATION_TTL_MS, UnifiedBillingError;
25452
+ var import_node_crypto12, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_SOURCE_SURFACE, AUTHORIZATION_TTL_MS, UnifiedBillingError;
24344
25453
  var init_unified_billing = __esm({
24345
25454
  "src/api/unified-billing.ts"() {
24346
25455
  "use strict";
24347
- import_node_crypto11 = require("crypto");
25456
+ import_node_crypto12 = require("crypto");
24348
25457
  init_db();
24349
25458
  init_rates();
24350
25459
  init_scheduling_access();
@@ -24376,14 +25485,14 @@ function getSessionSecret() {
24376
25485
  function safeEqualHex(a, b) {
24377
25486
  if (a.length !== b.length) return false;
24378
25487
  try {
24379
- return (0, import_node_crypto12.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
25488
+ return (0, import_node_crypto13.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
24380
25489
  } catch {
24381
25490
  return false;
24382
25491
  }
24383
25492
  }
24384
25493
  function signSession(userId) {
24385
25494
  const payload = String(userId);
24386
- const sig = (0, import_node_crypto12.createHmac)("sha256", secret()).update(payload).digest("hex");
25495
+ const sig = (0, import_node_crypto13.createHmac)("sha256", secret()).update(payload).digest("hex");
24387
25496
  return `${payload}.${sig}`;
24388
25497
  }
24389
25498
  function verifySession(token2) {
@@ -24391,16 +25500,16 @@ function verifySession(token2) {
24391
25500
  if (dot === -1) return null;
24392
25501
  const payload = token2.slice(0, dot);
24393
25502
  const sig = token2.slice(dot + 1);
24394
- const expected = (0, import_node_crypto12.createHmac)("sha256", secret()).update(payload).digest("hex");
25503
+ const expected = (0, import_node_crypto13.createHmac)("sha256", secret()).update(payload).digest("hex");
24395
25504
  if (!safeEqualHex(sig, expected)) return null;
24396
25505
  const id = parseInt(payload);
24397
25506
  return isNaN(id) ? null : id;
24398
25507
  }
24399
- var import_node_crypto12, isProduction, secret;
25508
+ var import_node_crypto13, isProduction, secret;
24400
25509
  var init_session = __esm({
24401
25510
  "src/api/session.ts"() {
24402
25511
  "use strict";
24403
- import_node_crypto12 = require("crypto");
25512
+ import_node_crypto13 = require("crypto");
24404
25513
  isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
24405
25514
  secret = () => getSessionSecret();
24406
25515
  }
@@ -24418,11 +25527,11 @@ function isMemoryOperator(email) {
24418
25527
  return ops.includes(email.trim().toLowerCase());
24419
25528
  }
24420
25529
  function encKey() {
24421
- return (0, import_node_crypto13.scryptSync)(getSessionSecret(), "mcp-memory-key-v1", 32);
25530
+ return (0, import_node_crypto14.scryptSync)(getSessionSecret(), "mcp-memory-key-v1", 32);
24422
25531
  }
24423
25532
  function encryptMemoryKey(secret2) {
24424
- const iv = (0, import_node_crypto13.randomBytes)(12);
24425
- const cipher = (0, import_node_crypto13.createCipheriv)("aes-256-gcm", encKey(), iv);
25533
+ const iv = (0, import_node_crypto14.randomBytes)(12);
25534
+ const cipher = (0, import_node_crypto14.createCipheriv)("aes-256-gcm", encKey(), iv);
24426
25535
  const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
24427
25536
  const tag = cipher.getAuthTag();
24428
25537
  return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
@@ -24430,7 +25539,7 @@ function encryptMemoryKey(secret2) {
24430
25539
  function decryptMemoryKey(stored) {
24431
25540
  try {
24432
25541
  const [ivB, tagB, dataB] = stored.split(":");
24433
- const decipher = (0, import_node_crypto13.createDecipheriv)("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
25542
+ const decipher = (0, import_node_crypto14.createDecipheriv)("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
24434
25543
  decipher.setAuthTag(Buffer.from(tagB, "base64"));
24435
25544
  return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
24436
25545
  } catch {
@@ -24544,11 +25653,11 @@ async function syncScheduledActionCredentials(user) {
24544
25653
  }, {});
24545
25654
  return { ok: res.ok, error: res.error };
24546
25655
  }
24547
- var import_node_crypto13, import_provision_defaults, import_set_schedule_entitlement, MEMORY_BASE_URL, ADMIN_KEY;
25656
+ var import_node_crypto14, import_provision_defaults, import_set_schedule_entitlement, MEMORY_BASE_URL, ADMIN_KEY;
24548
25657
  var init_memory = __esm({
24549
25658
  "src/api/memory.ts"() {
24550
25659
  "use strict";
24551
- import_node_crypto13 = require("crypto");
25660
+ import_node_crypto14 = require("crypto");
24552
25661
  init_session();
24553
25662
  init_db();
24554
25663
  init_rates();
@@ -24598,7 +25707,7 @@ var init_connected_cost_telemetry = __esm({
24598
25707
 
24599
25708
  // src/api/connected-usage-billing.ts
24600
25709
  function hash(value) {
24601
- return (0, import_node_crypto14.createHash)("sha256").update(value).digest("hex");
25710
+ return (0, import_node_crypto15.createHash)("sha256").update(value).digest("hex");
24602
25711
  }
24603
25712
  function eventKey(idempotencyKey2) {
24604
25713
  return `connected-usage:${hash(idempotencyKey2)}`;
@@ -24870,7 +25979,7 @@ async function settleConnectedUsage(rawInput) {
24870
25979
  sql: `INSERT OR IGNORE INTO billing_events
24871
25980
  (id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
24872
25981
  VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
24873
- args: [(0, import_node_crypto14.randomUUID)(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
25982
+ args: [(0, import_node_crypto15.randomUUID)(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
24874
25983
  });
24875
25984
  let event = await readEvent(key);
24876
25985
  if (!event) throw new Error("connected usage receipt insert completed without a readable event");
@@ -24959,11 +26068,11 @@ async function listConnectedUsageHistory(userId, limit = 100) {
24959
26068
  }
24960
26069
  return history;
24961
26070
  }
24962
- var import_node_crypto14, import_zod14, CONNECTED_USAGE_BILLING_CLASS, CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE, ConnectedUsageSafeMetadataSchema, ConnectedUsageSettlementInputSchema, ConnectedUsagePreflightInputSchema, ConnectedUsageBillingError;
26071
+ var import_node_crypto15, import_zod14, CONNECTED_USAGE_BILLING_CLASS, CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE, ConnectedUsageSafeMetadataSchema, ConnectedUsageSettlementInputSchema, ConnectedUsagePreflightInputSchema, ConnectedUsageBillingError;
24963
26072
  var init_connected_usage_billing = __esm({
24964
26073
  "src/api/connected-usage-billing.ts"() {
24965
26074
  "use strict";
24966
- import_node_crypto14 = require("crypto");
26075
+ import_node_crypto15 = require("crypto");
24967
26076
  import_zod14 = require("zod");
24968
26077
  init_connected_cost_telemetry();
24969
26078
  init_db();
@@ -27854,7 +28963,7 @@ async function kernelLaunchOptsResidential() {
27854
28963
  }
27855
28964
  return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
27856
28965
  }
27857
- var import_hono7, import_zod21, import_client5, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, AD_LIBRARY_APP_CHROME, AUTH_WALL_URL, MIN_RENDERED_BODY_CHARS, facebookAdApp, ALLOWED_MEDIA_HOSTS;
28966
+ var import_hono7, import_zod21, import_client6, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, AD_LIBRARY_APP_CHROME, AUTH_WALL_URL, MIN_RENDERED_BODY_CHARS, facebookAdApp, ALLOWED_MEDIA_HOSTS;
27858
28967
  var init_facebook_ad_routes = __esm({
27859
28968
  "src/api/facebook-ad-routes.ts"() {
27860
28969
  "use strict";
@@ -27869,7 +28978,7 @@ var init_facebook_ad_routes = __esm({
27869
28978
  init_FacebookOrganicVideoExtractor();
27870
28979
  init_kernel_proxy_resolver();
27871
28980
  init_schemas2();
27872
- import_client5 = require("@fal-ai/client");
28981
+ import_client6 = require("@fal-ai/client");
27873
28982
  init_api_auth();
27874
28983
  init_url_utils();
27875
28984
  init_concurrency_gates();
@@ -28016,7 +29125,7 @@ var init_facebook_ad_routes = __esm({
28016
29125
  metadata: { videoUrl }
28017
29126
  });
28018
29127
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
28019
- import_client5.fal.config({ credentials: process.env.FAL_KEY });
29128
+ import_client6.fal.config({ credentials: process.env.FAL_KEY });
28020
29129
  const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
28021
29130
  let debited = false;
28022
29131
  try {
@@ -28060,7 +29169,7 @@ var init_facebook_ad_routes = __esm({
28060
29169
  metadata: { url: sourceUrl.href }
28061
29170
  });
28062
29171
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
28063
- import_client5.fal.config({ credentials: process.env.FAL_KEY });
29172
+ import_client6.fal.config({ credentials: process.env.FAL_KEY });
28064
29173
  const driver = new BrowserDriver();
28065
29174
  const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
28066
29175
  let debited = false;
@@ -28512,7 +29621,7 @@ async function kernelLaunchOptsResidential2() {
28512
29621
  function isBlockedMessage(msg) {
28513
29622
  return msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha");
28514
29623
  }
28515
- var import_hono8, import_zod22, import_client6, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
29624
+ var import_hono8, import_zod22, import_client7, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
28516
29625
  var init_google_ads_routes = __esm({
28517
29626
  "src/api/google-ads-routes.ts"() {
28518
29627
  "use strict";
@@ -28525,7 +29634,7 @@ var init_google_ads_routes = __esm({
28525
29634
  init_GoogleAdsExtractor();
28526
29635
  init_kernel_proxy_resolver();
28527
29636
  init_schemas2();
28528
- import_client6 = require("@fal-ai/client");
29637
+ import_client7 = require("@fal-ai/client");
28529
29638
  init_api_auth();
28530
29639
  init_url_utils();
28531
29640
  init_concurrency_gates();
@@ -28637,7 +29746,7 @@ var init_google_ads_routes = __esm({
28637
29746
  const user = c.get("user");
28638
29747
  const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
28639
29748
  if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
28640
- import_client6.fal.config({ credentials: process.env.FAL_KEY });
29749
+ import_client7.fal.config({ credentials: process.env.FAL_KEY });
28641
29750
  const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
28642
29751
  let debited = false;
28643
29752
  try {
@@ -28813,7 +29922,7 @@ async function applyAdjustment(input) {
28813
29922
  const credits = validateCredits(input.credits, input.confirmLarge === true);
28814
29923
  const reason = validateReason(input.reason);
28815
29924
  const actor = validateActor(input.actor);
28816
- const reference = input.reference?.trim() || `adj_${(0, import_node_crypto15.randomUUID)()}`;
29925
+ const reference = input.reference?.trim() || `adj_${(0, import_node_crypto16.randomUUID)()}`;
28817
29926
  if (reference.length > 120) throw new AdminCreditError(400, "reference must be at most 120 characters");
28818
29927
  const db = getDb();
28819
29928
  const existing = await db.execute({
@@ -28958,11 +30067,11 @@ async function lookupAccount(target, limit = 20) {
28958
30067
  }))
28959
30068
  };
28960
30069
  }
28961
- var import_node_crypto15, MIN_ADJUSTMENT_CREDITS, LARGE_ADJUSTMENT_CREDITS, MAX_ADJUSTMENT_CREDITS, MIN_REASON_LENGTH, MAX_REASON_LENGTH, MAX_ACTOR_LENGTH, AdminCreditError;
30070
+ var import_node_crypto16, MIN_ADJUSTMENT_CREDITS, LARGE_ADJUSTMENT_CREDITS, MAX_ADJUSTMENT_CREDITS, MIN_REASON_LENGTH, MAX_REASON_LENGTH, MAX_ACTOR_LENGTH, AdminCreditError;
28962
30071
  var init_admin_credits = __esm({
28963
30072
  "src/api/admin-credits.ts"() {
28964
30073
  "use strict";
28965
- import_node_crypto15 = require("crypto");
30074
+ import_node_crypto16 = require("crypto");
28966
30075
  init_db();
28967
30076
  init_rates();
28968
30077
  MIN_ADJUSTMENT_CREDITS = 1;
@@ -30487,14 +31596,28 @@ var init_PAAExtractor = __esm({
30487
31596
  }
30488
31597
  return parts.join(" ").replace(/\s+/g, " ").trim();
30489
31598
  }
31599
+ function cleanSourceLabel(value) {
31600
+ return (value ?? "").replace(/\s*\(\+\d+\)\s*-\s*View related links.*$/i, "").replace(/\s*-\s*Opens in new tab.*$/i, "").trim();
31601
+ }
30490
31602
  return Array.from(document.querySelectorAll(selectors.item)).map((pair) => {
30491
- let sourceTitle = pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0;
30492
- let sourceSite = pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0;
30493
- let sourceCite = pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0;
30494
- if (!sourceCite) {
31603
+ const clickTarget = pair.querySelector(selectors.clickTarget);
31604
+ const expanded = pair.classList.contains(selectors.expandedClass) || clickTarget?.getAttribute("aria-expanded") === "true";
31605
+ const answerContainer = expanded ? pair.querySelector(selectors.answerContainer) : null;
31606
+ const aiAnswerBody = answerContainer?.querySelector(selectors.aiAnswerBody);
31607
+ const answerElement = aiAnswerBody ?? answerContainer;
31608
+ let sourceTitle = expanded ? pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0 : void 0;
31609
+ let sourceSite = expanded ? pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0 : void 0;
31610
+ let sourceCite = expanded ? pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0 : void 0;
31611
+ if (expanded && !sourceCite) {
30495
31612
  const seenHrefs = /* @__PURE__ */ new Set();
30496
31613
  const anchors = [];
30497
- for (const a of Array.from(pair.querySelectorAll("a[href]"))) {
31614
+ const anchorRoot = answerElement ?? pair;
31615
+ const prioritized = [
31616
+ ...Array.from(anchorRoot.querySelectorAll(selectors.aiCitation)),
31617
+ ...Array.from(anchorRoot.querySelectorAll(selectors.aiSourceCard)),
31618
+ ...Array.from(anchorRoot.querySelectorAll("a[href]"))
31619
+ ];
31620
+ for (const a of prioritized) {
30498
31621
  const href = a.href;
30499
31622
  if (!/^https?:\/\//.test(href) || seenHrefs.has(href)) continue;
30500
31623
  let host = "";
@@ -30505,18 +31628,23 @@ var init_PAAExtractor = __esm({
30505
31628
  }
30506
31629
  if (/(^|\.)google(usercontent)?\.[a-z.]+$/.test(host)) continue;
30507
31630
  seenHrefs.add(href);
30508
- anchors.push({ text: (a.textContent || "").trim(), href, host });
31631
+ anchors.push({
31632
+ text: (a.textContent || "").trim(),
31633
+ label: cleanSourceLabel(a.getAttribute("aria-label")),
31634
+ href,
31635
+ host
31636
+ });
30509
31637
  }
30510
- const primary = anchors.find((x) => x.text.length > 2) ?? anchors[0];
31638
+ const primary = anchors.find((x) => x.label.length > 2 || x.text.length > 2) ?? anchors[0];
30511
31639
  if (primary) {
30512
- sourceTitle = sourceTitle || primary.text || void 0;
31640
+ sourceTitle = sourceTitle || primary.label || primary.text || void 0;
30513
31641
  sourceSite = sourceSite || primary.host.replace(/^www\./, "");
30514
31642
  sourceCite = primary.href;
30515
31643
  }
30516
31644
  }
30517
31645
  return {
30518
31646
  question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
30519
- answer: cleanText(pair.querySelector(selectors.answerContainer)) || void 0,
31647
+ answer: cleanText(answerElement) || void 0,
30520
31648
  sourceTitle,
30521
31649
  sourceSite,
30522
31650
  sourceCite
@@ -33129,12 +34257,15 @@ function mapsErrorResponse(c, err, errorCode) {
33129
34257
  const msg = err instanceof Error ? err.message : String(err);
33130
34258
  const rotationError = err instanceof MapsSearchRotationError ? err : null;
33131
34259
  const blocked = msg.includes("CAPTCHA") || msg.includes("blocked");
33132
- const retryable = rotationError?.retryable ?? blocked;
33133
- const message = sanitizeVendorName(msg);
34260
+ const publicCode = normalizePublicErrorCode(
34261
+ rotationError?.errorCode ?? (blocked ? "captcha_or_blocked" : "service_unavailable")
34262
+ );
34263
+ const retryable = publicCode === "service_unavailable" || rotationError?.retryable === true || blocked;
34264
+ const message = publicErrorMessage(publicCode);
33134
34265
  return c.json({
33135
34266
  error: message,
33136
34267
  message,
33137
- error_code: rotationError?.errorCode ?? (blocked ? "captcha_or_blocked" : errorCode),
34268
+ error_code: publicCode,
33138
34269
  retryable
33139
34270
  }, retryable ? 503 : 500);
33140
34271
  }
@@ -33693,7 +34824,7 @@ function retryDelaySeconds(attempts) {
33693
34824
  }
33694
34825
  async function dispatchPendingDirectoryWorkflows(limit = 25) {
33695
34826
  const rows = await claimDirectoryWorkflowOutbox({
33696
- workerId: `directory-dispatch-${process.pid}-${(0, import_node_crypto16.randomUUID)().slice(0, 8)}`,
34827
+ workerId: `directory-dispatch-${process.pid}-${(0, import_node_crypto17.randomUUID)().slice(0, 8)}`,
33697
34828
  limit
33698
34829
  });
33699
34830
  const result = { claimed: rows.length, dispatched: 0, failed: 0 };
@@ -33715,11 +34846,11 @@ async function dispatchPendingDirectoryWorkflows(limit = 25) {
33715
34846
  }
33716
34847
  return result;
33717
34848
  }
33718
- var import_node_crypto16;
34849
+ var import_node_crypto17;
33719
34850
  var init_directory_workflow_dispatch = __esm({
33720
34851
  "src/api/directory-workflow-dispatch.ts"() {
33721
34852
  "use strict";
33722
- import_node_crypto16 = require("crypto");
34853
+ import_node_crypto17 = require("crypto");
33723
34854
  init_client();
33724
34855
  init_directory_workflow_repository();
33725
34856
  }
@@ -33731,7 +34862,7 @@ function safeOptions(options) {
33731
34862
  return safe2;
33732
34863
  }
33733
34864
  function requestFingerprint(options) {
33734
- return (0, import_node_crypto17.createHash)("sha256").update(JSON.stringify(safeOptions(options))).digest("hex");
34865
+ return (0, import_node_crypto18.createHash)("sha256").update(JSON.stringify(safeOptions(options))).digest("hex");
33735
34866
  }
33736
34867
  function idempotencyKey(raw) {
33737
34868
  if (raw !== void 0) {
@@ -33740,14 +34871,14 @@ function idempotencyKey(raw) {
33740
34871
  if (trimmed.length > 500) return { ok: false, message: "Idempotency-Key must be 500 characters or fewer." };
33741
34872
  return { ok: true, key: trimmed };
33742
34873
  }
33743
- return { ok: true, key: `directory-${(0, import_node_crypto17.randomUUID)()}` };
34874
+ return { ok: true, key: `directory-${(0, import_node_crypto18.randomUUID)()}` };
33744
34875
  }
33745
34876
  function debitKeyFor(userId, responseKey) {
33746
- const digest = (0, import_node_crypto17.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
34877
+ const digest = (0, import_node_crypto18.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
33747
34878
  return `directory-workflow:${userId}:${digest}`;
33748
34879
  }
33749
34880
  function jobId() {
33750
- return `dir_${(0, import_node_crypto17.randomUUID)().replace(/-/g, "")}`;
34881
+ return `dir_${(0, import_node_crypto18.randomUUID)().replace(/-/g, "")}`;
33751
34882
  }
33752
34883
  function publicStatus(job, result) {
33753
34884
  if (job.status === "failed") return "failed";
@@ -33856,7 +34987,7 @@ async function runSynchronously(c, user, options, plan, responseKey) {
33856
34987
  const csv = renderDirectoryWorkflowCsv(result);
33857
34988
  const artifact = await createDirectoryCsvArtifact({
33858
34989
  ownerId: String(user.id),
33859
- jobId: (0, import_node_crypto17.createHash)("sha256").update(debitKey2).digest("hex").slice(0, 32),
34990
+ jobId: (0, import_node_crypto18.createHash)("sha256").update(debitKey2).digest("hex").slice(0, 32),
33860
34991
  createdAt: result.extractedAt,
33861
34992
  filename: `${options.state}-${options.query}-directory.csv`,
33862
34993
  csv,
@@ -33912,11 +35043,11 @@ async function runSynchronously(c, user, options, plan, responseKey) {
33912
35043
  await releaseConcurrencyGate(gate.lockId);
33913
35044
  }
33914
35045
  }
33915
- var import_node_crypto17, import_hono16, directoryApp;
35046
+ var import_node_crypto18, import_hono16, directoryApp;
33916
35047
  var init_directory_routes = __esm({
33917
35048
  "src/api/directory-routes.ts"() {
33918
35049
  "use strict";
33919
- import_node_crypto17 = require("crypto");
35050
+ import_node_crypto18 = require("crypto");
33920
35051
  import_hono16 = require("hono");
33921
35052
  init_api_auth();
33922
35053
  init_db();
@@ -34303,6 +35434,67 @@ var init_location_data_routes = __esm({
34303
35434
  }
34304
35435
  });
34305
35436
 
35437
+ // src/api/paa-harvest-dispatch.ts
35438
+ async function dispatchPaaHarvest(jobId2) {
35439
+ await inngest.send({
35440
+ id: `paa-harvest:${jobId2}`,
35441
+ name: "mcp-scraper/paa.requested",
35442
+ data: { jobId: jobId2 }
35443
+ });
35444
+ }
35445
+ async function redispatchPendingPaaHarvests(limit = 25) {
35446
+ const jobs = await listPendingInngestPaaJobs(limit);
35447
+ let dispatched = 0;
35448
+ let failed = 0;
35449
+ for (const job of jobs) {
35450
+ try {
35451
+ await dispatchPaaHarvest(job.id);
35452
+ dispatched += 1;
35453
+ } catch {
35454
+ failed += 1;
35455
+ }
35456
+ }
35457
+ return { checked: jobs.length, dispatched, failed };
35458
+ }
35459
+ var init_paa_harvest_dispatch = __esm({
35460
+ "src/api/paa-harvest-dispatch.ts"() {
35461
+ "use strict";
35462
+ init_client();
35463
+ init_db();
35464
+ }
35465
+ });
35466
+
35467
+ // src/api/paa-harvest-reconciliation.ts
35468
+ async function reconcilePaaHarvestSettlements(limit = 25) {
35469
+ const stale = await listStaleRunningInngestPaaJobs(limit);
35470
+ let staleFailed = 0;
35471
+ for (const job of stale) {
35472
+ try {
35473
+ await failJob(job.id, "durable PAA invocation exceeded its recovery window; preserved progress remains in result");
35474
+ staleFailed += 1;
35475
+ } catch {
35476
+ }
35477
+ }
35478
+ const jobs = await listRecentTerminalInngestPaaJobs(limit);
35479
+ let settled = 0;
35480
+ let failed = 0;
35481
+ for (const job of jobs) {
35482
+ try {
35483
+ if (await settlePaaHarvestJob(job)) settled += 1;
35484
+ } catch {
35485
+ failed += 1;
35486
+ }
35487
+ }
35488
+ return { checked: jobs.length, staleFailed, settled, failed };
35489
+ }
35490
+ var init_paa_harvest_reconciliation = __esm({
35491
+ "src/api/paa-harvest-reconciliation.ts"() {
35492
+ "use strict";
35493
+ init_db();
35494
+ init_paa_harvest_settlement();
35495
+ }
35496
+ });
35497
+
34306
35498
  // src/api/directory-workflow-reconciliation.ts
34307
35499
  async function reconcileDirectoryWorkflowSettlements(limit = 25) {
34308
35500
  const expiredStarts = await listExpiredDirectoryWorkflowStarts(limit);
@@ -36337,7 +37529,7 @@ async function readManifestFromSummary(summary) {
36337
37529
  function webhookSignature(body, timestamp2) {
36338
37530
  const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
36339
37531
  if (!secret2) return null;
36340
- return (0, import_node_crypto18.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
37532
+ return (0, import_node_crypto19.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
36341
37533
  }
36342
37534
  async function deliverWorkflowWebhook(input) {
36343
37535
  if (!input.webhookUrl) return;
@@ -36544,11 +37736,11 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
36544
37736
  }
36545
37737
  return { dispatched: results.length, results };
36546
37738
  }
36547
- var import_node_crypto18, import_promises12, import_hono18, import_zod36, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
37739
+ var import_node_crypto19, import_promises12, import_hono18, import_zod36, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
36548
37740
  var init_workflow_routes = __esm({
36549
37741
  "src/api/workflow-routes.ts"() {
36550
37742
  "use strict";
36551
- import_node_crypto18 = require("crypto");
37743
+ import_node_crypto19 = require("crypto");
36552
37744
  import_promises12 = require("fs/promises");
36553
37745
  import_hono18 = require("hono");
36554
37746
  import_zod36 = require("zod");
@@ -36829,7 +38021,7 @@ var init_workflow_routes = __esm({
36829
38021
  // src/serp-intelligence/page-snapshot-extractor.ts
36830
38022
  function sha256(value) {
36831
38023
  if (!value) return null;
36832
- return (0, import_node_crypto19.createHash)("sha256").update(value).digest("hex");
38024
+ return (0, import_node_crypto20.createHash)("sha256").update(value).digest("hex");
36833
38025
  }
36834
38026
  function countWords(markdown) {
36835
38027
  const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
@@ -37129,11 +38321,11 @@ async function capturePageSnapshots(targets, options = {}) {
37129
38321
  }
37130
38322
  };
37131
38323
  }
37132
- var import_node_crypto19, import_p_limit5, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
38324
+ var import_node_crypto20, import_p_limit5, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
37133
38325
  var init_page_snapshot_extractor = __esm({
37134
38326
  "src/serp-intelligence/page-snapshot-extractor.ts"() {
37135
38327
  "use strict";
37136
- import_node_crypto19 = require("crypto");
38328
+ import_node_crypto20 = require("crypto");
37137
38329
  import_p_limit5 = __toESM(require("p-limit"), 1);
37138
38330
  init_kpo_extractor();
37139
38331
  init_url_utils();
@@ -37160,13 +38352,16 @@ function looksLikeProxyTunnelFailure3(message) {
37160
38352
  function looksLikeProxyUnavailable3(message) {
37161
38353
  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
37162
38354
  }
38355
+ function looksLikeInternalServiceUnavailable(message) {
38356
+ return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
38357
+ }
37163
38358
  function classifyHarvestProblem(err) {
37164
38359
  const message = errorMessage3(err);
37165
38360
  if (err instanceof RequestAbortedError) {
37166
38361
  return {
37167
38362
  error_code: "request_aborted",
37168
38363
  error_type: "request_aborted",
37169
- message,
38364
+ message: publicErrorMessage("request_aborted"),
37170
38365
  retryable: true,
37171
38366
  httpStatus: 408,
37172
38367
  terminalStatus: "cancelled"
@@ -37176,7 +38371,7 @@ function classifyHarvestProblem(err) {
37176
38371
  return {
37177
38372
  error_code: "captcha_exhausted",
37178
38373
  error_type: "captcha",
37179
- message,
38374
+ message: publicErrorMessage("captcha_exhausted"),
37180
38375
  retryable: true,
37181
38376
  httpStatus: 503,
37182
38377
  terminalStatus: "failed"
@@ -37186,7 +38381,7 @@ function classifyHarvestProblem(err) {
37186
38381
  return {
37187
38382
  error_code: "location_mismatch",
37188
38383
  error_type: "location_mismatch",
37189
- message,
38384
+ message: publicErrorMessage("location_mismatch"),
37190
38385
  retryable: true,
37191
38386
  httpStatus: 503,
37192
38387
  terminalStatus: "failed"
@@ -37196,7 +38391,7 @@ function classifyHarvestProblem(err) {
37196
38391
  return {
37197
38392
  error_code: "proxy_tunnel_failed",
37198
38393
  error_type: "proxy_tunnel_failed",
37199
- message,
38394
+ message: publicErrorMessage("proxy_tunnel_failed"),
37200
38395
  retryable: true,
37201
38396
  httpStatus: 503,
37202
38397
  terminalStatus: "failed"
@@ -37206,7 +38401,7 @@ function classifyHarvestProblem(err) {
37206
38401
  return {
37207
38402
  error_code: "proxy_unavailable",
37208
38403
  error_type: "proxy_unavailable",
37209
- message,
38404
+ message: publicErrorMessage("proxy_unavailable"),
37210
38405
  retryable: true,
37211
38406
  httpStatus: 503,
37212
38407
  terminalStatus: "failed"
@@ -37216,16 +38411,26 @@ function classifyHarvestProblem(err) {
37216
38411
  return {
37217
38412
  error_code: "harvest_timeout",
37218
38413
  error_type: "timeout",
37219
- message,
38414
+ message: publicErrorMessage("harvest_timeout"),
37220
38415
  retryable: true,
37221
38416
  httpStatus: 504,
37222
38417
  terminalStatus: "failed"
37223
38418
  };
37224
38419
  }
38420
+ if (looksLikeInternalServiceUnavailable(message)) {
38421
+ return {
38422
+ error_code: "service_unavailable",
38423
+ error_type: "service_unavailable",
38424
+ message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
38425
+ retryable: true,
38426
+ httpStatus: 503,
38427
+ terminalStatus: "failed"
38428
+ };
38429
+ }
37225
38430
  return {
37226
38431
  error_code: "extraction_failed",
37227
38432
  error_type: "extraction",
37228
- message,
38433
+ message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
37229
38434
  retryable: false,
37230
38435
  httpStatus: 500,
37231
38436
  terminalStatus: "failed"
@@ -37740,21 +38945,21 @@ async function logRequestEventBestEffort(input) {
37740
38945
  }
37741
38946
  }
37742
38947
  function captureBillingKeys(userId, suppliedKey, body) {
37743
- const responseKey = suppliedKey?.trim() || (0, import_node_crypto20.randomUUID)();
37744
- const requestFingerprint2 = (0, import_node_crypto20.createHash)("sha256").update(JSON.stringify(body)).digest("hex");
37745
- const keyDigest = (0, import_node_crypto20.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
38948
+ const responseKey = suppliedKey?.trim() || (0, import_node_crypto21.randomUUID)();
38949
+ const requestFingerprint2 = (0, import_node_crypto21.createHash)("sha256").update(JSON.stringify(body)).digest("hex");
38950
+ const keyDigest = (0, import_node_crypto21.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
37746
38951
  return {
37747
38952
  responseKey,
37748
38953
  debitKey: `serp-capture:${userId}:${keyDigest}`,
37749
38954
  debitDescription: `${body.query} [request:${requestFingerprint2}]`
37750
38955
  };
37751
38956
  }
37752
- var import_hono19, import_node_crypto20, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, SERP_CAPTURE_BILLING_SOURCE, serpIntelligenceApp;
38957
+ var import_hono19, import_node_crypto21, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, SERP_CAPTURE_BILLING_SOURCE, serpIntelligenceApp;
37753
38958
  var init_serp_intelligence_routes = __esm({
37754
38959
  "src/api/serp-intelligence-routes.ts"() {
37755
38960
  "use strict";
37756
38961
  import_hono19 = require("hono");
37757
- import_node_crypto20 = require("crypto");
38962
+ import_node_crypto21 = require("crypto");
37758
38963
  init_browser_service_env();
37759
38964
  init_page_snapshot_extractor();
37760
38965
  init_serp_capture_service();
@@ -37991,7 +39196,7 @@ var PACKAGE_VERSION;
37991
39196
  var init_version = __esm({
37992
39197
  "src/version.ts"() {
37993
39198
  "use strict";
37994
- PACKAGE_VERSION = "0.40.2";
39199
+ PACKAGE_VERSION = "0.40.3";
37995
39200
  }
37996
39201
  });
37997
39202
 
@@ -40799,7 +42004,7 @@ var init_editorial_reading_room = __esm({
40799
42004
 
40800
42005
  // src/mcp/paa-mcp-server.ts
40801
42006
  function hashOwnerId(callerKey) {
40802
- return (0, import_node_crypto21.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
42007
+ return (0, import_node_crypto22.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
40803
42008
  }
40804
42009
  function liveWebToolAnnotations(title) {
40805
42010
  return {
@@ -41347,14 +42552,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
41347
42552
  annotations: { title: "Set Scheduled Action Connections", readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }
41348
42553
  }, async (input) => executor.setScheduledActionConnections(input));
41349
42554
  }
41350
- var import_mcp, import_node_fs11, import_node_path15, import_node_crypto21;
42555
+ var import_mcp, import_node_fs11, import_node_path15, import_node_crypto22;
41351
42556
  var init_paa_mcp_server = __esm({
41352
42557
  "src/mcp/paa-mcp-server.ts"() {
41353
42558
  "use strict";
41354
42559
  import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
41355
42560
  import_node_fs11 = require("fs");
41356
42561
  import_node_path15 = require("path");
41357
- import_node_crypto21 = require("crypto");
42562
+ import_node_crypto22 = require("crypto");
41358
42563
  init_version();
41359
42564
  init_mcp_response_formatter();
41360
42565
  init_server_instructions();
@@ -41403,7 +42608,20 @@ async function readResponseData(res) {
41403
42608
  function httpErrorPayload(path6, res, data) {
41404
42609
  const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
41405
42610
  const rawCode = objectData?.code ?? objectData?.errorCode ?? objectData?.error_code;
41406
- const safeCode = typeof rawCode === "string" && /^[a-z0-9][a-z0-9_-]{0,99}$/.test(rawCode) ? rawCode : "mcp_http_error";
42611
+ const candidateCode = typeof rawCode === "string" && /^[a-z0-9][a-z0-9_-]{0,99}$/.test(rawCode) ? rawCode : "mcp_http_error";
42612
+ if (res.status >= 500) {
42613
+ const safeCode2 = normalizePublicErrorCode(candidateCode);
42614
+ return {
42615
+ error: safeCode2,
42616
+ error_code: safeCode2,
42617
+ error_type: "http",
42618
+ retryable: true,
42619
+ status: res.status,
42620
+ path: path6,
42621
+ message: publicErrorMessage(safeCode2)
42622
+ };
42623
+ }
42624
+ const safeCode = candidateCode;
41407
42625
  const bodyMessage = objectData ? objectData.message ?? objectData.error ?? rawCode : data;
41408
42626
  return {
41409
42627
  ...objectData ?? { body: data },
@@ -41426,13 +42644,15 @@ function withDefaultHostedProfile(input) {
41426
42644
  ...typeof saveProfileChanges === "boolean" ? { saveProfileChanges } : {}
41427
42645
  };
41428
42646
  }
41429
- var import_node_crypto22, HttpMcpToolExecutor;
42647
+ var import_node_crypto23, HttpMcpToolExecutor;
41430
42648
  var init_http_mcp_tool_executor = __esm({
41431
42649
  "src/mcp/http-mcp-tool-executor.ts"() {
41432
42650
  "use strict";
41433
- import_node_crypto22 = require("crypto");
42651
+ import_node_crypto23 = require("crypto");
41434
42652
  init_harvest_timeout();
41435
42653
  init_browser_service_env();
42654
+ init_errors();
42655
+ init_outbound_sanitize();
41436
42656
  HttpMcpToolExecutor = class {
41437
42657
  baseUrl;
41438
42658
  apiKey;
@@ -41469,7 +42689,6 @@ var init_http_mcp_tool_executor = __esm({
41469
42689
  }
41470
42690
  return { content: [{ type: "text", text: JSON.stringify(data) }] };
41471
42691
  } catch (err) {
41472
- const msg = err instanceof Error ? err.message : String(err);
41473
42692
  if (err instanceof DOMException && err.name === "TimeoutError") {
41474
42693
  return {
41475
42694
  content: [{
@@ -41486,17 +42705,17 @@ var init_http_mcp_tool_executor = __esm({
41486
42705
  isError: true
41487
42706
  };
41488
42707
  }
41489
- return { content: [{ type: "text", text: msg }], isError: true };
42708
+ return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
41490
42709
  }
41491
42710
  }
41492
42711
  async callConnectedMutation(path6, body, timeoutMs = this.timeoutMs) {
41493
- const fingerprint = (0, import_node_crypto22.createHash)("sha256").update("POST").update("\0").update(path6).update("\0").update(JSON.stringify(body)).digest("hex");
42712
+ const fingerprint = (0, import_node_crypto23.createHash)("sha256").update("POST").update("\0").update(path6).update("\0").update(JSON.stringify(body)).digest("hex");
41494
42713
  const now = Date.now();
41495
42714
  for (const [pendingFingerprint, pendingEntry] of this.pendingConnectedMutationKeys) {
41496
42715
  if (pendingEntry.expiresAt <= now) this.pendingConnectedMutationKeys.delete(pendingFingerprint);
41497
42716
  }
41498
42717
  const pending = this.pendingConnectedMutationKeys.get(fingerprint);
41499
- const idempotencyKey2 = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto22.randomUUID)();
42718
+ const idempotencyKey2 = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto23.randomUUID)();
41500
42719
  this.pendingConnectedMutationKeys.set(fingerprint, {
41501
42720
  key: idempotencyKey2,
41502
42721
  expiresAt: now + 15 * 6e4
@@ -41510,13 +42729,13 @@ var init_http_mcp_tool_executor = __esm({
41510
42729
  return result;
41511
42730
  }
41512
42731
  async callDirectoryWorkflowStart(body, explicitIdempotencyKey) {
41513
- const idempotencyKey2 = `mcp-directory-${(0, import_node_crypto22.createHash)("sha256").update(explicitIdempotencyKey).digest("hex")}`;
42732
+ const idempotencyKey2 = `mcp-directory-${(0, import_node_crypto23.createHash)("sha256").update(explicitIdempotencyKey).digest("hex")}`;
41514
42733
  return this.call("/directory/run", body, this.timeoutMs, "POST", {
41515
42734
  "Idempotency-Key": idempotencyKey2
41516
42735
  });
41517
42736
  }
41518
42737
  async callSiteExtractStart(toolName, body, explicitIdempotencyKey) {
41519
- const idempotencyKey2 = `mcp-site-${(0, import_node_crypto22.createHash)("sha256").update(toolName).update("\0").update(explicitIdempotencyKey).digest("hex")}`;
42738
+ const idempotencyKey2 = `mcp-site-${(0, import_node_crypto23.createHash)("sha256").update(toolName).update("\0").update(explicitIdempotencyKey).digest("hex")}`;
41520
42739
  return this.call("/extract-site", body, this.timeoutMs, "POST", {
41521
42740
  "Idempotency-Key": idempotencyKey2
41522
42741
  });
@@ -41535,9 +42754,8 @@ var init_http_mcp_tool_executor = __esm({
41535
42754
  return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path6, res, data)) }], isError: true };
41536
42755
  }
41537
42756
  return { content: [{ type: "text", text: JSON.stringify(data) }] };
41538
- } catch (err) {
41539
- const msg = err instanceof Error ? err.message : String(err);
41540
- return { content: [{ type: "text", text: msg }], isError: true };
42757
+ } catch {
42758
+ return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
41541
42759
  }
41542
42760
  }
41543
42761
  async getTextArtifact(path6, maxBytes, timeoutMs = this.timeoutMs) {
@@ -41551,7 +42769,13 @@ var init_http_mcp_tool_executor = __esm({
41551
42769
  });
41552
42770
  if (!res.ok) {
41553
42771
  const data = await res.json().catch(async () => ({ error: await res.text().catch(() => `HTTP ${res.status}`) }));
41554
- return { content: [{ type: "text", text: JSON.stringify(data) }], isError: true };
42772
+ return {
42773
+ content: [{
42774
+ type: "text",
42775
+ text: JSON.stringify(res.status >= 500 ? httpErrorPayload(path6, res, data) : sanitizeOutboundDiagnostics(data))
42776
+ }],
42777
+ isError: true
42778
+ };
41555
42779
  }
41556
42780
  const bytes = Buffer.from(await res.arrayBuffer());
41557
42781
  const sliced = bytes.subarray(0, Math.min(maxBytes, bytes.length));
@@ -41568,8 +42792,7 @@ var init_http_mcp_tool_executor = __esm({
41568
42792
  }]
41569
42793
  };
41570
42794
  } catch (err) {
41571
- const msg = err instanceof Error ? err.message : String(err);
41572
- return { content: [{ type: "text", text: msg }], isError: true };
42795
+ return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
41573
42796
  }
41574
42797
  }
41575
42798
  harvestPaa(input) {
@@ -41789,13 +43012,13 @@ var init_http_mcp_tool_executor = __esm({
41789
43012
  return this.call("/editorial-reading-room/renew-download", input);
41790
43013
  }
41791
43014
  async captureSerpSnapshot(input) {
41792
- const fingerprint = (0, import_node_crypto22.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
43015
+ const fingerprint = (0, import_node_crypto23.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
41793
43016
  const now = Date.now();
41794
43017
  for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
41795
43018
  if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
41796
43019
  }
41797
43020
  const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint);
41798
- const idempotencyKey2 = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto22.randomUUID)();
43021
+ const idempotencyKey2 = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto23.randomUUID)();
41799
43022
  this.pendingSerpCaptureBillingKeys.set(fingerprint, {
41800
43023
  key: idempotencyKey2,
41801
43024
  expiresAt: now + 15 * 6e4
@@ -42632,8 +43855,7 @@ function errorResult(tool, value, sessionId = null, replayId = null) {
42632
43855
  tool,
42633
43856
  session_id: sessionId,
42634
43857
  ...replayId !== null ? { replay_id: replayId } : {},
42635
- error: errorMessage4(value),
42636
- raw: value && typeof value === "object" ? value : { value }
43858
+ error: errorMessage4(value)
42637
43859
  }, true);
42638
43860
  }
42639
43861
  function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_screenshot") {
@@ -42718,8 +43940,8 @@ function registerBrowserAgentMcpTools(server, opts) {
42718
43940
  });
42719
43941
  const data = await res.json().catch(() => ({}));
42720
43942
  return { ok: res.ok, data };
42721
- } catch (err) {
42722
- return { ok: false, data: { error: err instanceof Error ? err.message : String(err) } };
43943
+ } catch {
43944
+ return { ok: false, data: { error: publicErrorMessage("service_unavailable") } };
42723
43945
  }
42724
43946
  }
42725
43947
  async function downloadReplay(sessionId, replayId, filename) {
@@ -42748,8 +43970,8 @@ function registerBrowserAgentMcpTools(server, opts) {
42748
43970
  download_url: `${baseUrl}${path6}`
42749
43971
  }
42750
43972
  };
42751
- } catch (err) {
42752
- return { ok: false, data: { error: err instanceof Error ? err.message : String(err) } };
43973
+ } catch {
43974
+ return { ok: false, data: { error: publicErrorMessage("service_unavailable") } };
42753
43975
  }
42754
43976
  }
42755
43977
  const annotations = (title, readOnly = false) => ({
@@ -43421,6 +44643,7 @@ var init_browser_agent_mcp_server = __esm({
43421
44643
  init_browser_agent_tool_schemas();
43422
44644
  init_replay_annotator();
43423
44645
  init_outbound_sanitize();
44646
+ init_errors();
43424
44647
  init_output_schema_registry();
43425
44648
  }
43426
44649
  });
@@ -46812,7 +48035,7 @@ async function runBrowserAgentMigration() {
46812
48035
  }
46813
48036
  async function createExtensionRow(input) {
46814
48037
  const db = getDb();
46815
- const id = `bext_${(0, import_node_crypto23.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
48038
+ const id = `bext_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
46816
48039
  await db.execute({
46817
48040
  sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
46818
48041
  VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
@@ -46847,7 +48070,7 @@ async function deleteExtensionRow(userId, name) {
46847
48070
  }
46848
48071
  async function createAuthConnectionRow(input) {
46849
48072
  const db = getDb();
46850
- const connectionId = `authc_${(0, import_node_crypto23.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
48073
+ const connectionId = `authc_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
46851
48074
  await db.execute({
46852
48075
  sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
46853
48076
  VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
@@ -46932,7 +48155,7 @@ async function deleteProfileLabel(userId, profile) {
46932
48155
  }
46933
48156
  async function createSessionRow(input) {
46934
48157
  const db = getDb();
46935
- const id = `bas_${(0, import_node_crypto23.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
48158
+ const id = `bas_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
46936
48159
  await db.execute({
46937
48160
  sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
46938
48161
  VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
@@ -47001,7 +48224,7 @@ async function recordAction(input) {
47001
48224
  sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
47002
48225
  VALUES (?, ?, ?, ?, ?, ?)`,
47003
48226
  args: [
47004
- `baa_${(0, import_node_crypto23.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
48227
+ `baa_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
47005
48228
  input.sessionId,
47006
48229
  input.type,
47007
48230
  input.params == null ? null : JSON.stringify(input.params),
@@ -47041,11 +48264,11 @@ async function listReplayRows(sessionId) {
47041
48264
  });
47042
48265
  return res.rows;
47043
48266
  }
47044
- var import_node_crypto23, _ready2, _migrationPromise2;
48267
+ var import_node_crypto24, _ready2, _migrationPromise2;
47045
48268
  var init_browser_agent_db = __esm({
47046
48269
  "src/api/browser-agent-db.ts"() {
47047
48270
  "use strict";
47048
- import_node_crypto23 = require("crypto");
48271
+ import_node_crypto24 = require("crypto");
47049
48272
  init_db();
47050
48273
  _ready2 = false;
47051
48274
  _migrationPromise2 = null;
@@ -47807,7 +49030,7 @@ function keepHostedBrowserAlive(_browser) {
47807
49030
  function client() {
47808
49031
  const apiKey = browserServiceApiKey();
47809
49032
  if (!apiKey) throw new Error("Browser backend API key is required");
47810
- return new import_sdk8.default({ apiKey });
49033
+ return new import_sdk9.default({ apiKey });
47811
49034
  }
47812
49035
  function isProfileConflict(err) {
47813
49036
  const message = err instanceof Error ? err.message : String(err);
@@ -48165,11 +49388,11 @@ async function replayList(runtimeSessionId) {
48165
49388
  finishedAt: r.finished_at ?? null
48166
49389
  }));
48167
49390
  }
48168
- var import_sdk8, import_playwright5, DEFAULT_TIMEOUT_SECONDS;
49391
+ var import_sdk9, import_playwright5, DEFAULT_TIMEOUT_SECONDS;
48169
49392
  var init_browser_agent_service = __esm({
48170
49393
  "src/services/browser-agent/browser-agent-service.ts"() {
48171
49394
  "use strict";
48172
- import_sdk8 = __toESM(require("@onkernel/sdk"), 1);
49395
+ import_sdk9 = __toESM(require("@onkernel/sdk"), 1);
48173
49396
  import_playwright5 = require("playwright");
48174
49397
  init_browser_service_env();
48175
49398
  init_run_capture();
@@ -48801,7 +50024,7 @@ function buildBrowserAgentRoutes() {
48801
50024
  }
48802
50025
  const existing = await getExtensionRow(user.id, name);
48803
50026
  if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
48804
- const backendName = `u${user.id}_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "")}`;
50027
+ const backendName = `u${user.id}_${(0, import_node_crypto25.randomUUID)().replace(/-/g, "")}`;
48805
50028
  try {
48806
50029
  const imported = await importExtensionFromStore(storeUrl, backendName);
48807
50030
  const row = await createExtensionRow({
@@ -48860,11 +50083,11 @@ function buildBrowserAgentRoutes() {
48860
50083
  });
48861
50084
  return app2;
48862
50085
  }
48863
- var import_node_crypto24, import_hono21, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE;
50086
+ var import_node_crypto25, import_hono21, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE;
48864
50087
  var init_browser_agent_routes = __esm({
48865
50088
  "src/api/browser-agent-routes.ts"() {
48866
50089
  "use strict";
48867
- import_node_crypto24 = require("crypto");
50090
+ import_node_crypto25 = require("crypto");
48868
50091
  import_hono21 = require("hono");
48869
50092
  init_api_auth();
48870
50093
  init_errors();
@@ -49343,7 +50566,7 @@ async function getKeys() {
49343
50566
  const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
49344
50567
  const full = await (0, import_jose2.exportJWK)(privateKey);
49345
50568
  const publicJwk = { kty: full.kty, n: full.n, e: full.e };
49346
- const kid = (0, import_node_crypto25.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
50569
+ const kid = (0, import_node_crypto26.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
49347
50570
  publicJwk.kid = kid;
49348
50571
  publicJwk.alg = "RS256";
49349
50572
  publicJwk.use = "sig";
@@ -49522,23 +50745,23 @@ async function validateAuthRequest(p) {
49522
50745
  }
49523
50746
  function pkceMatches(verifier, challenge) {
49524
50747
  if (!verifier) return false;
49525
- const computed = (0, import_node_crypto25.createHash)("sha256").update(verifier).digest("base64url");
50748
+ const computed = (0, import_node_crypto26.createHash)("sha256").update(verifier).digest("base64url");
49526
50749
  return computed === challenge;
49527
50750
  }
49528
50751
  async function mintAccessToken(identity, scope, plan, audience) {
49529
50752
  const { privateKey, kid } = await getKeys();
49530
- return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto25.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
50753
+ return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto26.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
49531
50754
  }
49532
50755
  function tokenErrorResponse(c, error, description, status) {
49533
50756
  return c.json({ error, error_description: description }, status);
49534
50757
  }
49535
- var import_hono23, import_cookie, import_node_crypto25, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
50758
+ var import_hono23, import_cookie, import_node_crypto26, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
49536
50759
  var init_oauth_routes = __esm({
49537
50760
  "src/api/oauth-routes.ts"() {
49538
50761
  "use strict";
49539
50762
  import_hono23 = require("hono");
49540
50763
  import_cookie = require("hono/cookie");
49541
- import_node_crypto25 = require("crypto");
50764
+ import_node_crypto26 = require("crypto");
49542
50765
  import_jose2 = require("jose");
49543
50766
  init_session();
49544
50767
  init_db();
@@ -49624,7 +50847,7 @@ var init_oauth_routes = __esm({
49624
50847
  }
49625
50848
  }
49626
50849
  const clientName = typeof body.client_name === "string" ? body.client_name : null;
49627
- const clientId = `client_${(0, import_node_crypto25.randomBytes)(16).toString("hex")}`;
50850
+ const clientId = `client_${(0, import_node_crypto26.randomBytes)(16).toString("hex")}`;
49628
50851
  await registerClient(clientId, redirectUris, clientName);
49629
50852
  console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
49630
50853
  return c.json({
@@ -49677,7 +50900,7 @@ var init_oauth_routes = __esm({
49677
50900
  if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
49678
50901
  if (action !== "approve") return c.text("unsupported action", 400);
49679
50902
  const scope = negotiateScope(p.scope, user, p.resource);
49680
- const code = `code_${(0, import_node_crypto25.randomBytes)(32).toString("base64url")}`;
50903
+ const code = `code_${(0, import_node_crypto26.randomBytes)(32).toString("base64url")}`;
49681
50904
  const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
49682
50905
  await putCode({
49683
50906
  code,
@@ -49716,7 +50939,7 @@ var init_oauth_routes = __esm({
49716
50939
  const plan = user ? resolvePlan(user) : "free";
49717
50940
  const audience = record.resource ?? RESOURCE();
49718
50941
  const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
49719
- const refreshToken = `rt_${(0, import_node_crypto25.randomBytes)(40).toString("base64url")}`;
50942
+ const refreshToken = `rt_${(0, import_node_crypto26.randomBytes)(40).toString("base64url")}`;
49720
50943
  await putRefresh({
49721
50944
  refresh_token: refreshToken,
49722
50945
  client_id: clientId,
@@ -49746,7 +50969,7 @@ var init_oauth_routes = __esm({
49746
50969
  const plan = user ? resolvePlan(user) : "free";
49747
50970
  const audience = record.resource ?? RESOURCE();
49748
50971
  const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
49749
- const nextRefresh = `rt_${(0, import_node_crypto25.randomBytes)(40).toString("base64url")}`;
50972
+ const nextRefresh = `rt_${(0, import_node_crypto26.randomBytes)(40).toString("base64url")}`;
49750
50973
  await rotateRefresh(refreshToken, {
49751
50974
  refresh_token: nextRefresh,
49752
50975
  client_id: record.client_id,
@@ -51166,7 +52389,7 @@ function quoteUntrusted(value) {
51166
52389
  return clean2.split("\n").map((line) => `> ${line}`).join("\n");
51167
52390
  }
51168
52391
  function shortHash(value, length = 24) {
51169
- return (0, import_node_crypto26.createHash)("sha256").update(value).digest("hex").slice(0, length);
52392
+ return (0, import_node_crypto27.createHash)("sha256").update(value).digest("hex").slice(0, length);
51170
52393
  }
51171
52394
  function safePathPart(value) {
51172
52395
  return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "contact";
@@ -51654,11 +52877,11 @@ async function captureSupportMessage(memoryKey, email, options) {
51654
52877
  direction: options.direction
51655
52878
  };
51656
52879
  }
51657
- var import_node_crypto26, SUPPORT_TAGS, ISSUE_TAGS;
52880
+ var import_node_crypto27, SUPPORT_TAGS, ISSUE_TAGS;
51658
52881
  var init_resend_support_thread = __esm({
51659
52882
  "src/api/resend-support-thread.ts"() {
51660
52883
  "use strict";
51661
- import_node_crypto26 = require("crypto");
52884
+ import_node_crypto27 = require("crypto");
51662
52885
  init_memory();
51663
52886
  SUPPORT_TAGS = [
51664
52887
  {
@@ -52243,7 +53466,7 @@ async function claimWorkshopRegistration(input) {
52243
53466
  ) < ?
52244
53467
  `,
52245
53468
  args: [
52246
- (0, import_node_crypto27.randomUUID)(),
53469
+ (0, import_node_crypto28.randomUUID)(),
52247
53470
  input.eventSlug,
52248
53471
  input.email,
52249
53472
  input.firstName,
@@ -52260,7 +53483,7 @@ async function claimWorkshopRegistration(input) {
52260
53483
  id, event_slug, email, first_name, last_name, status
52261
53484
  ) VALUES (?, ?, ?, ?, ?, 'waitlisted')
52262
53485
  `,
52263
- args: [(0, import_node_crypto27.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
53486
+ args: [(0, import_node_crypto28.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
52264
53487
  });
52265
53488
  const waitlisted = await getWorkshopRegistration(input.eventSlug, input.email);
52266
53489
  if (!waitlisted) throw new Error("workshop registration could not be claimed");
@@ -52275,7 +53498,7 @@ async function registerWorkshopInterest(input) {
52275
53498
  id, event_slug, email, first_name, last_name, status
52276
53499
  ) VALUES (?, ?, ?, ?, ?, 'interested')
52277
53500
  `,
52278
- args: [(0, import_node_crypto27.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
53501
+ args: [(0, import_node_crypto28.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
52279
53502
  });
52280
53503
  const registration = await getWorkshopRegistration(input.eventSlug, input.email);
52281
53504
  if (!registration) throw new Error("workshop interest could not be recorded");
@@ -52305,11 +53528,11 @@ async function updateWorkshopRegistration(id, patch) {
52305
53528
  args: [...entries.map(([, value]) => value), id]
52306
53529
  });
52307
53530
  }
52308
- var import_node_crypto27;
53531
+ var import_node_crypto28;
52309
53532
  var init_workshop_registration_repository = __esm({
52310
53533
  "src/api/workshop-registration-repository.ts"() {
52311
53534
  "use strict";
52312
- import_node_crypto27 = require("crypto");
53535
+ import_node_crypto28 = require("crypto");
52313
53536
  init_db();
52314
53537
  }
52315
53538
  });
@@ -52854,15 +54077,15 @@ function renderEditorialReadingRoom(input, now = /* @__PURE__ */ new Date()) {
52854
54077
  articleCount: articles.length,
52855
54078
  wordCount: totalWordCount,
52856
54079
  bytes,
52857
- sha256: (0, import_node_crypto28.createHash)("sha256").update(html).digest("hex"),
54080
+ sha256: (0, import_node_crypto29.createHash)("sha256").update(html).digest("hex"),
52858
54081
  warnings
52859
54082
  };
52860
54083
  }
52861
- var import_node_crypto28, import_node_fs14, import_node_path19, import_marked, runtimeEntryDir, assetCache;
54084
+ var import_node_crypto29, import_node_fs14, import_node_path19, import_marked, runtimeEntryDir, assetCache;
52862
54085
  var init_render = __esm({
52863
54086
  "src/editorial-reading-room/render.ts"() {
52864
54087
  "use strict";
52865
- import_node_crypto28 = require("crypto");
54088
+ import_node_crypto29 = require("crypto");
52866
54089
  import_node_fs14 = require("fs");
52867
54090
  import_node_path19 = require("path");
52868
54091
  import_marked = require("marked");
@@ -53137,7 +54360,7 @@ var init_site_extract_reconciliation = __esm({
53137
54360
 
53138
54361
  // src/api/page-diff.ts
53139
54362
  function sha256Hex(value) {
53140
- return (0, import_node_crypto29.createHash)("sha256").update(value).digest("hex");
54363
+ return (0, import_node_crypto30.createHash)("sha256").update(value).digest("hex");
53141
54364
  }
53142
54365
  function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
53143
54366
  if (value.length <= maxChars) return { value, truncated: false };
@@ -53194,11 +54417,11 @@ function diffPageContent(oldContent, newContent) {
53194
54417
  totalChangedLineCount
53195
54418
  };
53196
54419
  }
53197
- var import_node_crypto29, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
54420
+ var import_node_crypto30, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
53198
54421
  var init_page_diff = __esm({
53199
54422
  "src/api/page-diff.ts"() {
53200
54423
  "use strict";
53201
- import_node_crypto29 = require("crypto");
54424
+ import_node_crypto30 = require("crypto");
53202
54425
  import_diff = require("diff");
53203
54426
  MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
53204
54427
  MAX_DIFF_HUNKS = 200;
@@ -53697,7 +54920,7 @@ function canonicalJson(value) {
53697
54920
  return JSON.stringify(value);
53698
54921
  }
53699
54922
  function sha2562(value) {
53700
- return (0, import_node_crypto30.createHash)("sha256").update(value).digest("hex");
54923
+ return (0, import_node_crypto31.createHash)("sha256").update(value).digest("hex");
53701
54924
  }
53702
54925
  function sensitiveKey(key) {
53703
54926
  const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
@@ -53912,11 +55135,11 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
53912
55135
  ...!searchReady ? { warning: "The snapshot was stored, but no search chunks were indexed yet." } : {}
53913
55136
  };
53914
55137
  }
53915
- var import_node_crypto30, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
55138
+ var import_node_crypto31, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
53916
55139
  var init_connection_memory_import = __esm({
53917
55140
  "src/api/connection-memory-import.ts"() {
53918
55141
  "use strict";
53919
- import_node_crypto30 = require("crypto");
55142
+ import_node_crypto31 = require("crypto");
53920
55143
  init_slugify();
53921
55144
  CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
53922
55145
  CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
@@ -54090,7 +55313,7 @@ function finitePositive(value, fallback) {
54090
55313
  return Number.isFinite(value) && value > 0 ? value : fallback;
54091
55314
  }
54092
55315
  async function collectConnectedDataExport(args) {
54093
- const exportId = (0, import_node_crypto31.randomUUID)();
55316
+ const exportId = (0, import_node_crypto32.randomUUID)();
54094
55317
  const now = args.now ?? Date.now;
54095
55318
  const startedAt = now();
54096
55319
  const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
@@ -54200,11 +55423,11 @@ ${lines.length ? `${lines.join("\n")}
54200
55423
  untrustedContent: true
54201
55424
  };
54202
55425
  }
54203
- var import_node_crypto31, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
55426
+ var import_node_crypto32, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
54204
55427
  var init_connected_data_export = __esm({
54205
55428
  "src/api/connected-data-export.ts"() {
54206
55429
  "use strict";
54207
- import_node_crypto31 = require("crypto");
55430
+ import_node_crypto32 = require("crypto");
54208
55431
  CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
54209
55432
  process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
54210
55433
  );
@@ -54317,7 +55540,7 @@ async function exportSearchConsoleTableData(args) {
54317
55540
  offset += rows.length;
54318
55541
  if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
54319
55542
  }
54320
- const exportId = (0, import_node_crypto32.randomUUID)();
55543
+ const exportId = (0, import_node_crypto33.randomUUID)();
54321
55544
  const artifact = await args.writeArtifact({
54322
55545
  ownerId: args.ownerId,
54323
55546
  exportId,
@@ -54339,11 +55562,11 @@ async function exportSearchConsoleTableData(args) {
54339
55562
  warnings
54340
55563
  };
54341
55564
  }
54342
- var import_node_crypto32, SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS, SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE, SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES, SEARCH_CONSOLE_TABLE_COLUMNS, SearchConsoleTableExportValidationError;
55565
+ var import_node_crypto33, SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS, SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE, SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES, SEARCH_CONSOLE_TABLE_COLUMNS, SearchConsoleTableExportValidationError;
54343
55566
  var init_search_console_table_export = __esm({
54344
55567
  "src/api/search-console-table-export.ts"() {
54345
55568
  "use strict";
54346
- import_node_crypto32 = require("crypto");
55569
+ import_node_crypto33 = require("crypto");
54347
55570
  SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
54348
55571
  SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
54349
55572
  SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
@@ -54771,7 +55994,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
54771
55994
  updated_at = excluded.updated_at
54772
55995
  `,
54773
55996
  args: [
54774
- (0, import_node_crypto33.randomUUID)(),
55997
+ (0, import_node_crypto34.randomUUID)(),
54775
55998
  userId,
54776
55999
  connection.providerConfigKey,
54777
56000
  connection.provider,
@@ -54842,7 +56065,7 @@ async function recordServiceConnectionHealth(args) {
54842
56065
  });
54843
56066
  await getDb().execute({
54844
56067
  sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
54845
- args: [(0, import_node_crypto33.randomUUID)(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
56068
+ args: [(0, import_node_crypto34.randomUUID)(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
54846
56069
  });
54847
56070
  }
54848
56071
  async function setServiceConnectionActions(identity, connectionId, enabled) {
@@ -54882,7 +56105,7 @@ async function claimServiceConnectionAction(args) {
54882
56105
  if (!connection) throw new Error("service_connection_not_found");
54883
56106
  const inserted = await getDb().execute({
54884
56107
  sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status) VALUES (?, ?, ?, ?, ?, 'started')`,
54885
- args: [(0, import_node_crypto33.randomUUID)(), connection.id, connection.userId, args.tool, args.requestId]
56108
+ args: [(0, import_node_crypto34.randomUUID)(), connection.id, connection.userId, args.tool, args.requestId]
54886
56109
  });
54887
56110
  if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
54888
56111
  const existing = await getDb().execute({
@@ -54895,11 +56118,11 @@ async function claimServiceConnectionAction(args) {
54895
56118
  errorCode: typeof existing.rows[0]?.error_code === "string" ? existing.rows[0].error_code : null
54896
56119
  };
54897
56120
  }
54898
- var import_node_crypto33, schemaReady2, schemaDb3;
56121
+ var import_node_crypto34, schemaReady2, schemaDb3;
54899
56122
  var init_service_connections = __esm({
54900
56123
  "src/api/service-connections.ts"() {
54901
56124
  "use strict";
54902
- import_node_crypto33 = require("crypto");
56125
+ import_node_crypto34 = require("crypto");
54903
56126
  init_db();
54904
56127
  schemaReady2 = null;
54905
56128
  schemaDb3 = null;
@@ -55163,7 +56386,7 @@ async function withNangoClient(connection, run) {
55163
56386
  }
55164
56387
  }
55165
56388
  });
55166
- const client2 = new import_client9.Client({ name: "mcp-scraper-integrations", version: "1.0.0" }, { capabilities: {} });
56389
+ const client2 = new import_client11.Client({ name: "mcp-scraper-integrations", version: "1.0.0" }, { capabilities: {} });
55167
56390
  try {
55168
56391
  await client2.connect(transport);
55169
56392
  const value = await run(client2);
@@ -55216,7 +56439,7 @@ async function listNangoToolsDirect(identity, connectionId) {
55216
56439
  });
55217
56440
  const readTools = tools.filter((tool) => classifyTool(tool) === "read").map((tool) => tool.name);
55218
56441
  const actionTools = tools.filter((tool) => classifyTool(tool) === "action").map((tool) => tool.name);
55219
- const revision = (0, import_node_crypto34.createHash)("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
56442
+ const revision = (0, import_node_crypto35.createHash)("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
55220
56443
  await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
55221
56444
  const refreshed = await getOwnedServiceConnection(identity, connection.id);
55222
56445
  return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
@@ -55236,8 +56459,8 @@ async function callNangoToolDirect(args) {
55236
56459
  identity: args.identity,
55237
56460
  ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
55238
56461
  });
55239
- const requestId = args.requestId?.trim() || (0, import_node_crypto34.randomUUID)();
55240
- const idempotencyKey2 = `main-nango:${(0, import_node_crypto34.createHash)("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
56462
+ const requestId = args.requestId?.trim() || (0, import_node_crypto35.randomUUID)();
56463
+ const idempotencyKey2 = `main-nango:${(0, import_node_crypto35.createHash)("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
55241
56464
  const startedAt = /* @__PURE__ */ new Date();
55242
56465
  const started = performance.now();
55243
56466
  let result;
@@ -55262,7 +56485,7 @@ async function callNangoToolDirect(args) {
55262
56485
  toolName: args.tool,
55263
56486
  operationKind: args.operationKind ?? args.classification,
55264
56487
  outcome: providerError ? "error" : "partial",
55265
- requestId: requestId.length <= 200 ? requestId : (0, import_node_crypto34.createHash)("sha256").update(requestId).digest("hex"),
56488
+ requestId: requestId.length <= 200 ? requestId : (0, import_node_crypto35.createHash)("sha256").update(requestId).digest("hex"),
55266
56489
  startedAt: startedAt.toISOString(),
55267
56490
  completedAt: completedAt.toISOString()
55268
56491
  }
@@ -55299,16 +56522,16 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
55299
56522
  providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
55300
56523
  protocolVersion: null,
55301
56524
  schemaSource: "live_tools_list",
55302
- schemaHash: (0, import_node_crypto34.createHash)("sha256").update(JSON.stringify(projected)).digest("hex"),
56525
+ schemaHash: (0, import_node_crypto35.createHash)("sha256").update(JSON.stringify(projected)).digest("hex"),
55303
56526
  fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
55304
56527
  };
55305
56528
  }
55306
- var import_node_crypto34, import_client9, import_streamableHttp, DEFAULT_NANGO_MCP_URL, NANGO_TIMEOUT_MS, NANGO_CONNECTION_PAGE_SIZE, NANGO_CONNECTION_MAX_PAGES, MAIN_INTEGRATION_CONTRACT_VERSION, MAIN_INTEGRATION_CONTRACT_HASH, MainNangoTransportError;
56529
+ var import_node_crypto35, import_client11, import_streamableHttp, DEFAULT_NANGO_MCP_URL, NANGO_TIMEOUT_MS, NANGO_CONNECTION_PAGE_SIZE, NANGO_CONNECTION_MAX_PAGES, MAIN_INTEGRATION_CONTRACT_VERSION, MAIN_INTEGRATION_CONTRACT_HASH, MainNangoTransportError;
55307
56530
  var init_main_nango_transport = __esm({
55308
56531
  "src/api/main-nango-transport.ts"() {
55309
56532
  "use strict";
55310
- import_node_crypto34 = require("crypto");
55311
- import_client9 = require("@modelcontextprotocol/sdk/client/index.js");
56533
+ import_node_crypto35 = require("crypto");
56534
+ import_client11 = require("@modelcontextprotocol/sdk/client/index.js");
55312
56535
  import_streamableHttp = require("@modelcontextprotocol/sdk/client/streamableHttp.js");
55313
56536
  init_service_connections();
55314
56537
  init_connected_usage_billing();
@@ -55929,7 +57152,7 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
55929
57152
  return data.connection.actionsEnabled === true;
55930
57153
  }
55931
57154
  async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey2) {
55932
- const requestId = `main-connected-action:${(0, import_node_crypto35.createHash)("sha256").update(identity).update("\0").update(idempotencyKey2?.trim() || (0, import_node_crypto35.randomUUID)()).digest("hex")}`;
57155
+ const requestId = `main-connected-action:${(0, import_node_crypto36.createHash)("sha256").update(identity).update("\0").update(idempotencyKey2?.trim() || (0, import_node_crypto36.randomUUID)()).digest("hex")}`;
55933
57156
  if (mainOwnsIntegrations()) {
55934
57157
  const selectedTool = tool?.trim();
55935
57158
  if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
@@ -56071,7 +57294,7 @@ function canonicalJson2(value) {
56071
57294
  return JSON.stringify(value);
56072
57295
  }
56073
57296
  function projectedToolSchemaHash(tool) {
56074
- return (0, import_node_crypto35.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
57297
+ return (0, import_node_crypto36.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
56075
57298
  }
56076
57299
  async function describeNangoTool(identity, connectionId, tool, fresh) {
56077
57300
  if (mainOwnsIntegrations()) {
@@ -56332,11 +57555,11 @@ async function callMainOwnedExportPage(identity, input) {
56332
57555
  untrustedContent: true
56333
57556
  };
56334
57557
  }
56335
- var import_node_crypto35, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, FIXED_CONTROL_ERROR_MESSAGES, CONTROL_ERROR_CODE_ALIASES;
57558
+ var import_node_crypto36, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, FIXED_CONTROL_ERROR_MESSAGES, CONTROL_ERROR_CODE_ALIASES;
56336
57559
  var init_nango_control = __esm({
56337
57560
  "src/api/nango-control.ts"() {
56338
57561
  "use strict";
56339
- import_node_crypto35 = require("crypto");
57562
+ import_node_crypto36 = require("crypto");
56340
57563
  init_connected_data_export();
56341
57564
  init_main_nango_transport();
56342
57565
  init_service_connections();
@@ -56804,8 +58027,8 @@ function signingSecret() {
56804
58027
  return secret2;
56805
58028
  }
56806
58029
  function schedulerIntegrationSignature(args) {
56807
- const bodyHash = (0, import_node_crypto36.createHash)("sha256").update(args.body).digest("hex");
56808
- return (0, import_node_crypto36.createHmac)("sha256", args.secret).update(`${args.method.toUpperCase()}
58030
+ const bodyHash = (0, import_node_crypto37.createHash)("sha256").update(args.body).digest("hex");
58031
+ return (0, import_node_crypto37.createHmac)("sha256", args.secret).update(`${args.method.toUpperCase()}
56809
58032
  ${args.path}
56810
58033
  ${args.timestamp}
56811
58034
  ${args.nonce}
@@ -56852,17 +58075,17 @@ async function verifySchedulerIntegrationRequest(request, rawBody) {
56852
58075
  });
56853
58076
  const suppliedBytes = Buffer.from(signature, "hex");
56854
58077
  const expectedBytes = Buffer.from(expected, "hex");
56855
- if (suppliedBytes.length !== expectedBytes.length || !(0, import_node_crypto36.timingSafeEqual)(suppliedBytes, expectedBytes)) {
58078
+ if (suppliedBytes.length !== expectedBytes.length || !(0, import_node_crypto37.timingSafeEqual)(suppliedBytes, expectedBytes)) {
56856
58079
  throw new SchedulerIntegrationAuthError("invalid_signature");
56857
58080
  }
56858
58081
  await claimNonce(nonce, timestampMs);
56859
58082
  return { requestId };
56860
58083
  }
56861
- var import_node_crypto36, MAX_CLOCK_SKEW_MS, SchedulerIntegrationAuthError;
58084
+ var import_node_crypto37, MAX_CLOCK_SKEW_MS, SchedulerIntegrationAuthError;
56862
58085
  var init_scheduler_integration_auth = __esm({
56863
58086
  "src/api/scheduler-integration-auth.ts"() {
56864
58087
  "use strict";
56865
- import_node_crypto36 = require("crypto");
58088
+ import_node_crypto37 = require("crypto");
56866
58089
  init_db();
56867
58090
  init_service_connections();
56868
58091
  MAX_CLOCK_SKEW_MS = 5 * 60 * 1e3;
@@ -56877,31 +58100,6 @@ var init_scheduler_integration_auth = __esm({
56877
58100
  }
56878
58101
  });
56879
58102
 
56880
- // src/api/webhook.ts
56881
- async function deliverWebhook(url, payload, retries = 3) {
56882
- for (let attempt = 1; attempt <= retries; attempt++) {
56883
- try {
56884
- const res = await fetch(url, {
56885
- method: "POST",
56886
- headers: { "content-type": "application/json" },
56887
- body: JSON.stringify(payload),
56888
- signal: AbortSignal.timeout(1e4)
56889
- });
56890
- if (res.ok) return;
56891
- console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
56892
- } catch (err) {
56893
- console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
56894
- }
56895
- if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
56896
- }
56897
- console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
56898
- }
56899
- var init_webhook = __esm({
56900
- "src/api/webhook.ts"() {
56901
- "use strict";
56902
- }
56903
- });
56904
-
56905
58103
  // src/api/worker.ts
56906
58104
  var worker_exports = {};
56907
58105
  __export(worker_exports, {
@@ -56915,8 +58113,8 @@ function countPaaQuestions(result) {
56915
58113
  if (typeof value.totalQuestions === "number") return value.totalQuestions;
56916
58114
  return Array.isArray(value.flat) ? value.flat.length : 0;
56917
58115
  }
56918
- function paaCostForQuestionCount(questionCount) {
56919
- return MC_COSTS.paa_base + Math.max(1, questionCount) * MC_COSTS.paa;
58116
+ function paaCostForQuestionCount(questionCount2) {
58117
+ return MC_COSTS.paa_base + Math.max(1, questionCount2) * MC_COSTS.paa;
56920
58118
  }
56921
58119
  async function processJob(job) {
56922
58120
  running++;
@@ -56948,9 +58146,18 @@ async function processJob(job) {
56948
58146
  else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
56949
58147
  }
56950
58148
  if (job.callback_url) {
56951
- await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result, attempts });
58149
+ await deliverWebhook(job.callback_url, {
58150
+ job_id: job.id,
58151
+ status: "done",
58152
+ result: sanitizeHarvestResult(result),
58153
+ attempts: sanitizeAttempts(attempts)
58154
+ });
56952
58155
  }
56953
58156
  } catch (err) {
58157
+ console.error("[harvest/worker] failed", {
58158
+ jobId: job.id,
58159
+ error: err instanceof Error ? err.message : String(err)
58160
+ });
56954
58161
  const problem = classifyHarvestProblem(err);
56955
58162
  await failJob(job.id, serializeHarvestProblem(problem));
56956
58163
  const attempts = await listHarvestAttempts(job.id, job.user_id);
@@ -56962,7 +58169,12 @@ async function processJob(job) {
56962
58169
  } catch {
56963
58170
  }
56964
58171
  if (job.callback_url) {
56965
- await deliverWebhook(job.callback_url, { job_id: job.id, status: "failed", ...harvestProblemResponse(problem), attempts });
58172
+ await deliverWebhook(job.callback_url, {
58173
+ job_id: job.id,
58174
+ status: "failed",
58175
+ ...harvestProblemResponse(problem),
58176
+ attempts: sanitizeAttempts(attempts)
58177
+ });
56966
58178
  }
56967
58179
  } finally {
56968
58180
  running--;
@@ -57005,6 +58217,7 @@ var init_worker = __esm({
57005
58217
  init_harvest_problems();
57006
58218
  init_harvest_attempt_events();
57007
58219
  init_cost_context();
58220
+ init_outbound_sanitize();
57008
58221
  MAX_CONCURRENT = 2;
57009
58222
  running = 0;
57010
58223
  }
@@ -57324,8 +58537,8 @@ function countPaaQuestions2(result) {
57324
58537
  if (typeof value.totalQuestions === "number") return value.totalQuestions;
57325
58538
  return Array.isArray(value.flat) ? value.flat.length : 0;
57326
58539
  }
57327
- function paaCostForQuestionCount2(questionCount) {
57328
- return MC_COSTS.paa_base + Math.max(1, questionCount) * MC_COSTS.paa;
58540
+ function paaCostForQuestionCount2(questionCount2) {
58541
+ return MC_COSTS.paa_base + Math.max(1, questionCount2) * MC_COSTS.paa;
57329
58542
  }
57330
58543
  async function checkHarvestLimits(user, reuseLockId) {
57331
58544
  if (reuseLockId && await reuseExistingConcurrencyGate(user.id, reuseLockId, 30 * 60)) return null;
@@ -57359,7 +58572,7 @@ async function chargeTierChangeNow(stripeClient, subscriptionId, customerId) {
57359
58572
  return { ok: false, amountDue: 0, error: err instanceof Error ? err.message : "Unable to charge the plan change immediately." };
57360
58573
  }
57361
58574
  }
57362
- var import_resend3, import_hono30, import_hono31, import_factory7, import_cookie2, import_stripe2, secureCookies2, isProduction2, sessionCookieOptions2, requireAllowedOrigin, auth2, sessionAuth, requireIntegrationsTier, requirePaidSchedulingTier, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
58575
+ var import_resend3, import_node_crypto38, import_hono30, import_hono31, import_factory7, import_cookie2, import_stripe2, secureCookies2, isProduction2, sessionCookieOptions2, requireAllowedOrigin, auth2, sessionAuth, requireIntegrationsTier, requirePaidSchedulingTier, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
57363
58576
  var init_server = __esm({
57364
58577
  "src/api/server.ts"() {
57365
58578
  "use strict";
@@ -57372,6 +58585,7 @@ var init_server = __esm({
57372
58585
  init_og();
57373
58586
  import_resend3 = require("resend");
57374
58587
  init_url_utils();
58588
+ import_node_crypto38 = require("crypto");
57375
58589
  init_kpo_extractor();
57376
58590
  init_screenshot();
57377
58591
  init_media_extractor();
@@ -57383,6 +58597,7 @@ var init_server = __esm({
57383
58597
  init_site_audit();
57384
58598
  init_site_extract();
57385
58599
  init_directory_workflow2();
58600
+ init_paa_harvest();
57386
58601
  init_site_extract_repository();
57387
58602
  init_site_extract_start();
57388
58603
  init_catalog_seed();
@@ -57407,6 +58622,8 @@ var init_server = __esm({
57407
58622
  init_directory_routes();
57408
58623
  init_location_data_routes();
57409
58624
  init_directory_workflow_dispatch();
58625
+ init_paa_harvest_dispatch();
58626
+ init_paa_harvest_reconciliation();
57410
58627
  init_directory_workflow_reconciliation();
57411
58628
  init_directory_artifacts();
57412
58629
  init_workflow_routes();
@@ -57516,6 +58733,18 @@ var init_server = __esm({
57516
58733
  STRIPE_API_VERSION = "2026-02-25.clover";
57517
58734
  app.use("*", async (c, next) => {
57518
58735
  await next();
58736
+ if (c.res.status >= 500 && c.res.headers.get("content-type")?.includes("application/json")) {
58737
+ try {
58738
+ const payload = await c.res.clone().json();
58739
+ const headers = new Headers(c.res.headers);
58740
+ c.res = new Response(JSON.stringify(sanitizeOutboundDiagnostics(payload)), {
58741
+ status: c.res.status,
58742
+ statusText: c.res.statusText,
58743
+ headers
58744
+ });
58745
+ } catch {
58746
+ }
58747
+ }
57519
58748
  c.header("Cache-Control", "no-store");
57520
58749
  c.header("X-Content-Type-Options", "nosniff");
57521
58750
  c.header("Referrer-Policy", "same-origin");
@@ -58828,9 +60057,50 @@ var init_server = __esm({
58828
60057
  pages: Math.min(2, Math.max(1, body.pages ?? 1))
58829
60058
  };
58830
60059
  const harvestCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount2(options.maxQuestions);
58831
- const { ok: harvestOk, balance_mc: harvestBal } = await debitMc(user.id, harvestCost, options.serpOnly ? LedgerOperation.SERP : LedgerOperation.PAA, options.query);
58832
- if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
58833
- const jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
60060
+ let jobId2;
60061
+ if (options.serpOnly) {
60062
+ const { ok: harvestOk, balance_mc: harvestBal } = await debitMc(
60063
+ user.id,
60064
+ harvestCost,
60065
+ LedgerOperation.SERP,
60066
+ options.query
60067
+ );
60068
+ if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
60069
+ jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
60070
+ } else {
60071
+ jobId2 = (0, import_node_crypto38.randomUUID)();
60072
+ const billingDebitKey = `paa-harvest:${jobId2}:hold`;
60073
+ const description = `PAA harvest: ${options.query}`.slice(0, 500);
60074
+ const hold = await debitMcIdempotent(
60075
+ user.id,
60076
+ harvestCost,
60077
+ LedgerOperation.PAA,
60078
+ description,
60079
+ billingDebitKey
60080
+ );
60081
+ if (!hold.ok) return c.json(insufficientBalanceResponse(hold.balance_mc, harvestCost), 402);
60082
+ try {
60083
+ await createJobWithId(jobId2, user.id, options.query, {
60084
+ ...options,
60085
+ executionOwner: "inngest",
60086
+ billingHoldMc: harvestCost,
60087
+ billingDebitKey
60088
+ }, body.callback_url);
60089
+ } catch (error) {
60090
+ await settleDebitMcIdempotent(
60091
+ user.id,
60092
+ billingDebitKey,
60093
+ 0,
60094
+ LedgerOperation.PAA_REFUND,
60095
+ "PAA job creation failed",
60096
+ "paa_harvest"
60097
+ ).catch(() => void 0);
60098
+ throw error;
60099
+ }
60100
+ await dispatchPaaHarvest(jobId2).catch((error) => {
60101
+ console.error("[harvest] durable PAA dispatch pending reconciliation:", error instanceof Error ? error.message : String(error));
60102
+ });
60103
+ }
58834
60104
  if (process.env.CRON_SECRET) {
58835
60105
  const url = new URL(c.req.url);
58836
60106
  void fetch(`${url.origin}/cron/tick`, {
@@ -58905,6 +60175,10 @@ var init_server = __esm({
58905
60175
  }
58906
60176
  return c.json({ job_id: jobId2, status: "done", result: sanitizeHarvestResult(result), attempts: sanitizeAttempts(attempts) });
58907
60177
  } catch (err) {
60178
+ console.error("[harvest/sync] failed", {
60179
+ jobId: jobId2,
60180
+ error: err instanceof Error ? err.message : String(err)
60181
+ });
58908
60182
  const problem = classifyHarvestProblem(err);
58909
60183
  const response = harvestProblemResponse(problem);
58910
60184
  const attempts = await listHarvestAttempts(jobId2, user.id);
@@ -59935,7 +61209,7 @@ var init_server = __esm({
59935
61209
  const { drainQueue: drainQueue2 } = await Promise.resolve().then(() => (init_worker(), worker_exports));
59936
61210
  const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
59937
61211
  const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
59938
- const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation] = await Promise.all([
61212
+ const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation] = await Promise.all([
59939
61213
  drainQueue2(budget),
59940
61214
  runMonthlyRefreshSweep(),
59941
61215
  reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
@@ -59948,7 +61222,9 @@ var init_server = __esm({
59948
61222
  cleanupExpiredDirectoryArtifacts().catch(() => ({ deleted: 0, store: "none" })),
59949
61223
  cleanupExpiredSiteExtractArtifacts().catch(() => ({ deleted: 0, store: "none" })),
59950
61224
  redispatchFundedSiteExtractJobs(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
59951
- reconcileSiteExtractSettlements(25).catch(() => ({ checked: 0, settled: 0, failed: 0 }))
61225
+ reconcileSiteExtractSettlements(25).catch(() => ({ checked: 0, settled: 0, failed: 0 })),
61226
+ redispatchPendingPaaHarvests(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
61227
+ reconcilePaaHarvestSettlements(25).catch(() => ({ checked: 0, staleFailed: 0, settled: 0, failed: 0 }))
59952
61228
  ]);
59953
61229
  return c.json({
59954
61230
  drained: results.length,
@@ -59965,6 +61241,8 @@ var init_server = __esm({
59965
61241
  siteExtractArtifactCleanup,
59966
61242
  siteExtractRedispatch,
59967
61243
  siteExtractReconciliation,
61244
+ paaHarvestRedispatch,
61245
+ paaHarvestReconciliation,
59968
61246
  workflowDispatch: workflowDispatchResult
59969
61247
  });
59970
61248
  });
@@ -60024,7 +61302,10 @@ var init_server = __esm({
60024
61302
  const bundle = stored.find((a) => a.contentType === "application/zip" || a.filename?.endsWith(".zip"));
60025
61303
  return c.json({ ok: true, jobId: jobId2, artifacts: stored.length, settlement, bundleUrl: bundle?.url ?? null, bundleBytes: bundle?.bytes ?? null });
60026
61304
  });
60027
- app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono31.serve)({ client: inngest, functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn] }));
61305
+ app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono31.serve)({
61306
+ client: inngest,
61307
+ functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, paaHarvestFn]
61308
+ }));
60028
61309
  app.route("/admin/credits", adminCreditsApp);
60029
61310
  app.route("/api/internal/site-architecture-auditor", siteAuditApp);
60030
61311
  app.route("/api/internal/memory", internalMemoryApp);