mcp-scraper 0.40.2 → 0.40.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +1517 -236
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +155 -87
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +6 -5
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs +31 -9
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +4 -4
- package/dist/{chunk-XORPNO3Z.js → chunk-27DUCRAZ.js} +2 -2
- package/dist/{chunk-44HZLHDV.js → chunk-5UN33CGU.js} +38 -1
- package/dist/chunk-5UN33CGU.js.map +1 -0
- package/dist/{chunk-X2LKCX6H.js → chunk-6J57U6HA.js} +3 -3
- package/dist/{chunk-LP6E462I.js → chunk-6SZ52BQ5.js} +49 -101
- package/dist/chunk-6SZ52BQ5.js.map +1 -0
- package/dist/chunk-BBI7RGOT.js +99 -0
- package/dist/chunk-BBI7RGOT.js.map +1 -0
- package/dist/{chunk-NVXNEOUQ.js → chunk-CS4HE6QY.js} +47 -11
- package/dist/chunk-CS4HE6QY.js.map +1 -0
- package/dist/{chunk-3PIWJS6Y.js → chunk-EHES33KB.js} +2 -2
- package/dist/{chunk-GUVKHCKE.js → chunk-IQTT7CAB.js} +94 -3
- package/dist/chunk-IQTT7CAB.js.map +1 -0
- package/dist/{chunk-7XBBFBYY.js → chunk-JVIW4GK2.js} +44 -12
- package/dist/chunk-JVIW4GK2.js.map +1 -0
- package/dist/chunk-XU4ZLIB2.js +7 -0
- package/dist/chunk-XU4ZLIB2.js.map +1 -0
- package/dist/{db-W3CP562I.js → db-LVVU6NOK.js} +16 -2
- package/dist/{extract-bundle-K4PG3RZJ.js → extract-bundle-SKXEDG4Z.js} +4 -4
- package/dist/index.cjs +31 -9
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +4 -4
- package/dist/{location-data-repository-RLQX6SNM.js → location-data-repository-NBTCOA7B.js} +3 -3
- package/dist/{server-KM3CFWCF.js → server-W76RUYD3.js} +1108 -66
- package/dist/server-W76RUYD3.js.map +1 -0
- package/dist/{site-extract-repository-2SMMFKKL.js → site-extract-repository-6GX72U4N.js} +4 -4
- package/dist/{worker-FXAGFYOE.js → worker-5RWLFSAJ.js} +27 -28
- package/dist/worker-5RWLFSAJ.js.map +1 -0
- package/package.json +1 -1
- package/dist/chunk-44HZLHDV.js.map +0 -1
- package/dist/chunk-7XBBFBYY.js.map +0 -1
- package/dist/chunk-GUVKHCKE.js.map +0 -1
- package/dist/chunk-LP6E462I.js.map +0 -1
- package/dist/chunk-NVXNEOUQ.js.map +0 -1
- package/dist/chunk-ZUJLSICT.js +0 -7
- package/dist/chunk-ZUJLSICT.js.map +0 -1
- package/dist/server-KM3CFWCF.js.map +0 -1
- package/dist/worker-FXAGFYOE.js.map +0 -1
- /package/dist/{chunk-XORPNO3Z.js.map → chunk-27DUCRAZ.js.map} +0 -0
- /package/dist/{chunk-X2LKCX6H.js.map → chunk-6J57U6HA.js.map} +0 -0
- /package/dist/{chunk-3PIWJS6Y.js.map → chunk-EHES33KB.js.map} +0 -0
- /package/dist/{db-W3CP562I.js.map → db-LVVU6NOK.js.map} +0 -0
- /package/dist/{extract-bundle-K4PG3RZJ.js.map → extract-bundle-SKXEDG4Z.js.map} +0 -0
- /package/dist/{location-data-repository-RLQX6SNM.js.map → location-data-repository-NBTCOA7B.js.map} +0 -0
- /package/dist/{site-extract-repository-2SMMFKKL.js.map → site-extract-repository-6GX72U4N.js.map} +0 -0
package/dist/bin/api-server.cjs
CHANGED
|
@@ -76,14 +76,48 @@ var init_browser_service_env = __esm({
|
|
|
76
76
|
});
|
|
77
77
|
|
|
78
78
|
// src/errors.ts
|
|
79
|
+
function normalizePublicErrorCode(code) {
|
|
80
|
+
return typeof code === "string" && PUBLIC_ERROR_CODES.has(code) ? code : "service_unavailable";
|
|
81
|
+
}
|
|
82
|
+
function publicErrorMessage(code) {
|
|
83
|
+
switch (normalizePublicErrorCode(code)) {
|
|
84
|
+
case "request_aborted":
|
|
85
|
+
return "The request was cancelled before it completed.";
|
|
86
|
+
case "captcha_exhausted":
|
|
87
|
+
case "captcha_or_blocked":
|
|
88
|
+
return "The target site returned a verification challenge. Please retry later.";
|
|
89
|
+
case "location_mismatch":
|
|
90
|
+
return "The target site returned results for a different location. Please retry.";
|
|
91
|
+
case "proxy_tunnel_failed":
|
|
92
|
+
case "proxy_unavailable":
|
|
93
|
+
return "The web request could not establish a working connection. Please retry later.";
|
|
94
|
+
case "harvest_timeout":
|
|
95
|
+
case "mcp_request_timeout":
|
|
96
|
+
return "The request timed out before it completed. Please retry with a smaller request.";
|
|
97
|
+
default:
|
|
98
|
+
return PUBLIC_SERVICE_UNAVAILABLE_MESSAGE;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
79
101
|
function sanitizeVendorName(message) {
|
|
80
102
|
return message.replace(/kernel\.sh\s+sessions?/gi, "sessions").replace(/kernel\.sh\s+session/gi, "this session").replace(/kernel\.sh/gi, "the service").replace(/kernel\s+sessions?/gi, "sessions").replace(/kernel\s+session/gi, "this session").replace(/\bkernel\b/gi, "the service").replace(/ +/g, " ").trim();
|
|
81
103
|
}
|
|
82
|
-
var RECAPTCHA_INSTRUCTIONS, CaptchaError, ExtractionError, MapsSearchIncompletePageError, RequestAbortedError, LocationMismatchError;
|
|
104
|
+
var RECAPTCHA_INSTRUCTIONS, PUBLIC_SERVICE_UNAVAILABLE_MESSAGE, PUBLIC_ERROR_CODES, CaptchaError, ExtractionError, MapsSearchIncompletePageError, RequestAbortedError, LocationMismatchError;
|
|
83
105
|
var init_errors = __esm({
|
|
84
106
|
"src/errors.ts"() {
|
|
85
107
|
"use strict";
|
|
86
108
|
RECAPTCHA_INSTRUCTIONS = "Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.";
|
|
109
|
+
PUBLIC_SERVICE_UNAVAILABLE_MESSAGE = "This operation is temporarily unavailable. Please retry later.";
|
|
110
|
+
PUBLIC_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
111
|
+
"request_aborted",
|
|
112
|
+
"captcha_exhausted",
|
|
113
|
+
"captcha_or_blocked",
|
|
114
|
+
"location_mismatch",
|
|
115
|
+
"proxy_tunnel_failed",
|
|
116
|
+
"proxy_unavailable",
|
|
117
|
+
"harvest_timeout",
|
|
118
|
+
"mcp_request_timeout",
|
|
119
|
+
"service_unavailable"
|
|
120
|
+
]);
|
|
87
121
|
CaptchaError = class extends Error {
|
|
88
122
|
constructor(instructions) {
|
|
89
123
|
super(`CAPTCHA detected. ${instructions}`);
|
|
@@ -130,8 +164,8 @@ function redactVendorUrls(value) {
|
|
|
130
164
|
function sanitizeOutboundDiagnostics(value, parentKey = "") {
|
|
131
165
|
if (typeof value === "string") {
|
|
132
166
|
let out = redactVendorUrls(value);
|
|
133
|
-
if (SANITIZED_VALUE_KEYS.test(parentKey)
|
|
134
|
-
out =
|
|
167
|
+
if (SANITIZED_VALUE_KEYS.test(parentKey)) {
|
|
168
|
+
out = publicErrorMessage("service_unavailable");
|
|
135
169
|
}
|
|
136
170
|
return out;
|
|
137
171
|
}
|
|
@@ -139,6 +173,7 @@ function sanitizeOutboundDiagnostics(value, parentKey = "") {
|
|
|
139
173
|
if (value !== null && typeof value === "object") {
|
|
140
174
|
const out = {};
|
|
141
175
|
for (const [key, val] of Object.entries(value)) {
|
|
176
|
+
if (INTERNAL_DIAGNOSTIC_KEYS.has(key)) continue;
|
|
142
177
|
const renamed = KEY_RENAMES[key] ?? key;
|
|
143
178
|
if (VENDOR_URL_KEYS.has(key)) {
|
|
144
179
|
out[renamed] = null;
|
|
@@ -164,24 +199,47 @@ function sanitizeHarvestResult(result) {
|
|
|
164
199
|
}
|
|
165
200
|
};
|
|
166
201
|
}
|
|
167
|
-
var KEY_RENAMES, SANITIZED_VALUE_KEYS, VENDOR_URL_KEYS, VENDOR_URL_RE;
|
|
202
|
+
var KEY_RENAMES, SANITIZED_VALUE_KEYS, INTERNAL_DIAGNOSTIC_KEYS, VENDOR_URL_KEYS, VENDOR_URL_RE;
|
|
168
203
|
var init_outbound_sanitize = __esm({
|
|
169
204
|
"src/api/outbound-sanitize.ts"() {
|
|
170
205
|
"use strict";
|
|
171
206
|
init_errors();
|
|
172
207
|
KEY_RENAMES = {
|
|
173
|
-
kernel: "browserRuntime"
|
|
174
|
-
kernel_session_id: "browser_session_id",
|
|
175
|
-
kernel_delete_started: "session_cleanup_started",
|
|
176
|
-
kernel_delete_succeeded: "session_cleanup_succeeded",
|
|
177
|
-
kernel_delete_error: "session_cleanup_error",
|
|
178
|
-
kernelSessionId: "browserSessionId",
|
|
179
|
-
kernelDeleteStarted: "sessionCleanupStarted",
|
|
180
|
-
kernelDeleteSucceeded: "sessionCleanupSucceeded",
|
|
181
|
-
kernelDeleteError: "sessionCleanupError",
|
|
182
|
-
kernelProxyId: "proxyId"
|
|
208
|
+
kernel: "browserRuntime"
|
|
183
209
|
};
|
|
184
210
|
SANITIZED_VALUE_KEYS = /error|message/i;
|
|
211
|
+
INTERNAL_DIAGNOSTIC_KEYS = /* @__PURE__ */ new Set([
|
|
212
|
+
"kernel_session_id",
|
|
213
|
+
"browser_session_id",
|
|
214
|
+
"kernelSessionId",
|
|
215
|
+
"browserSessionId",
|
|
216
|
+
"sessionId",
|
|
217
|
+
"kernel_delete_started",
|
|
218
|
+
"session_cleanup_started",
|
|
219
|
+
"kernelDeleteStarted",
|
|
220
|
+
"sessionCleanupStarted",
|
|
221
|
+
"kernel_delete_succeeded",
|
|
222
|
+
"session_cleanup_succeeded",
|
|
223
|
+
"kernelDeleteSucceeded",
|
|
224
|
+
"sessionCleanupSucceeded",
|
|
225
|
+
"kernel_delete_error",
|
|
226
|
+
"session_cleanup_error",
|
|
227
|
+
"kernelDeleteError",
|
|
228
|
+
"sessionCleanupError",
|
|
229
|
+
"kernelProxyId",
|
|
230
|
+
"proxyId",
|
|
231
|
+
"requestedProxyIdSuffix",
|
|
232
|
+
"retrievedProxyIdSuffix",
|
|
233
|
+
"provider",
|
|
234
|
+
"providerName",
|
|
235
|
+
"service",
|
|
236
|
+
"serviceName",
|
|
237
|
+
"vendor",
|
|
238
|
+
"vendorName",
|
|
239
|
+
"organization",
|
|
240
|
+
"organizationId",
|
|
241
|
+
"accountId"
|
|
242
|
+
]);
|
|
185
243
|
VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
|
|
186
244
|
"hosted_url",
|
|
187
245
|
"hostedUrl",
|
|
@@ -3619,6 +3677,8 @@ __export(db_exports, {
|
|
|
3619
3677
|
appendWorkflowArtifacts: () => appendWorkflowArtifacts,
|
|
3620
3678
|
cancelJob: () => cancelJob,
|
|
3621
3679
|
checkRateLimit: () => checkRateLimit,
|
|
3680
|
+
checkpointJob: () => checkpointJob,
|
|
3681
|
+
claimInngestPaaJob: () => claimInngestPaaJob,
|
|
3622
3682
|
claimMonthlyFreeRefresh: () => claimMonthlyFreeRefresh,
|
|
3623
3683
|
claimPendingJob: () => claimPendingJob,
|
|
3624
3684
|
claimPendingKpoJob: () => claimPendingKpoJob,
|
|
@@ -3632,6 +3692,7 @@ __export(db_exports, {
|
|
|
3632
3692
|
countActiveJobsForUser: () => countActiveJobsForUser,
|
|
3633
3693
|
countActiveUsers: () => countActiveUsers,
|
|
3634
3694
|
createJob: () => createJob,
|
|
3695
|
+
createJobWithId: () => createJobWithId,
|
|
3635
3696
|
createKpoJob: () => createKpoJob,
|
|
3636
3697
|
createPasswordResetToken: () => createPasswordResetToken,
|
|
3637
3698
|
createRunningJob: () => createRunningJob,
|
|
@@ -3685,7 +3746,10 @@ __export(db_exports, {
|
|
|
3685
3746
|
listInboxMessages: () => listInboxMessages,
|
|
3686
3747
|
listJobs: () => listJobs,
|
|
3687
3748
|
listKpoJobs: () => listKpoJobs,
|
|
3749
|
+
listPendingInngestPaaJobs: () => listPendingInngestPaaJobs,
|
|
3750
|
+
listRecentTerminalInngestPaaJobs: () => listRecentTerminalInngestPaaJobs,
|
|
3688
3751
|
listRequestEvents: () => listRequestEvents,
|
|
3752
|
+
listStaleRunningInngestPaaJobs: () => listStaleRunningInngestPaaJobs,
|
|
3689
3753
|
listUsers: () => listUsers,
|
|
3690
3754
|
listUsersDueForConnectedAccountBillingReconciliation: () => listUsersDueForConnectedAccountBillingReconciliation,
|
|
3691
3755
|
listWorkflowArtifacts: () => listWorkflowArtifacts,
|
|
@@ -3694,6 +3758,7 @@ __export(db_exports, {
|
|
|
3694
3758
|
logKpoPhaseComplete: () => logKpoPhaseComplete,
|
|
3695
3759
|
logRequestEvent: () => logRequestEvent,
|
|
3696
3760
|
markInboxMessageOpened: () => markInboxMessageOpened,
|
|
3761
|
+
markJobBillingSettled: () => markJobBillingSettled,
|
|
3697
3762
|
markWorkflowRunRunning: () => markWorkflowRunRunning,
|
|
3698
3763
|
markWorkflowScheduleRan: () => markWorkflowScheduleRan,
|
|
3699
3764
|
migrate: () => migrate,
|
|
@@ -4977,11 +5042,15 @@ function deserialize(raw) {
|
|
|
4977
5042
|
}
|
|
4978
5043
|
async function createJob(userId, query, options, callbackUrl) {
|
|
4979
5044
|
const id = (0, import_node_crypto.randomUUID)();
|
|
5045
|
+
await createJobWithId(id, userId, query, options, callbackUrl);
|
|
5046
|
+
return id;
|
|
5047
|
+
}
|
|
5048
|
+
async function createJobWithId(id, userId, query, options, callbackUrl) {
|
|
5049
|
+
if (!/^[a-zA-Z0-9-]{1,100}$/.test(id)) throw new Error("job id is invalid");
|
|
4980
5050
|
await getDb().execute({
|
|
4981
5051
|
sql: "INSERT INTO jobs (id, user_id, query, options, callback_url) VALUES (?, ?, ?, ?, ?)",
|
|
4982
5052
|
args: [id, userId, query, JSON.stringify(options), callbackUrl ?? null]
|
|
4983
5053
|
});
|
|
4984
|
-
return id;
|
|
4985
5054
|
}
|
|
4986
5055
|
async function createRunningJob(userId, query, options) {
|
|
4987
5056
|
const id = (0, import_node_crypto.randomUUID)();
|
|
@@ -5511,7 +5580,13 @@ async function countActiveJobsForUser(userId) {
|
|
|
5511
5580
|
}
|
|
5512
5581
|
async function claimPendingJob() {
|
|
5513
5582
|
const db = getDb();
|
|
5514
|
-
const res = await db.execute(`
|
|
5583
|
+
const res = await db.execute(`
|
|
5584
|
+
SELECT * FROM jobs
|
|
5585
|
+
WHERE status = 'pending'
|
|
5586
|
+
AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') != 'inngest'
|
|
5587
|
+
ORDER BY created_at
|
|
5588
|
+
LIMIT 1
|
|
5589
|
+
`);
|
|
5515
5590
|
const job = res.rows[0] ? rowToRawJob(res.rows[0]) : void 0;
|
|
5516
5591
|
if (!job) return void 0;
|
|
5517
5592
|
const upd = await db.execute({
|
|
@@ -5520,6 +5595,80 @@ async function claimPendingJob() {
|
|
|
5520
5595
|
});
|
|
5521
5596
|
return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
|
|
5522
5597
|
}
|
|
5598
|
+
async function claimInngestPaaJob(id) {
|
|
5599
|
+
const db = getDb();
|
|
5600
|
+
const current = await getJob(id);
|
|
5601
|
+
if (!current || current.options.executionOwner !== "inngest" || current.options.serpOnly === true) return void 0;
|
|
5602
|
+
if (current.status === "running") return void 0;
|
|
5603
|
+
if (current.status === "pending") {
|
|
5604
|
+
const updated = await db.execute({
|
|
5605
|
+
sql: `UPDATE jobs
|
|
5606
|
+
SET status = 'running', started_at = COALESCE(started_at, datetime('now')), error = NULL
|
|
5607
|
+
WHERE id = ? AND status = 'pending'`,
|
|
5608
|
+
args: [id]
|
|
5609
|
+
});
|
|
5610
|
+
if (updated.rowsAffected === 0) return getJob(id);
|
|
5611
|
+
return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
|
|
5612
|
+
}
|
|
5613
|
+
return current;
|
|
5614
|
+
}
|
|
5615
|
+
async function checkpointJob(id, result) {
|
|
5616
|
+
const updated = await getDb().execute({
|
|
5617
|
+
sql: `UPDATE jobs SET result = ? WHERE id = ? AND status = 'running'`,
|
|
5618
|
+
args: [JSON.stringify(result), id]
|
|
5619
|
+
});
|
|
5620
|
+
return updated.rowsAffected === 1;
|
|
5621
|
+
}
|
|
5622
|
+
async function listPendingInngestPaaJobs(limit = 25) {
|
|
5623
|
+
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5624
|
+
const rows = await getDb().execute({
|
|
5625
|
+
sql: `SELECT * FROM jobs
|
|
5626
|
+
WHERE status = 'pending'
|
|
5627
|
+
AND json_extract(options, '$.executionOwner') = 'inngest'
|
|
5628
|
+
AND COALESCE(json_extract(options, '$.serpOnly'), 0) = 0
|
|
5629
|
+
ORDER BY created_at
|
|
5630
|
+
LIMIT ?`,
|
|
5631
|
+
args: [safeLimit]
|
|
5632
|
+
});
|
|
5633
|
+
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
5634
|
+
}
|
|
5635
|
+
async function listRecentTerminalInngestPaaJobs(limit = 25) {
|
|
5636
|
+
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5637
|
+
const rows = await getDb().execute({
|
|
5638
|
+
sql: `SELECT * FROM jobs
|
|
5639
|
+
WHERE status IN ('done', 'failed', 'cancelled')
|
|
5640
|
+
AND json_extract(options, '$.executionOwner') = 'inngest'
|
|
5641
|
+
AND json_extract(options, '$.billingDebitKey') IS NOT NULL
|
|
5642
|
+
AND json_extract(options, '$.billingSettledAt') IS NULL
|
|
5643
|
+
ORDER BY completed_at DESC
|
|
5644
|
+
LIMIT ?`,
|
|
5645
|
+
args: [safeLimit]
|
|
5646
|
+
});
|
|
5647
|
+
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
5648
|
+
}
|
|
5649
|
+
async function listStaleRunningInngestPaaJobs(limit = 25) {
|
|
5650
|
+
const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
|
|
5651
|
+
const rows = await getDb().execute({
|
|
5652
|
+
sql: `SELECT * FROM jobs
|
|
5653
|
+
WHERE status = 'running'
|
|
5654
|
+
AND started_at <= datetime('now', '-10 minutes')
|
|
5655
|
+
AND json_extract(options, '$.executionOwner') = 'inngest'
|
|
5656
|
+
ORDER BY started_at
|
|
5657
|
+
LIMIT ?`,
|
|
5658
|
+
args: [safeLimit]
|
|
5659
|
+
});
|
|
5660
|
+
return rows.rows.map((row) => deserialize(rowToRawJob(row)));
|
|
5661
|
+
}
|
|
5662
|
+
async function markJobBillingSettled(id, billedMc, settledAt = (/* @__PURE__ */ new Date()).toISOString()) {
|
|
5663
|
+
if (!Number.isSafeInteger(billedMc) || billedMc < 0) throw new Error("job billed amount must be a non-negative integer");
|
|
5664
|
+
const updated = await getDb().execute({
|
|
5665
|
+
sql: `UPDATE jobs
|
|
5666
|
+
SET options = json_set(options, '$.billingSettledAt', ?, '$.billedMc', ?)
|
|
5667
|
+
WHERE id = ?`,
|
|
5668
|
+
args: [settledAt, billedMc, id]
|
|
5669
|
+
});
|
|
5670
|
+
return updated.rowsAffected === 1;
|
|
5671
|
+
}
|
|
5523
5672
|
async function completeJob(id, result) {
|
|
5524
5673
|
await getDb().execute({ sql: `UPDATE jobs SET status = 'done', result = ?, completed_at = datetime('now') WHERE id = ?`, args: [JSON.stringify(result), id] });
|
|
5525
5674
|
}
|
|
@@ -15931,15 +16080,15 @@ function formatStructuredError(body, fallback) {
|
|
|
15931
16080
|
return `Insufficient credits. Balance: ${body.balance_credits} credits. This call requires ${body.required_credits} credits. Top up at ${body.topup_url}`;
|
|
15932
16081
|
}
|
|
15933
16082
|
if (body.error === "mcp_request_timeout") {
|
|
15934
|
-
return
|
|
16083
|
+
return publicErrorMessage("mcp_request_timeout");
|
|
15935
16084
|
}
|
|
15936
16085
|
if (typeof body.error_code === "string") {
|
|
15937
|
-
const
|
|
16086
|
+
const code = normalizePublicErrorCode(body.error_code);
|
|
16087
|
+
const message = publicErrorMessage(code);
|
|
15938
16088
|
const retryable = body.retryable === true ? " Retryable: yes." : "";
|
|
15939
|
-
return `${
|
|
16089
|
+
return `${code}: ${message}${retryable}${errorAttemptsSection(body)}`;
|
|
15940
16090
|
}
|
|
15941
|
-
|
|
15942
|
-
return fallback || "Tool error";
|
|
16091
|
+
return publicErrorMessage("service_unavailable");
|
|
15943
16092
|
}
|
|
15944
16093
|
function parseData(raw) {
|
|
15945
16094
|
const first = raw.content.find((b) => b.type === "text");
|
|
@@ -15950,7 +16099,7 @@ function parseData(raw) {
|
|
|
15950
16099
|
const data = parsed.result ?? parsed;
|
|
15951
16100
|
return { data };
|
|
15952
16101
|
} catch {
|
|
15953
|
-
if (raw.isError) return { error:
|
|
16102
|
+
if (raw.isError) return { error: publicErrorMessage("service_unavailable") };
|
|
15954
16103
|
return { error: "Failed to parse tool response" };
|
|
15955
16104
|
}
|
|
15956
16105
|
}
|
|
@@ -15992,10 +16141,9 @@ function debugSection(debug) {
|
|
|
15992
16141
|
const candidates = Array.isArray(locationEvidence?.candidates) ? locationEvidence.candidates.slice(0, 4).map((c) => `${c.city}, ${c.regionCode} (${c.count})`).join(", ") : "";
|
|
15993
16142
|
const lines = [
|
|
15994
16143
|
"\n## Debug",
|
|
15995
|
-
`- Proxy mode: ${request.proxyMode ?? kernel.proxyMode ?? "unknown"}
|
|
15996
|
-
`- Proxy resolution: ${proxyResolution.
|
|
15997
|
-
`- Browser
|
|
15998
|
-
`- Browser IP geo: ${[network.ip, network.city, network.region, network.country].filter(Boolean).join(" \xB7 ") || network.error || "unknown"}`,
|
|
16144
|
+
`- Proxy mode: ${request.proxyMode ?? kernel.proxyMode ?? "unknown"}`,
|
|
16145
|
+
`- Proxy resolution: ${proxyResolution.target ? `${proxyResolution.target.level ?? "city"} ${proxyResolution.target.city}, ${proxyResolution.target.state}` : proxyResolution.error ? publicErrorMessage("service_unavailable") : "unknown"}`,
|
|
16146
|
+
`- Browser IP geo: ${[network.ip, network.city, network.region, network.country].filter(Boolean).join(" \xB7 ") || (network.error ? publicErrorMessage("service_unavailable") : "unknown")}`,
|
|
15999
16147
|
`- Google URL: ${truncate(nav.requestedUrl, 240) || "unknown"}`,
|
|
16000
16148
|
`- Final URL: ${truncate(nav.finalUrl, 240) || "unknown"} \xB7 CAPTCHA: ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 redirected: ${nav.redirected === true ? "yes" : nav.redirected === false ? "no" : "unknown"}`
|
|
16001
16149
|
];
|
|
@@ -16011,7 +16159,6 @@ function errorAttemptsSection(body) {
|
|
|
16011
16159
|
const debug = attempt.debug ?? {};
|
|
16012
16160
|
const browser = debug.browser ?? {};
|
|
16013
16161
|
const kernel = browser.browserRuntime ?? browser.kernel ?? {};
|
|
16014
|
-
const proxyResolution = kernel.proxyResolution ?? {};
|
|
16015
16162
|
const network = browser.networkLocation ?? {
|
|
16016
16163
|
ip: attempt.observedIp ?? attempt.observed_ip,
|
|
16017
16164
|
city: attempt.observedCity ?? attempt.observed_city,
|
|
@@ -16019,10 +16166,7 @@ function errorAttemptsSection(body) {
|
|
|
16019
16166
|
};
|
|
16020
16167
|
const nav = browser.serpNavigation ?? {};
|
|
16021
16168
|
const geo = [network.ip, network.city, network.region].filter(Boolean).join(" / ") || "geo unknown";
|
|
16022
|
-
|
|
16023
|
-
const cleanupSucceeded2 = attempt.session_cleanup_succeeded ?? attempt.kernel_delete_succeeded;
|
|
16024
|
-
const proxySource = proxyResolution.source ?? attempt.proxyResolutionSource;
|
|
16025
|
-
return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 session ${sessionId} \xB7 proxy ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"}${proxySource ? `/${proxySource}` : ""} \xB7 ${geo} \xB7 CAPTCHA ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 cleanup ${cleanupSucceeded2 === true ? "yes" : cleanupSucceeded2 === false ? "no" : "unknown"}`;
|
|
16169
|
+
return `- Attempt ${attempt.attempt_number ?? attempt.attemptNumber ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 proxy mode ${debug.request?.proxyMode ?? kernel.proxyMode ?? attempt.proxyMode ?? "unknown"} \xB7 ${geo} \xB7 verification challenge ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"}`;
|
|
16026
16170
|
});
|
|
16027
16171
|
return `
|
|
16028
16172
|
|
|
@@ -18533,6 +18677,9 @@ var init_selectors = __esm({
|
|
|
18533
18677
|
itemDataInitQ: "data-initq",
|
|
18534
18678
|
itemQuestionEl: ".JlqpRe",
|
|
18535
18679
|
answerContainer: ".bCOlv, .hgKElc, .wDYxhc, .LGOjhe, .fo7IQd, .fmW3u",
|
|
18680
|
+
aiAnswerBody: ".pWvJNd",
|
|
18681
|
+
aiCitation: "a.PMDqCb[href]",
|
|
18682
|
+
aiSourceCard: "a.vIWmYe[href], a.Zbfntb[href]",
|
|
18536
18683
|
sourceTitle: "h3",
|
|
18537
18684
|
sourceSite: ".VuuXrf",
|
|
18538
18685
|
sourceCite: "cite",
|
|
@@ -23182,6 +23329,968 @@ var init_directory_workflow2 = __esm({
|
|
|
23182
23329
|
}
|
|
23183
23330
|
});
|
|
23184
23331
|
|
|
23332
|
+
// src/api/paa-capture-artifacts.ts
|
|
23333
|
+
function paaCaptureArtifactPolicy() {
|
|
23334
|
+
return {
|
|
23335
|
+
prefix: PAA_CAPTURE_ARTIFACT_PREFIX,
|
|
23336
|
+
artifactTtlMs: 7 * DAY_MS,
|
|
23337
|
+
downloadTtlMs: 15 * 60 * 1e3,
|
|
23338
|
+
token: process.env.PAA_CAPTURE_BLOB_READ_WRITE_TOKEN ?? process.env.BLOB_READ_WRITE_TOKEN ?? null
|
|
23339
|
+
};
|
|
23340
|
+
}
|
|
23341
|
+
async function createPaaRawDomArtifact(input) {
|
|
23342
|
+
return createPrivateArtifact({
|
|
23343
|
+
policy: paaCaptureArtifactPolicy(),
|
|
23344
|
+
ownerId: String(input.userId),
|
|
23345
|
+
artifactKey: `${input.jobId}-attempt-${input.attempt}.html.gz`,
|
|
23346
|
+
createdAt: input.createdAt ?? /* @__PURE__ */ new Date(),
|
|
23347
|
+
filename: `${input.jobId}-serp.html.gz`,
|
|
23348
|
+
contentType: "application/gzip",
|
|
23349
|
+
content: input.gzip
|
|
23350
|
+
});
|
|
23351
|
+
}
|
|
23352
|
+
async function tryCreatePaaRawDomArtifact(input) {
|
|
23353
|
+
try {
|
|
23354
|
+
return {
|
|
23355
|
+
artifact: await createPaaRawDomArtifact(input),
|
|
23356
|
+
status: "stored"
|
|
23357
|
+
};
|
|
23358
|
+
} catch (error) {
|
|
23359
|
+
console.error("[paa-capture-artifact] private DOM preservation unavailable:", error instanceof Error ? error.message : String(error));
|
|
23360
|
+
return { artifact: null, status: "unavailable" };
|
|
23361
|
+
}
|
|
23362
|
+
}
|
|
23363
|
+
var DAY_MS, PAA_CAPTURE_ARTIFACT_PREFIX;
|
|
23364
|
+
var init_paa_capture_artifacts = __esm({
|
|
23365
|
+
"src/api/paa-capture-artifacts.ts"() {
|
|
23366
|
+
"use strict";
|
|
23367
|
+
init_private_artifacts();
|
|
23368
|
+
DAY_MS = 24 * 60 * 60 * 1e3;
|
|
23369
|
+
PAA_CAPTURE_ARTIFACT_PREFIX = "paa-captures";
|
|
23370
|
+
}
|
|
23371
|
+
});
|
|
23372
|
+
|
|
23373
|
+
// src/api/paa-harvest-settlement.ts
|
|
23374
|
+
function capturedQuestionCount(result) {
|
|
23375
|
+
if (!result || typeof result !== "object") return 0;
|
|
23376
|
+
const value = result;
|
|
23377
|
+
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
23378
|
+
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
23379
|
+
}
|
|
23380
|
+
function finalCost(result, heldMc) {
|
|
23381
|
+
const questions = capturedQuestionCount(result);
|
|
23382
|
+
if (questions <= 0) return 0;
|
|
23383
|
+
return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
|
|
23384
|
+
}
|
|
23385
|
+
async function settlePaaHarvestJob(jobOrId) {
|
|
23386
|
+
const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
|
|
23387
|
+
if (!job) return false;
|
|
23388
|
+
const options = job.options;
|
|
23389
|
+
const debitKey2 = options.billingDebitKey;
|
|
23390
|
+
const heldMc = Number(options.billingHoldMc ?? 0);
|
|
23391
|
+
if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
|
|
23392
|
+
const settlement = await settleDebitMcIdempotent(
|
|
23393
|
+
job.user_id,
|
|
23394
|
+
debitKey2,
|
|
23395
|
+
finalCost(job.result, heldMc),
|
|
23396
|
+
LedgerOperation.PAA_REFUND,
|
|
23397
|
+
"durable PAA harvest settlement",
|
|
23398
|
+
"paa_harvest"
|
|
23399
|
+
);
|
|
23400
|
+
await markJobBillingSettled(job.id, settlement.final_amount_mc);
|
|
23401
|
+
return true;
|
|
23402
|
+
}
|
|
23403
|
+
var init_paa_harvest_settlement = __esm({
|
|
23404
|
+
"src/api/paa-harvest-settlement.ts"() {
|
|
23405
|
+
"use strict";
|
|
23406
|
+
init_db();
|
|
23407
|
+
init_rates();
|
|
23408
|
+
}
|
|
23409
|
+
});
|
|
23410
|
+
|
|
23411
|
+
// src/api/webhook.ts
|
|
23412
|
+
async function deliverWebhook(url, payload, retries = 3) {
|
|
23413
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
23414
|
+
try {
|
|
23415
|
+
const res = await fetch(url, {
|
|
23416
|
+
method: "POST",
|
|
23417
|
+
headers: { "content-type": "application/json" },
|
|
23418
|
+
body: JSON.stringify(payload),
|
|
23419
|
+
signal: AbortSignal.timeout(1e4)
|
|
23420
|
+
});
|
|
23421
|
+
if (res.ok) return;
|
|
23422
|
+
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
23423
|
+
} catch (err) {
|
|
23424
|
+
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
23425
|
+
}
|
|
23426
|
+
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
23427
|
+
}
|
|
23428
|
+
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
23429
|
+
}
|
|
23430
|
+
var init_webhook = __esm({
|
|
23431
|
+
"src/api/webhook.ts"() {
|
|
23432
|
+
"use strict";
|
|
23433
|
+
}
|
|
23434
|
+
});
|
|
23435
|
+
|
|
23436
|
+
// src/paa/durable-capture.ts
|
|
23437
|
+
function remainingSeconds(deadlineMs, ceiling) {
|
|
23438
|
+
return Math.max(1, Math.min(ceiling, Math.floor((deadlineMs - Date.now()) / 1e3)));
|
|
23439
|
+
}
|
|
23440
|
+
async function execute(kernel, sessionId, code, deadlineMs, ceilingSeconds) {
|
|
23441
|
+
if (Date.now() >= deadlineMs) throw new Error("paa_work_deadline_exhausted");
|
|
23442
|
+
const response = await kernel.browsers.playwright.execute(sessionId, {
|
|
23443
|
+
code,
|
|
23444
|
+
timeout_sec: remainingSeconds(deadlineMs, ceilingSeconds)
|
|
23445
|
+
});
|
|
23446
|
+
if (!response.success || response.result === null || response.result === void 0) {
|
|
23447
|
+
throw new Error(response.error || response.stderr || "co-located browser execution failed");
|
|
23448
|
+
}
|
|
23449
|
+
return response.result;
|
|
23450
|
+
}
|
|
23451
|
+
function navigationCode(searchUrl) {
|
|
23452
|
+
return `
|
|
23453
|
+
const wait = ms => page.waitForTimeout(ms);
|
|
23454
|
+
const block = async () => {
|
|
23455
|
+
if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
|
|
23456
|
+
if (await page.locator(${JSON.stringify(PAASelectors.captchaMarker)}).count().catch(() => 0)) return 'captcha';
|
|
23457
|
+
const title = await page.title().catch(() => '');
|
|
23458
|
+
const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
|
|
23459
|
+
if (/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(text)) return 'captcha';
|
|
23460
|
+
if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(title + '\\n' + text.slice(0, 3000))) return 'soft_block';
|
|
23461
|
+
return null;
|
|
23462
|
+
};
|
|
23463
|
+
await page.goto(${JSON.stringify(searchUrl)}, { waitUntil: 'domcontentloaded', timeout: 45000 });
|
|
23464
|
+
let captchaObserved = false;
|
|
23465
|
+
let solverCleared = false;
|
|
23466
|
+
let blocked = await block();
|
|
23467
|
+
const solverDeadline = Date.now() + ${PAA_SOLVER_BUDGET_MS};
|
|
23468
|
+
while (blocked === 'captcha' && Date.now() < solverDeadline) {
|
|
23469
|
+
captchaObserved = true;
|
|
23470
|
+
await wait(4000 + Math.floor(Math.random() * 1500));
|
|
23471
|
+
blocked = await block();
|
|
23472
|
+
if (!blocked) solverCleared = true;
|
|
23473
|
+
}
|
|
23474
|
+
let count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
|
|
23475
|
+
for (let scan = 0; !blocked && count === 0 && scan < 7; scan += 1) {
|
|
23476
|
+
await page.mouse.wheel(0, 520 + Math.floor(Math.random() * 420));
|
|
23477
|
+
await wait(350 + Math.floor(Math.random() * 500));
|
|
23478
|
+
count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
|
|
23479
|
+
}
|
|
23480
|
+
await page.evaluate((itemSelector) => {
|
|
23481
|
+
const state = window;
|
|
23482
|
+
if (state.__mcpTrustedListenerInstalled) return;
|
|
23483
|
+
document.addEventListener('click', event => {
|
|
23484
|
+
if (event.target instanceof Element && event.target.closest(itemSelector)) {
|
|
23485
|
+
state.__mcpLastPaaClickTrusted = event.isTrusted;
|
|
23486
|
+
}
|
|
23487
|
+
}, { capture: true });
|
|
23488
|
+
state.__mcpTrustedListenerInstalled = true;
|
|
23489
|
+
}, ${JSON.stringify(PAASelectors.item)}).catch(() => {});
|
|
23490
|
+
return {
|
|
23491
|
+
outcome: blocked || (count > 0 ? 'ready' : 'no_paa'),
|
|
23492
|
+
finalUrl: page.url(),
|
|
23493
|
+
title: await page.title().catch(() => ''),
|
|
23494
|
+
captchaObserved,
|
|
23495
|
+
solverCleared,
|
|
23496
|
+
environment: await page.evaluate(() => ({
|
|
23497
|
+
userAgent: navigator.userAgent,
|
|
23498
|
+
language: navigator.language,
|
|
23499
|
+
platform: navigator.platform,
|
|
23500
|
+
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
23501
|
+
innerWidth: window.innerWidth,
|
|
23502
|
+
innerHeight: window.innerHeight,
|
|
23503
|
+
outerWidth: window.outerWidth,
|
|
23504
|
+
outerHeight: window.outerHeight,
|
|
23505
|
+
screenWidth: window.screen.width,
|
|
23506
|
+
screenHeight: window.screen.height,
|
|
23507
|
+
devicePixelRatio: window.devicePixelRatio,
|
|
23508
|
+
})).catch(() => null),
|
|
23509
|
+
};
|
|
23510
|
+
`;
|
|
23511
|
+
}
|
|
23512
|
+
function recordReaderSource() {
|
|
23513
|
+
return `
|
|
23514
|
+
const clean = value => (value || '').replace(/\\u00a0/g, ' ').replace(/[ \\t]+/g, ' ').replace(/\\n{3,}/g, '\\n\\n').trim();
|
|
23515
|
+
const externalHref = anchor => {
|
|
23516
|
+
const href = anchor && anchor.href || '';
|
|
23517
|
+
if (!/^https?:\\/\\//i.test(href)) return '';
|
|
23518
|
+
try {
|
|
23519
|
+
const url = new URL(href);
|
|
23520
|
+
if (/(^|\\.)google(usercontent)?\\.[a-z.]+$/i.test(url.hostname)) {
|
|
23521
|
+
const redirected = url.searchParams.get('q') || url.searchParams.get('url');
|
|
23522
|
+
return redirected && redirected.startsWith('http') ? redirected : '';
|
|
23523
|
+
}
|
|
23524
|
+
} catch {}
|
|
23525
|
+
return href;
|
|
23526
|
+
};
|
|
23527
|
+
const sourceLabel = value => clean(value)
|
|
23528
|
+
.replace(/\\s*\\(\\+\\d+\\)\\s*-\\s*View related links.*$/i, '')
|
|
23529
|
+
.replace(/\\s*-\\s*Opens in new tab.*$/i, '').trim();
|
|
23530
|
+
const readRecords = () => Array.from(document.querySelectorAll(SEL.item)).map(item => {
|
|
23531
|
+
const question = clean(item.getAttribute(SEL.itemDataQ)
|
|
23532
|
+
|| item.getAttribute(SEL.itemDataInitQ)
|
|
23533
|
+
|| item.querySelector(SEL.itemQuestionEl)?.innerText);
|
|
23534
|
+
const expanded = item.classList.contains(SEL.expandedClass)
|
|
23535
|
+
|| item.querySelector(SEL.clickTarget)?.getAttribute('aria-expanded') === 'true';
|
|
23536
|
+
const container = expanded ? item.querySelector(SEL.answerContainer) : null;
|
|
23537
|
+
const answerElement = container?.querySelector(SEL.aiAnswerBody) || container;
|
|
23538
|
+
const anchors = expanded ? Array.from(answerElement?.querySelectorAll(
|
|
23539
|
+
SEL.aiCitation + ',' + SEL.aiSourceCard + ',a[href]'
|
|
23540
|
+
) || []) : [];
|
|
23541
|
+
const byUrl = new Map();
|
|
23542
|
+
for (const anchor of anchors) {
|
|
23543
|
+
const url = externalHref(anchor);
|
|
23544
|
+
if (!url || byUrl.has(url)) continue;
|
|
23545
|
+
let site = '';
|
|
23546
|
+
try { site = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
|
|
23547
|
+
byUrl.set(url, { title: sourceLabel(anchor.getAttribute('aria-label')) || clean(anchor.textContent), site, url });
|
|
23548
|
+
}
|
|
23549
|
+
const sources = Array.from(byUrl.values());
|
|
23550
|
+
let sourceTitle = expanded ? clean(item.querySelector(SEL.sourceTitle)?.textContent) : '';
|
|
23551
|
+
let sourceSite = expanded ? clean(item.querySelector(SEL.sourceSite)?.textContent) : '';
|
|
23552
|
+
let sourceCite = expanded ? externalHref(item.querySelector(SEL.sourceCite)) : '';
|
|
23553
|
+
const primary = sources.find(source => source.title.length > 2) || sources[0];
|
|
23554
|
+
if (!sourceCite && primary) {
|
|
23555
|
+
sourceTitle ||= primary.title;
|
|
23556
|
+
sourceSite ||= primary.site;
|
|
23557
|
+
sourceCite = primary.url;
|
|
23558
|
+
}
|
|
23559
|
+
return {
|
|
23560
|
+
question,
|
|
23561
|
+
answer: clean(answerElement?.innerText || answerElement?.textContent),
|
|
23562
|
+
sourceTitle,
|
|
23563
|
+
sourceSite,
|
|
23564
|
+
sourceCite,
|
|
23565
|
+
sources,
|
|
23566
|
+
depth: 1,
|
|
23567
|
+
parentQuestion: null,
|
|
23568
|
+
};
|
|
23569
|
+
}).filter(record => record.question);
|
|
23570
|
+
`;
|
|
23571
|
+
}
|
|
23572
|
+
function expansionCode(clickedQuestions) {
|
|
23573
|
+
return `
|
|
23574
|
+
const SEL = ${JSON.stringify(PAASelectors)};
|
|
23575
|
+
${recordReaderSource()}
|
|
23576
|
+
const deliveryBlock = async () => {
|
|
23577
|
+
if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
|
|
23578
|
+
if (await page.locator(SEL.captchaMarker).count().catch(() => 0)) return 'captcha';
|
|
23579
|
+
const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
|
|
23580
|
+
if (/recaptcha|unusual traffic|are you a robot|about this page/i.test(text)) return 'captcha';
|
|
23581
|
+
if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(text.slice(0, 3000))) return 'soft_block';
|
|
23582
|
+
return null;
|
|
23583
|
+
};
|
|
23584
|
+
const blocked = await deliveryBlock();
|
|
23585
|
+
const records = await page.evaluate(({ SEL }) => {
|
|
23586
|
+
${recordReaderSource()}
|
|
23587
|
+
return readRecords();
|
|
23588
|
+
}, { SEL });
|
|
23589
|
+
if (blocked) return {
|
|
23590
|
+
outcome: blocked, targetQuestion: null, records, newQuestions: [],
|
|
23591
|
+
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
23592
|
+
interactionTrusted: null, clickError: null,
|
|
23593
|
+
};
|
|
23594
|
+
const clicked = new Set(${JSON.stringify(clickedQuestions)});
|
|
23595
|
+
const targetQuestion = records.map(record => record.question).find(question => !clicked.has(question)) || null;
|
|
23596
|
+
if (!targetQuestion) return {
|
|
23597
|
+
outcome: 'exhausted', targetQuestion: null, records, newQuestions: [],
|
|
23598
|
+
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
23599
|
+
interactionTrusted: null, clickError: null,
|
|
23600
|
+
};
|
|
23601
|
+
const items = page.locator(SEL.item);
|
|
23602
|
+
let targetIndex = -1;
|
|
23603
|
+
for (let index = 0, count = await items.count().catch(() => 0); index < count; index += 1) {
|
|
23604
|
+
const question = await items.nth(index).evaluate((item, SEL) =>
|
|
23605
|
+
item.getAttribute(SEL.itemDataQ)
|
|
23606
|
+
|| item.getAttribute(SEL.itemDataInitQ)
|
|
23607
|
+
|| item.querySelector(SEL.itemQuestionEl)?.innerText?.trim()
|
|
23608
|
+
|| '', SEL).catch(() => '');
|
|
23609
|
+
if (question.replace(/\\s+/g, ' ').trim() === targetQuestion) { targetIndex = index; break; }
|
|
23610
|
+
}
|
|
23611
|
+
if (targetIndex < 0) return {
|
|
23612
|
+
outcome: 'exhausted', targetQuestion, records, newQuestions: [],
|
|
23613
|
+
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
23614
|
+
interactionTrusted: null, clickError: 'question disappeared before click',
|
|
23615
|
+
};
|
|
23616
|
+
const beforeQuestions = new Set(records.map(record => record.question));
|
|
23617
|
+
const item = items.nth(targetIndex);
|
|
23618
|
+
await item.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => {});
|
|
23619
|
+
await page.waitForTimeout(180 + Math.floor(Math.random() * 420));
|
|
23620
|
+
await page.evaluate(() => { window.__mcpLastPaaClickTrusted = null; }).catch(() => {});
|
|
23621
|
+
let clickError = null;
|
|
23622
|
+
try {
|
|
23623
|
+
await item.locator(SEL.clickTarget).first().click({
|
|
23624
|
+
timeout: 6000,
|
|
23625
|
+
delay: 55 + Math.floor(Math.random() * 120),
|
|
23626
|
+
});
|
|
23627
|
+
} catch (error) {
|
|
23628
|
+
clickError = String(error && error.message || error).slice(0, 240);
|
|
23629
|
+
}
|
|
23630
|
+
const interactionTrusted = await page.evaluate(() => window.__mcpLastPaaClickTrusted ?? null).catch(() => null);
|
|
23631
|
+
await page.waitForTimeout(650 + Math.floor(Math.random() * 750));
|
|
23632
|
+
const afterRecords = await page.evaluate(({ SEL }) => {
|
|
23633
|
+
${recordReaderSource()}
|
|
23634
|
+
return readRecords();
|
|
23635
|
+
}, { SEL });
|
|
23636
|
+
return {
|
|
23637
|
+
outcome: clickError ? 'exhausted' : 'clicked',
|
|
23638
|
+
targetQuestion,
|
|
23639
|
+
records: afterRecords,
|
|
23640
|
+
newQuestions: afterRecords.map(record => record.question).filter(question => !beforeQuestions.has(question)),
|
|
23641
|
+
finalUrl: page.url(),
|
|
23642
|
+
title: await page.title().catch(() => ''),
|
|
23643
|
+
interactionTrusted,
|
|
23644
|
+
clickError,
|
|
23645
|
+
};
|
|
23646
|
+
`;
|
|
23647
|
+
}
|
|
23648
|
+
function finalCaptureCode(maxQuestions, parentByQuestion) {
|
|
23649
|
+
const surfaces = {
|
|
23650
|
+
aiOverview: AIOverviewSelectors,
|
|
23651
|
+
aiMode: AIModeSelectors,
|
|
23652
|
+
organic: OrganicSelectors,
|
|
23653
|
+
localPack: LocalPackSelectors,
|
|
23654
|
+
video: VideoSelectors,
|
|
23655
|
+
forum: ForumSelectors,
|
|
23656
|
+
whatPeopleSaying: WhatPeopleSayingSelectors
|
|
23657
|
+
};
|
|
23658
|
+
return `
|
|
23659
|
+
const SEL = ${JSON.stringify(PAASelectors)};
|
|
23660
|
+
const SURF = ${JSON.stringify(surfaces)};
|
|
23661
|
+
const PARENTS = ${JSON.stringify(parentByQuestion)};
|
|
23662
|
+
${recordReaderSource()}
|
|
23663
|
+
const capture = await page.evaluate(({ SEL, SURF, PARENTS, maxQuestions }) => {
|
|
23664
|
+
${recordReaderSource()}
|
|
23665
|
+
const unique = (items, key) => {
|
|
23666
|
+
const seen = new Set();
|
|
23667
|
+
return items.filter(item => { const value = key(item); if (!value || seen.has(value)) return false; seen.add(value); return true; });
|
|
23668
|
+
};
|
|
23669
|
+
const paa = readRecords().map(record => {
|
|
23670
|
+
const parentQuestion = PARENTS[record.question] || null;
|
|
23671
|
+
let depth = 1, parent = parentQuestion;
|
|
23672
|
+
const seen = new Set();
|
|
23673
|
+
while (parent && !seen.has(parent)) { seen.add(parent); depth += 1; parent = PARENTS[parent] || null; }
|
|
23674
|
+
return { ...record, depth, parentQuestion };
|
|
23675
|
+
}).slice(0, maxQuestions);
|
|
23676
|
+
const organicResults = [];
|
|
23677
|
+
document.querySelectorAll(SURF.organic.result).forEach(card => {
|
|
23678
|
+
const titleElement = card.querySelector(SURF.organic.title);
|
|
23679
|
+
const title = clean(titleElement?.textContent);
|
|
23680
|
+
const link = titleElement?.closest('a');
|
|
23681
|
+
const url = externalHref(link);
|
|
23682
|
+
if (!title || !url) return;
|
|
23683
|
+
let domain = '';
|
|
23684
|
+
try { domain = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
|
|
23685
|
+
const rating = card.querySelector(SURF.organic.ratingWrap);
|
|
23686
|
+
organicResults.push({
|
|
23687
|
+
position: organicResults.length + 1, title, url, domain,
|
|
23688
|
+
cite: clean(card.querySelector(SURF.organic.cite)?.textContent) || null,
|
|
23689
|
+
snippet: clean(card.querySelector(SURF.organic.snippet)?.textContent) || null,
|
|
23690
|
+
isRedditStyle: Boolean(card.querySelector(SURF.organic.redditCite)),
|
|
23691
|
+
inlineRating: rating ? {
|
|
23692
|
+
value: clean(rating.querySelector(SURF.organic.ratingValue)?.textContent),
|
|
23693
|
+
count: clean(rating.querySelector(SURF.organic.reviewCount)?.textContent),
|
|
23694
|
+
} : null,
|
|
23695
|
+
});
|
|
23696
|
+
});
|
|
23697
|
+
const localPack = [];
|
|
23698
|
+
document.querySelectorAll(SURF.localPack.card).forEach(card => {
|
|
23699
|
+
const name = clean(card.querySelector(SURF.localPack.name)?.textContent);
|
|
23700
|
+
if (!name) return;
|
|
23701
|
+
const links = Array.from(card.querySelectorAll('a[href]'));
|
|
23702
|
+
let cid = card.querySelector('a[data-cid]')?.getAttribute('data-cid') || null;
|
|
23703
|
+
for (const link of links) {
|
|
23704
|
+
if (cid) break;
|
|
23705
|
+
const match = link.href.match(/[?&]cid=(\\d+)/);
|
|
23706
|
+
if (match) cid = match[1];
|
|
23707
|
+
}
|
|
23708
|
+
localPack.push({
|
|
23709
|
+
position: localPack.length + 1, name, cid,
|
|
23710
|
+
rating: clean(card.querySelector(SURF.localPack.ratingValue)?.textContent) || null,
|
|
23711
|
+
reviewCount: clean(card.querySelector(SURF.localPack.reviewCount)?.textContent).replace(/[()]/g, '') || null,
|
|
23712
|
+
metadata: [], websiteUrl: links.map(externalHref).find(Boolean) || null,
|
|
23713
|
+
directionsUrl: links.find(link => link.href.includes('google.com/maps'))?.href || null,
|
|
23714
|
+
});
|
|
23715
|
+
});
|
|
23716
|
+
const videos = unique(Array.from(document.querySelectorAll(SURF.video.item)).map(link => {
|
|
23717
|
+
const url = externalHref(link);
|
|
23718
|
+
const raw = clean(link.textContent);
|
|
23719
|
+
const platform = /youtu/i.test(url) ? 'YouTube' : /tiktok/i.test(url) ? 'TikTok' : /instagram/i.test(url) ? 'Instagram' : /facebook/i.test(url) ? 'Facebook' : '';
|
|
23720
|
+
return { type: 'video', title: raw, channel: '', platform, duration: raw.match(/\\b\\d{1,2}:\\d{2}\\b/)?.[0] || '', url };
|
|
23721
|
+
}).filter(item => item.title && item.url), item => item.url);
|
|
23722
|
+
const forums = unique(Array.from(document.querySelectorAll(SURF.forum.item)).map(link => ({
|
|
23723
|
+
title: clean(link.querySelector(SURF.forum.title)?.textContent),
|
|
23724
|
+
source: clean(link.querySelector(SURF.forum.source)?.textContent),
|
|
23725
|
+
url: externalHref(link),
|
|
23726
|
+
})).filter(item => item.title && item.url), item => item.url);
|
|
23727
|
+
const socialCards = Array.from(document.querySelectorAll(SURF.whatPeopleSaying.card));
|
|
23728
|
+
const whatPeopleSaying = unique(socialCards.map(card => {
|
|
23729
|
+
const link = card.querySelector(SURF.whatPeopleSaying.cardLink);
|
|
23730
|
+
const url = externalHref(link);
|
|
23731
|
+
const title = clean(card.querySelector(SURF.whatPeopleSaying.titleH1)?.textContent || card.querySelector(SURF.whatPeopleSaying.titleDiv)?.textContent);
|
|
23732
|
+
const source = clean(card.querySelector(SURF.whatPeopleSaying.ytChannel)?.textContent || card.querySelector(SURF.whatPeopleSaying.source)?.textContent);
|
|
23733
|
+
const platform = clean(card.querySelector(SURF.whatPeopleSaying.platformBadge)?.textContent) || (/youtube/i.test(url) ? 'YouTube' : '');
|
|
23734
|
+
const identity = (platform + source + url).toLowerCase();
|
|
23735
|
+
const type = identity.includes('reddit') ? 'reddit' : identity.includes('facebook') ? 'facebook' : identity.includes('instagram') ? 'instagram' : identity.includes('tiktok') ? 'tiktok' : identity.includes('youtube') ? 'youtube' : 'unknown';
|
|
23736
|
+
return { type, title, url, source, platform, popularComment: null, engagement: '', date: '', duration: null, authorNote: null };
|
|
23737
|
+
}).filter(item => item.title || item.url), item => item.url || item.type + ':' + item.title);
|
|
23738
|
+
const aioRoot = document.querySelector(SURF.aiOverview.root) || document.querySelector(SURF.aiOverview.legacyRoot);
|
|
23739
|
+
const aimRoot = document.querySelector(SURF.aiMode.root);
|
|
23740
|
+
const citations = root => unique(Array.from(root?.querySelectorAll('a[href]') || []).map(link => ({
|
|
23741
|
+
text: clean(link.textContent || link.getAttribute('aria-label')), href: externalHref(link),
|
|
23742
|
+
})).filter(item => item.href), item => item.href);
|
|
23743
|
+
const aiOverview = { detected: Boolean(aioRoot), text: aioRoot ? clean(aioRoot.innerText || aioRoot.textContent) : null, citations: citations(aioRoot), expanded: true, fullyExpanded: true, sections: [] };
|
|
23744
|
+
const aiMode = { detected: Boolean(aimRoot), text: aimRoot ? clean(aimRoot.innerText || aimRoot.textContent) : null, citations: citations(aimRoot) };
|
|
23745
|
+
const html = document.documentElement.outerHTML;
|
|
23746
|
+
const kgIds = unique(Array.from(html.matchAll(/\\/g\\/[a-zA-Z0-9_-]{5,20}/g)).map(match => match[0]), value => value);
|
|
23747
|
+
const cids = unique([
|
|
23748
|
+
...localPack.map(item => item.cid).filter(Boolean),
|
|
23749
|
+
...Array.from(html.matchAll(/[?&]cid=(\\d+)/g)).map(match => match[1]),
|
|
23750
|
+
], value => value);
|
|
23751
|
+
const gcids = unique(Array.from(html.matchAll(/gcid:[a-zA-Z0-9_]+/g)).map(match => match[0]), value => value);
|
|
23752
|
+
const entities = localPack.filter(item => item.cid).map(item => ({ name: item.name, kgId: null, cid: item.cid, gcid: null }));
|
|
23753
|
+
const headings = unique(Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"]')).map(node => clean(node.textContent)).filter(text => text.length > 1 && text.length < 160), value => value);
|
|
23754
|
+
const relatedSearches = unique(Array.from(document.querySelectorAll('a[href*="/search?"]')).map(link => ({
|
|
23755
|
+
text: clean(link.textContent), url: link.href,
|
|
23756
|
+
})).filter(item => item.text && /related/i.test(item.url + ' ' + item.text)), item => item.url);
|
|
23757
|
+
const counts = {
|
|
23758
|
+
peopleAlsoAsk: paa.length, organic: organicResults.length, localPack: localPack.length,
|
|
23759
|
+
videos: videos.length, discussions: forums.length, whatPeopleSaying: whatPeopleSaying.length,
|
|
23760
|
+
aiOverview: aiOverview.detected ? 1 : 0, aiMode: aiMode.detected ? 1 : 0,
|
|
23761
|
+
relatedSearches: relatedSearches.length,
|
|
23762
|
+
};
|
|
23763
|
+
return {
|
|
23764
|
+
paa, organicResults: unique(organicResults, item => item.url), localPack: unique(localPack, item => item.cid || item.name),
|
|
23765
|
+
videos, forums, whatPeopleSaying, aiOverview, aiMode,
|
|
23766
|
+
surface: window.google?.sn === 'aim' ? 'aim' : window.google?.sn === 'web' ? 'web' : 'unknown',
|
|
23767
|
+
entityIds: { entities, kgIds, cids, gcids },
|
|
23768
|
+
serpFeatures: { sectionHeadings: headings, detected: Object.fromEntries(Object.entries(counts).map(([key, count]) => [key, count > 0])), counts },
|
|
23769
|
+
relatedSearches,
|
|
23770
|
+
};
|
|
23771
|
+
}, { SEL, SURF, PARENTS, maxQuestions: ${maxQuestions} });
|
|
23772
|
+
return capture;
|
|
23773
|
+
`;
|
|
23774
|
+
}
|
|
23775
|
+
function rawChunkCode(offset, length) {
|
|
23776
|
+
return `return await page.evaluate(({ offset, length }) => ({ offset, chunk: window.__mcpPaaRawDom.slice(offset, offset + length) }), { offset: ${offset}, length: ${length} });`;
|
|
23777
|
+
}
|
|
23778
|
+
function buildPublicResult(input) {
|
|
23779
|
+
const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
23780
|
+
const selected = input.records;
|
|
23781
|
+
const flat = selected.map((record) => ({
|
|
23782
|
+
seed_query: input.query,
|
|
23783
|
+
question: record.question,
|
|
23784
|
+
answer: record.answer,
|
|
23785
|
+
source_title: record.sourceTitle,
|
|
23786
|
+
source_site: record.sourceSite,
|
|
23787
|
+
source_cite: record.sourceCite,
|
|
23788
|
+
depth: record.depth,
|
|
23789
|
+
parent_question: record.parentQuestion ?? "",
|
|
23790
|
+
extracted_at: extractedAt
|
|
23791
|
+
}));
|
|
23792
|
+
const tree = selected.map((record) => ({
|
|
23793
|
+
question: record.question,
|
|
23794
|
+
answer: record.answer || null,
|
|
23795
|
+
sourceTitle: record.sourceTitle || null,
|
|
23796
|
+
sourceSite: record.sourceSite || null,
|
|
23797
|
+
sourceCite: record.sourceCite || null,
|
|
23798
|
+
depth: record.depth,
|
|
23799
|
+
parentQuestion: record.parentQuestion,
|
|
23800
|
+
children: []
|
|
23801
|
+
}));
|
|
23802
|
+
return {
|
|
23803
|
+
seed: input.query,
|
|
23804
|
+
location: input.location,
|
|
23805
|
+
extractedAt,
|
|
23806
|
+
diagnostics: {
|
|
23807
|
+
completionStatus: selected.length > 0 ? "paa_found" : "no_paa",
|
|
23808
|
+
problem: null,
|
|
23809
|
+
resultQuality: input.partial ? "partial" : "complete",
|
|
23810
|
+
degradedResult: input.partial,
|
|
23811
|
+
retryRecommended: false,
|
|
23812
|
+
completeness: {
|
|
23813
|
+
paaWithoutAnswer: selected.filter((record) => !record.answer).length,
|
|
23814
|
+
paaWithoutSource: selected.filter((record) => !record.sourceCite).length,
|
|
23815
|
+
paaAnswersRecovered: 0,
|
|
23816
|
+
aioShareCaptured: null
|
|
23817
|
+
}
|
|
23818
|
+
},
|
|
23819
|
+
totalQuestions: selected.length,
|
|
23820
|
+
surface: input.capture.surface,
|
|
23821
|
+
aiOverview: input.capture.aiOverview,
|
|
23822
|
+
aiMode: input.capture.aiMode,
|
|
23823
|
+
whatPeopleSaying: input.capture.whatPeopleSaying,
|
|
23824
|
+
tree,
|
|
23825
|
+
flat,
|
|
23826
|
+
videos: input.capture.videos,
|
|
23827
|
+
forums: input.capture.forums,
|
|
23828
|
+
organicResults: input.capture.organicResults,
|
|
23829
|
+
localPack: input.capture.localPack,
|
|
23830
|
+
entityIds: input.capture.entityIds,
|
|
23831
|
+
serpFeatures: input.capture.serpFeatures,
|
|
23832
|
+
relatedSearches: input.capture.relatedSearches,
|
|
23833
|
+
stats: {
|
|
23834
|
+
seed: input.query,
|
|
23835
|
+
totalQuestions: selected.length,
|
|
23836
|
+
maxDepthReached: selected.reduce((max, item) => Math.max(max, item.depth), 0),
|
|
23837
|
+
durationMs: input.durationMs,
|
|
23838
|
+
errorCount: input.partial ? 1 : 0
|
|
23839
|
+
},
|
|
23840
|
+
capture: {
|
|
23841
|
+
execution: "kernel_colocated_playwright",
|
|
23842
|
+
stealth: true,
|
|
23843
|
+
managedProxy: "kernel_default",
|
|
23844
|
+
metadataOverrides: false,
|
|
23845
|
+
locationSignal: input.location ? "uule_only" : "none",
|
|
23846
|
+
attempt: input.attempt,
|
|
23847
|
+
finalUrl: input.navigation.finalUrl,
|
|
23848
|
+
captchaObserved: input.navigation.captchaObserved,
|
|
23849
|
+
solverCleared: input.navigation.solverCleared,
|
|
23850
|
+
environment: input.navigation.environment,
|
|
23851
|
+
rawDomSha256: input.rawDomSha256
|
|
23852
|
+
}
|
|
23853
|
+
};
|
|
23854
|
+
}
|
|
23855
|
+
async function runDurablePaaCapture(input) {
|
|
23856
|
+
const startedAt = input.startedAtMs ?? Date.now();
|
|
23857
|
+
const workDeadline = startedAt + PAA_BROWSER_WORK_BUDGET_MS;
|
|
23858
|
+
const location2 = input.location?.trim() || null;
|
|
23859
|
+
const params = new URLSearchParams({
|
|
23860
|
+
q: input.query,
|
|
23861
|
+
gl: input.gl ?? "us",
|
|
23862
|
+
hl: input.hl ?? "en",
|
|
23863
|
+
pws: "0"
|
|
23864
|
+
});
|
|
23865
|
+
if (location2) params.set("uule", encodeUule(normalizeLocation(location2)));
|
|
23866
|
+
const searchUrl = `https://www.google.com/search?${params.toString()}`;
|
|
23867
|
+
const kernel = new import_sdk8.default({ apiKey: input.apiKey });
|
|
23868
|
+
let lastCheckpoint = {
|
|
23869
|
+
phase: "navigating",
|
|
23870
|
+
attempt: 1,
|
|
23871
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
23872
|
+
records: [],
|
|
23873
|
+
clickedQuestions: [],
|
|
23874
|
+
finalUrl: null,
|
|
23875
|
+
captchaObserved: false,
|
|
23876
|
+
solverCleared: false,
|
|
23877
|
+
error: null
|
|
23878
|
+
};
|
|
23879
|
+
const persist = async (patch) => {
|
|
23880
|
+
lastCheckpoint = { ...lastCheckpoint, ...patch, observedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
23881
|
+
await input.onCheckpoint?.(lastCheckpoint);
|
|
23882
|
+
};
|
|
23883
|
+
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
23884
|
+
if (Date.now() >= workDeadline) break;
|
|
23885
|
+
let sessionId = null;
|
|
23886
|
+
const records = /* @__PURE__ */ new Map();
|
|
23887
|
+
const clicked = /* @__PURE__ */ new Set();
|
|
23888
|
+
const parents = {};
|
|
23889
|
+
let navigation = null;
|
|
23890
|
+
try {
|
|
23891
|
+
await persist({ phase: "navigating", attempt, records: [], clickedQuestions: [], error: null });
|
|
23892
|
+
const browser = await kernel.browsers.create({
|
|
23893
|
+
stealth: true,
|
|
23894
|
+
timeout_seconds: Math.max(60, Math.min(280, Math.ceil((workDeadline - Date.now()) / 1e3) + 20))
|
|
23895
|
+
});
|
|
23896
|
+
sessionId = browser.session_id;
|
|
23897
|
+
navigation = await execute(kernel, sessionId, navigationCode(searchUrl), workDeadline, 110);
|
|
23898
|
+
await persist({
|
|
23899
|
+
phase: navigation.outcome === "ready" ? "expanding" : "blocked",
|
|
23900
|
+
finalUrl: navigation.finalUrl,
|
|
23901
|
+
captchaObserved: navigation.captchaObserved,
|
|
23902
|
+
solverCleared: navigation.solverCleared
|
|
23903
|
+
});
|
|
23904
|
+
if (navigation.outcome === "captcha" || navigation.outcome === "soft_block") {
|
|
23905
|
+
if (attempt === 1 && records.size === 0) continue;
|
|
23906
|
+
throw new Error(navigation.outcome);
|
|
23907
|
+
}
|
|
23908
|
+
let stagnant = 0;
|
|
23909
|
+
for (let step = 0; step < input.maxQuestions * 2 && Date.now() < workDeadline - 35e3; step += 1) {
|
|
23910
|
+
const observed = await execute(
|
|
23911
|
+
kernel,
|
|
23912
|
+
sessionId,
|
|
23913
|
+
expansionCode(Array.from(clicked)),
|
|
23914
|
+
workDeadline,
|
|
23915
|
+
25
|
|
23916
|
+
);
|
|
23917
|
+
const beforeSize = records.size;
|
|
23918
|
+
for (const record of observed.records) {
|
|
23919
|
+
const existing = records.get(record.question);
|
|
23920
|
+
records.set(record.question, {
|
|
23921
|
+
...record,
|
|
23922
|
+
answer: record.answer || existing?.answer || "",
|
|
23923
|
+
sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
|
|
23924
|
+
sourceSite: record.sourceSite || existing?.sourceSite || "",
|
|
23925
|
+
sourceCite: record.sourceCite || existing?.sourceCite || "",
|
|
23926
|
+
sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
|
|
23927
|
+
parentQuestion: parents[record.question] ?? null,
|
|
23928
|
+
depth: parents[record.question] ? 2 : 1
|
|
23929
|
+
});
|
|
23930
|
+
}
|
|
23931
|
+
if (observed.outcome === "clicked" && observed.targetQuestion) {
|
|
23932
|
+
clicked.add(observed.targetQuestion);
|
|
23933
|
+
for (const question of observed.newQuestions) {
|
|
23934
|
+
if (!parents[question]) parents[question] = observed.targetQuestion;
|
|
23935
|
+
}
|
|
23936
|
+
}
|
|
23937
|
+
const selected2 = Array.from(records.values()).slice(0, input.maxQuestions);
|
|
23938
|
+
await persist({
|
|
23939
|
+
phase: "expanding",
|
|
23940
|
+
records: selected2,
|
|
23941
|
+
clickedQuestions: Array.from(clicked),
|
|
23942
|
+
finalUrl: observed.finalUrl,
|
|
23943
|
+
captchaObserved: lastCheckpoint.captchaObserved || observed.outcome === "captcha"
|
|
23944
|
+
});
|
|
23945
|
+
if (observed.outcome === "captcha" || observed.outcome === "soft_block") break;
|
|
23946
|
+
if (selected2.length >= input.maxQuestions && selected2.every((record) => clicked.has(record.question))) break;
|
|
23947
|
+
stagnant = records.size === beforeSize || observed.outcome === "exhausted" ? stagnant + 1 : 0;
|
|
23948
|
+
if (stagnant >= 4) break;
|
|
23949
|
+
}
|
|
23950
|
+
const finalCapture = await execute(
|
|
23951
|
+
kernel,
|
|
23952
|
+
sessionId,
|
|
23953
|
+
finalCaptureCode(input.maxQuestions, parents),
|
|
23954
|
+
workDeadline,
|
|
23955
|
+
45
|
|
23956
|
+
);
|
|
23957
|
+
for (const record of finalCapture.paa) {
|
|
23958
|
+
const existing = records.get(record.question);
|
|
23959
|
+
records.set(record.question, {
|
|
23960
|
+
...record,
|
|
23961
|
+
answer: record.answer || existing?.answer || "",
|
|
23962
|
+
sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
|
|
23963
|
+
sourceSite: record.sourceSite || existing?.sourceSite || "",
|
|
23964
|
+
sourceCite: record.sourceCite || existing?.sourceCite || "",
|
|
23965
|
+
sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
|
|
23966
|
+
parentQuestion: parents[record.question] ?? record.parentQuestion,
|
|
23967
|
+
depth: record.depth
|
|
23968
|
+
});
|
|
23969
|
+
}
|
|
23970
|
+
let rawDomGzip = null;
|
|
23971
|
+
let rawDomSha256 = null;
|
|
23972
|
+
if (Date.now() < workDeadline - 12e3) {
|
|
23973
|
+
const prepared = await execute(
|
|
23974
|
+
kernel,
|
|
23975
|
+
sessionId,
|
|
23976
|
+
RAW_PREPARE_CODE,
|
|
23977
|
+
workDeadline,
|
|
23978
|
+
35
|
|
23979
|
+
);
|
|
23980
|
+
const chunks = [];
|
|
23981
|
+
for (let offset = 0; offset < prepared.chars && Date.now() < workDeadline - 5e3; offset += 18e4) {
|
|
23982
|
+
const part = await execute(
|
|
23983
|
+
kernel,
|
|
23984
|
+
sessionId,
|
|
23985
|
+
rawChunkCode(offset, 18e4),
|
|
23986
|
+
workDeadline,
|
|
23987
|
+
15
|
|
23988
|
+
);
|
|
23989
|
+
if (part.offset !== offset) throw new Error("raw_dom_chunk_offset_mismatch");
|
|
23990
|
+
chunks.push(part.chunk);
|
|
23991
|
+
}
|
|
23992
|
+
if (chunks.join("").length === prepared.chars) {
|
|
23993
|
+
const transported = Buffer.from(chunks.join(""), "base64");
|
|
23994
|
+
if (prepared.encoding === "gzip") rawDomGzip = transported;
|
|
23995
|
+
else {
|
|
23996
|
+
const { gzipSync } = await import("zlib");
|
|
23997
|
+
rawDomGzip = gzipSync(transported);
|
|
23998
|
+
}
|
|
23999
|
+
rawDomSha256 = (0, import_node_crypto11.createHash)("sha256").update(rawDomGzip).digest("hex");
|
|
24000
|
+
}
|
|
24001
|
+
}
|
|
24002
|
+
const selected = Array.from(records.values()).slice(0, input.maxQuestions);
|
|
24003
|
+
const partial = selected.length < input.maxQuestions || selected.some((record) => !record.answer || !record.sourceCite);
|
|
24004
|
+
await persist({ phase: "captured", records: selected, clickedQuestions: Array.from(clicked) });
|
|
24005
|
+
return {
|
|
24006
|
+
result: buildPublicResult({
|
|
24007
|
+
query: input.query,
|
|
24008
|
+
location: location2,
|
|
24009
|
+
capture: finalCapture,
|
|
24010
|
+
records: selected,
|
|
24011
|
+
navigation,
|
|
24012
|
+
durationMs: Date.now() - startedAt,
|
|
24013
|
+
partial,
|
|
24014
|
+
attempt,
|
|
24015
|
+
rawDomSha256
|
|
24016
|
+
}),
|
|
24017
|
+
rawDomGzip,
|
|
24018
|
+
rawDomSha256,
|
|
24019
|
+
attempt,
|
|
24020
|
+
partial,
|
|
24021
|
+
checkpoint: lastCheckpoint
|
|
24022
|
+
};
|
|
24023
|
+
} catch (error) {
|
|
24024
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
24025
|
+
await persist({
|
|
24026
|
+
phase: records.size > 0 ? "captured" : "failed",
|
|
24027
|
+
records: Array.from(records.values()).slice(0, input.maxQuestions),
|
|
24028
|
+
clickedQuestions: Array.from(clicked),
|
|
24029
|
+
error: message.slice(0, 500)
|
|
24030
|
+
});
|
|
24031
|
+
if (records.size > 0) {
|
|
24032
|
+
const emptyCapture = {
|
|
24033
|
+
paa: [],
|
|
24034
|
+
organicResults: [],
|
|
24035
|
+
localPack: [],
|
|
24036
|
+
videos: [],
|
|
24037
|
+
forums: [],
|
|
24038
|
+
whatPeopleSaying: [],
|
|
24039
|
+
aiOverview: { detected: false, text: null, citations: [] },
|
|
24040
|
+
aiMode: { detected: false, text: null, citations: [] },
|
|
24041
|
+
surface: "unknown",
|
|
24042
|
+
entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
|
|
24043
|
+
serpFeatures: { sectionHeadings: [], detected: {}, counts: {} },
|
|
24044
|
+
relatedSearches: []
|
|
24045
|
+
};
|
|
24046
|
+
return {
|
|
24047
|
+
result: buildPublicResult({
|
|
24048
|
+
query: input.query,
|
|
24049
|
+
location: location2,
|
|
24050
|
+
capture: emptyCapture,
|
|
24051
|
+
records: Array.from(records.values()).slice(0, input.maxQuestions),
|
|
24052
|
+
navigation: navigation ?? {
|
|
24053
|
+
outcome: "soft_block",
|
|
24054
|
+
finalUrl: lastCheckpoint.finalUrl ?? "",
|
|
24055
|
+
title: "",
|
|
24056
|
+
captchaObserved: lastCheckpoint.captchaObserved,
|
|
24057
|
+
solverCleared: lastCheckpoint.solverCleared,
|
|
24058
|
+
environment: null
|
|
24059
|
+
},
|
|
24060
|
+
durationMs: Date.now() - startedAt,
|
|
24061
|
+
partial: true,
|
|
24062
|
+
attempt,
|
|
24063
|
+
rawDomSha256: null
|
|
24064
|
+
}),
|
|
24065
|
+
rawDomGzip: null,
|
|
24066
|
+
rawDomSha256: null,
|
|
24067
|
+
attempt,
|
|
24068
|
+
partial: true,
|
|
24069
|
+
checkpoint: lastCheckpoint
|
|
24070
|
+
};
|
|
24071
|
+
}
|
|
24072
|
+
if (attempt === 2 || Date.now() >= workDeadline - 35e3) throw error;
|
|
24073
|
+
} finally {
|
|
24074
|
+
if (sessionId) {
|
|
24075
|
+
await kernel.browsers.deleteByID(sessionId).catch(() => void 0);
|
|
24076
|
+
}
|
|
24077
|
+
}
|
|
24078
|
+
}
|
|
24079
|
+
throw new Error("paa_work_deadline_exhausted_before_capture");
|
|
24080
|
+
}
|
|
24081
|
+
var import_sdk8, import_node_crypto11, PAA_INVOCATION_BUDGET_MS, PAA_BROWSER_WORK_BUDGET_MS, PAA_SOLVER_BUDGET_MS, RAW_PREPARE_CODE;
|
|
24082
|
+
var init_durable_capture = __esm({
|
|
24083
|
+
"src/paa/durable-capture.ts"() {
|
|
24084
|
+
"use strict";
|
|
24085
|
+
import_sdk8 = __toESM(require("@onkernel/sdk"), 1);
|
|
24086
|
+
import_node_crypto11 = require("crypto");
|
|
24087
|
+
init_selectors();
|
|
24088
|
+
init_uule();
|
|
24089
|
+
PAA_INVOCATION_BUDGET_MS = 28e4;
|
|
24090
|
+
PAA_BROWSER_WORK_BUDGET_MS = 25e4;
|
|
24091
|
+
PAA_SOLVER_BUDGET_MS = 6e4;
|
|
24092
|
+
RAW_PREPARE_CODE = `
|
|
24093
|
+
const result = await page.evaluate(async () => {
|
|
24094
|
+
const source = new TextEncoder().encode(document.documentElement.outerHTML);
|
|
24095
|
+
const compressed = typeof CompressionStream === 'undefined'
|
|
24096
|
+
? source
|
|
24097
|
+
: new Uint8Array(await new Response(new Blob([source]).stream().pipeThrough(new CompressionStream('gzip'))).arrayBuffer());
|
|
24098
|
+
const pieces = [];
|
|
24099
|
+
for (let offset = 0; offset < compressed.length; offset += 32768) {
|
|
24100
|
+
pieces.push(String.fromCharCode(...compressed.subarray(offset, offset + 32768)));
|
|
24101
|
+
}
|
|
24102
|
+
window.__mcpPaaRawDom = btoa(pieces.join(''));
|
|
24103
|
+
return { encoding: typeof CompressionStream === 'undefined' ? 'plain' : 'gzip', chars: window.__mcpPaaRawDom.length };
|
|
24104
|
+
});
|
|
24105
|
+
return result;
|
|
24106
|
+
`;
|
|
24107
|
+
}
|
|
24108
|
+
});
|
|
24109
|
+
|
|
24110
|
+
// src/inngest/functions/paa-harvest.ts
|
|
24111
|
+
function questionCount(result) {
|
|
24112
|
+
if (!result || typeof result !== "object") return 0;
|
|
24113
|
+
const value = result;
|
|
24114
|
+
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
24115
|
+
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
24116
|
+
}
|
|
24117
|
+
var paaHarvestFn;
|
|
24118
|
+
var init_paa_harvest = __esm({
|
|
24119
|
+
"src/inngest/functions/paa-harvest.ts"() {
|
|
24120
|
+
"use strict";
|
|
24121
|
+
init_client();
|
|
24122
|
+
init_browser_service_env();
|
|
24123
|
+
init_db();
|
|
24124
|
+
init_paa_capture_artifacts();
|
|
24125
|
+
init_paa_harvest_settlement();
|
|
24126
|
+
init_webhook();
|
|
24127
|
+
init_durable_capture();
|
|
24128
|
+
paaHarvestFn = inngest.createFunction(
|
|
24129
|
+
{
|
|
24130
|
+
id: "paa-harvest",
|
|
24131
|
+
retries: 0,
|
|
24132
|
+
triggers: [{ event: "mcp-scraper/paa.requested" }],
|
|
24133
|
+
onFailure: async ({ event }) => {
|
|
24134
|
+
const jobId2 = event?.data?.event?.data?.jobId;
|
|
24135
|
+
if (!jobId2) return;
|
|
24136
|
+
const current = await getJob(jobId2);
|
|
24137
|
+
if (!current) return;
|
|
24138
|
+
if (current.status === "done" || current.status === "failed" || current.status === "cancelled") {
|
|
24139
|
+
await settlePaaHarvestJob(jobId2).catch((error) => {
|
|
24140
|
+
console.error("[paa-harvest/onFailure] terminal settlement pending:", error instanceof Error ? error.message : String(error));
|
|
24141
|
+
});
|
|
24142
|
+
return;
|
|
24143
|
+
}
|
|
24144
|
+
const message = String(event?.data?.error?.message ?? "durable PAA invocation failed").slice(0, 1e3);
|
|
24145
|
+
await failJob(jobId2, message);
|
|
24146
|
+
await settlePaaHarvestJob(jobId2).catch((error) => {
|
|
24147
|
+
console.error("[paa-harvest/onFailure] settlement pending:", error instanceof Error ? error.message : String(error));
|
|
24148
|
+
});
|
|
24149
|
+
}
|
|
24150
|
+
},
|
|
24151
|
+
async ({ event }) => {
|
|
24152
|
+
const invocationStartedAt = Date.now();
|
|
24153
|
+
const job = await claimInngestPaaJob(event.data.jobId);
|
|
24154
|
+
if (!job) return { jobId: event.data.jobId, status: "missing_or_not_paa" };
|
|
24155
|
+
if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
|
|
24156
|
+
await settlePaaHarvestJob(job.id);
|
|
24157
|
+
return { jobId: job.id, status: job.status };
|
|
24158
|
+
}
|
|
24159
|
+
const options = job.options;
|
|
24160
|
+
const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
|
|
24161
|
+
const apiKey = browserServiceApiKey();
|
|
24162
|
+
if (!apiKey) throw new Error("browser service is not configured");
|
|
24163
|
+
const attemptStartedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
24164
|
+
await startHarvestAttempt({
|
|
24165
|
+
jobId: job.id,
|
|
24166
|
+
userId: job.user_id,
|
|
24167
|
+
attemptNumber: 1,
|
|
24168
|
+
maxAttempts: 2,
|
|
24169
|
+
query: options.query ?? job.query,
|
|
24170
|
+
location: options.location ?? null,
|
|
24171
|
+
maxQuestions,
|
|
24172
|
+
startedAt: attemptStartedAt
|
|
24173
|
+
});
|
|
24174
|
+
const checkpointState = { value: null };
|
|
24175
|
+
try {
|
|
24176
|
+
const capture = await runDurablePaaCapture({
|
|
24177
|
+
apiKey,
|
|
24178
|
+
query: options.query ?? job.query,
|
|
24179
|
+
location: options.location ?? null,
|
|
24180
|
+
gl: options.gl ?? "us",
|
|
24181
|
+
hl: options.hl ?? "en",
|
|
24182
|
+
maxQuestions,
|
|
24183
|
+
startedAtMs: invocationStartedAt,
|
|
24184
|
+
onCheckpoint: async (checkpoint) => {
|
|
24185
|
+
checkpointState.value = checkpoint;
|
|
24186
|
+
const persisted = await checkpointJob(job.id, {
|
|
24187
|
+
progress: checkpoint,
|
|
24188
|
+
durable: {
|
|
24189
|
+
invocationBudgetMs: PAA_INVOCATION_BUDGET_MS,
|
|
24190
|
+
workBudgetMs: 25e4,
|
|
24191
|
+
cleanupReserveMs: 3e4
|
|
24192
|
+
}
|
|
24193
|
+
});
|
|
24194
|
+
if (!persisted) throw new Error("job_left_running_state_during_capture");
|
|
24195
|
+
}
|
|
24196
|
+
});
|
|
24197
|
+
const artifactResult = capture.rawDomGzip ? await tryCreatePaaRawDomArtifact({
|
|
24198
|
+
userId: job.user_id,
|
|
24199
|
+
jobId: job.id,
|
|
24200
|
+
attempt: capture.attempt,
|
|
24201
|
+
gzip: capture.rawDomGzip
|
|
24202
|
+
}) : { artifact: null, status: "unavailable" };
|
|
24203
|
+
const artifact = artifactResult.artifact;
|
|
24204
|
+
const artifactMissing = capture.rawDomGzip !== null && artifactResult.status !== "stored";
|
|
24205
|
+
const result = {
|
|
24206
|
+
...capture.result,
|
|
24207
|
+
diagnostics: {
|
|
24208
|
+
...capture.result.diagnostics,
|
|
24209
|
+
...artifactMissing ? {
|
|
24210
|
+
resultQuality: "partial",
|
|
24211
|
+
degradedResult: true,
|
|
24212
|
+
retryRecommended: false
|
|
24213
|
+
} : {}
|
|
24214
|
+
},
|
|
24215
|
+
capture: {
|
|
24216
|
+
...capture.result.capture,
|
|
24217
|
+
rawDomArtifact: artifact,
|
|
24218
|
+
rawDomArtifactStatus: artifactResult.status
|
|
24219
|
+
}
|
|
24220
|
+
};
|
|
24221
|
+
await completeJob(job.id, result);
|
|
24222
|
+
await finishHarvestAttempt({
|
|
24223
|
+
jobId: job.id,
|
|
24224
|
+
attemptNumber: 1,
|
|
24225
|
+
outcome: capture.partial ? "partial" : "paa_found",
|
|
24226
|
+
kernelSessionId: null,
|
|
24227
|
+
questionCount: questionCount(result),
|
|
24228
|
+
durationMs: Date.now() - invocationStartedAt,
|
|
24229
|
+
error: capture.checkpoint.error,
|
|
24230
|
+
willRetry: false,
|
|
24231
|
+
kernelDeleteStarted: true,
|
|
24232
|
+
kernelDeleteSucceeded: true,
|
|
24233
|
+
kernelDeleteError: null,
|
|
24234
|
+
browserCloseSucceeded: true,
|
|
24235
|
+
browserCloseError: null,
|
|
24236
|
+
debug: {
|
|
24237
|
+
execution: "kernel_colocated_playwright",
|
|
24238
|
+
stealth: true,
|
|
24239
|
+
attemptUsed: capture.attempt,
|
|
24240
|
+
rawDomStored: Boolean(artifact)
|
|
24241
|
+
},
|
|
24242
|
+
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
24243
|
+
});
|
|
24244
|
+
await settlePaaHarvestJob(job.id);
|
|
24245
|
+
if (job.callback_url) {
|
|
24246
|
+
await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
|
|
24247
|
+
}
|
|
24248
|
+
return {
|
|
24249
|
+
jobId: job.id,
|
|
24250
|
+
status: "done",
|
|
24251
|
+
totalQuestions: questionCount(result),
|
|
24252
|
+
partial: capture.partial,
|
|
24253
|
+
artifactStored: Boolean(artifact)
|
|
24254
|
+
};
|
|
24255
|
+
} catch (error) {
|
|
24256
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
24257
|
+
const latestCheckpoint = checkpointState.value;
|
|
24258
|
+
await finishHarvestAttempt({
|
|
24259
|
+
jobId: job.id,
|
|
24260
|
+
attemptNumber: 1,
|
|
24261
|
+
outcome: latestCheckpoint?.records.length ? "partial_session_lost" : "failed_before_capture",
|
|
24262
|
+
kernelSessionId: null,
|
|
24263
|
+
questionCount: latestCheckpoint?.records.length ?? 0,
|
|
24264
|
+
durationMs: Date.now() - invocationStartedAt,
|
|
24265
|
+
error: message.slice(0, 1e3),
|
|
24266
|
+
willRetry: false,
|
|
24267
|
+
kernelDeleteStarted: true,
|
|
24268
|
+
kernelDeleteSucceeded: null,
|
|
24269
|
+
kernelDeleteError: null,
|
|
24270
|
+
browserCloseSucceeded: null,
|
|
24271
|
+
browserCloseError: null,
|
|
24272
|
+
debug: latestCheckpoint,
|
|
24273
|
+
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
24274
|
+
});
|
|
24275
|
+
await failJob(job.id, message.slice(0, 1e3));
|
|
24276
|
+
await settlePaaHarvestJob(job.id).catch((settlementError) => {
|
|
24277
|
+
console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
|
|
24278
|
+
});
|
|
24279
|
+
if (job.callback_url) {
|
|
24280
|
+
await deliverWebhook(job.callback_url, {
|
|
24281
|
+
job_id: job.id,
|
|
24282
|
+
status: "failed",
|
|
24283
|
+
error: message,
|
|
24284
|
+
progress: latestCheckpoint
|
|
24285
|
+
});
|
|
24286
|
+
}
|
|
24287
|
+
throw error;
|
|
24288
|
+
}
|
|
24289
|
+
}
|
|
24290
|
+
);
|
|
24291
|
+
}
|
|
24292
|
+
});
|
|
24293
|
+
|
|
23185
24294
|
// src/api/site-extract-start.ts
|
|
23186
24295
|
async function prepareSiteExtractStart(input) {
|
|
23187
24296
|
const existing = await getExtractJobByIdempotencyKey(input.userId, input.idempotencyKey);
|
|
@@ -23953,15 +25062,15 @@ function normalizeIdempotencyKey(value) {
|
|
|
23953
25062
|
return key;
|
|
23954
25063
|
}
|
|
23955
25064
|
function tokenFor(id, idempotencyKey2) {
|
|
23956
|
-
return (0,
|
|
25065
|
+
return (0, import_node_crypto12.createHmac)("sha256", billingSecret()).update(id).update(":").update(idempotencyKey2).digest("base64url");
|
|
23957
25066
|
}
|
|
23958
25067
|
function tokenHash(token2) {
|
|
23959
|
-
return (0,
|
|
25068
|
+
return (0, import_node_crypto12.createHash)("sha256").update(token2).digest("hex");
|
|
23960
25069
|
}
|
|
23961
25070
|
function verifyToken(row, token2) {
|
|
23962
25071
|
const expected = Buffer.from(row.token_hash, "hex");
|
|
23963
25072
|
const actual = Buffer.from(tokenHash(token2), "hex");
|
|
23964
|
-
if (expected.length !== actual.length || !(0,
|
|
25073
|
+
if (expected.length !== actual.length || !(0, import_node_crypto12.timingSafeEqual)(expected, actual)) {
|
|
23965
25074
|
throw new UnifiedBillingError("unauthorized", "invalid billing authorization token", 401);
|
|
23966
25075
|
}
|
|
23967
25076
|
}
|
|
@@ -24076,7 +25185,7 @@ async function authorizeScheduledRun(args) {
|
|
|
24076
25185
|
{ balanceMc, requiredMc: SCHEDULED_RUN_BASE_MC }
|
|
24077
25186
|
);
|
|
24078
25187
|
}
|
|
24079
|
-
const id = (0,
|
|
25188
|
+
const id = (0, import_node_crypto12.randomUUID)();
|
|
24080
25189
|
const token2 = tokenFor(id, idempotencyKey2);
|
|
24081
25190
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
24082
25191
|
const inserted = await getDb().execute({
|
|
@@ -24144,7 +25253,7 @@ async function startScheduledRun(args) {
|
|
|
24144
25253
|
if (!authorization || !["starting", "started"].includes(authorization.status)) {
|
|
24145
25254
|
throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer startable", 409);
|
|
24146
25255
|
}
|
|
24147
|
-
const eventId = existing?.id ?? (0,
|
|
25256
|
+
const eventId = existing?.id ?? (0, import_node_crypto12.randomUUID)();
|
|
24148
25257
|
if (!existing) {
|
|
24149
25258
|
await getDb().execute({
|
|
24150
25259
|
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, multiplier_bps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'capturing', ?, ?, ?)",
|
|
@@ -24281,7 +25390,7 @@ async function settleScheduledRun(args) {
|
|
|
24281
25390
|
chargeMode: "model_delta",
|
|
24282
25391
|
...pendingReason ? { reason: pendingReason } : {}
|
|
24283
25392
|
});
|
|
24284
|
-
const eventId = existing?.id ?? (0,
|
|
25393
|
+
const eventId = existing?.id ?? (0, import_node_crypto12.randomUUID)();
|
|
24285
25394
|
const status = pendingReason ? "cost_pending" : "settling";
|
|
24286
25395
|
const eventValues = [
|
|
24287
25396
|
modelMc,
|
|
@@ -24340,11 +25449,11 @@ async function voidScheduledRunAuthorization(args) {
|
|
|
24340
25449
|
}
|
|
24341
25450
|
return { ok: true, status: authorization.status };
|
|
24342
25451
|
}
|
|
24343
|
-
var
|
|
25452
|
+
var import_node_crypto12, SCHEDULED_RUN_BILLING_CLASS, SCHEDULED_RUN_SOURCE_SURFACE, AUTHORIZATION_TTL_MS, UnifiedBillingError;
|
|
24344
25453
|
var init_unified_billing = __esm({
|
|
24345
25454
|
"src/api/unified-billing.ts"() {
|
|
24346
25455
|
"use strict";
|
|
24347
|
-
|
|
25456
|
+
import_node_crypto12 = require("crypto");
|
|
24348
25457
|
init_db();
|
|
24349
25458
|
init_rates();
|
|
24350
25459
|
init_scheduling_access();
|
|
@@ -24376,14 +25485,14 @@ function getSessionSecret() {
|
|
|
24376
25485
|
function safeEqualHex(a, b) {
|
|
24377
25486
|
if (a.length !== b.length) return false;
|
|
24378
25487
|
try {
|
|
24379
|
-
return (0,
|
|
25488
|
+
return (0, import_node_crypto13.timingSafeEqual)(Buffer.from(a, "hex"), Buffer.from(b, "hex"));
|
|
24380
25489
|
} catch {
|
|
24381
25490
|
return false;
|
|
24382
25491
|
}
|
|
24383
25492
|
}
|
|
24384
25493
|
function signSession(userId) {
|
|
24385
25494
|
const payload = String(userId);
|
|
24386
|
-
const sig = (0,
|
|
25495
|
+
const sig = (0, import_node_crypto13.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
24387
25496
|
return `${payload}.${sig}`;
|
|
24388
25497
|
}
|
|
24389
25498
|
function verifySession(token2) {
|
|
@@ -24391,16 +25500,16 @@ function verifySession(token2) {
|
|
|
24391
25500
|
if (dot === -1) return null;
|
|
24392
25501
|
const payload = token2.slice(0, dot);
|
|
24393
25502
|
const sig = token2.slice(dot + 1);
|
|
24394
|
-
const expected = (0,
|
|
25503
|
+
const expected = (0, import_node_crypto13.createHmac)("sha256", secret()).update(payload).digest("hex");
|
|
24395
25504
|
if (!safeEqualHex(sig, expected)) return null;
|
|
24396
25505
|
const id = parseInt(payload);
|
|
24397
25506
|
return isNaN(id) ? null : id;
|
|
24398
25507
|
}
|
|
24399
|
-
var
|
|
25508
|
+
var import_node_crypto13, isProduction, secret;
|
|
24400
25509
|
var init_session = __esm({
|
|
24401
25510
|
"src/api/session.ts"() {
|
|
24402
25511
|
"use strict";
|
|
24403
|
-
|
|
25512
|
+
import_node_crypto13 = require("crypto");
|
|
24404
25513
|
isProduction = () => process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
|
|
24405
25514
|
secret = () => getSessionSecret();
|
|
24406
25515
|
}
|
|
@@ -24418,11 +25527,11 @@ function isMemoryOperator(email) {
|
|
|
24418
25527
|
return ops.includes(email.trim().toLowerCase());
|
|
24419
25528
|
}
|
|
24420
25529
|
function encKey() {
|
|
24421
|
-
return (0,
|
|
25530
|
+
return (0, import_node_crypto14.scryptSync)(getSessionSecret(), "mcp-memory-key-v1", 32);
|
|
24422
25531
|
}
|
|
24423
25532
|
function encryptMemoryKey(secret2) {
|
|
24424
|
-
const iv = (0,
|
|
24425
|
-
const cipher = (0,
|
|
25533
|
+
const iv = (0, import_node_crypto14.randomBytes)(12);
|
|
25534
|
+
const cipher = (0, import_node_crypto14.createCipheriv)("aes-256-gcm", encKey(), iv);
|
|
24426
25535
|
const enc = Buffer.concat([cipher.update(secret2, "utf8"), cipher.final()]);
|
|
24427
25536
|
const tag = cipher.getAuthTag();
|
|
24428
25537
|
return `${iv.toString("base64")}:${tag.toString("base64")}:${enc.toString("base64")}`;
|
|
@@ -24430,7 +25539,7 @@ function encryptMemoryKey(secret2) {
|
|
|
24430
25539
|
function decryptMemoryKey(stored) {
|
|
24431
25540
|
try {
|
|
24432
25541
|
const [ivB, tagB, dataB] = stored.split(":");
|
|
24433
|
-
const decipher = (0,
|
|
25542
|
+
const decipher = (0, import_node_crypto14.createDecipheriv)("aes-256-gcm", encKey(), Buffer.from(ivB, "base64"));
|
|
24434
25543
|
decipher.setAuthTag(Buffer.from(tagB, "base64"));
|
|
24435
25544
|
return Buffer.concat([decipher.update(Buffer.from(dataB, "base64")), decipher.final()]).toString("utf8");
|
|
24436
25545
|
} catch {
|
|
@@ -24544,11 +25653,11 @@ async function syncScheduledActionCredentials(user) {
|
|
|
24544
25653
|
}, {});
|
|
24545
25654
|
return { ok: res.ok, error: res.error };
|
|
24546
25655
|
}
|
|
24547
|
-
var
|
|
25656
|
+
var import_node_crypto14, import_provision_defaults, import_set_schedule_entitlement, MEMORY_BASE_URL, ADMIN_KEY;
|
|
24548
25657
|
var init_memory = __esm({
|
|
24549
25658
|
"src/api/memory.ts"() {
|
|
24550
25659
|
"use strict";
|
|
24551
|
-
|
|
25660
|
+
import_node_crypto14 = require("crypto");
|
|
24552
25661
|
init_session();
|
|
24553
25662
|
init_db();
|
|
24554
25663
|
init_rates();
|
|
@@ -24598,7 +25707,7 @@ var init_connected_cost_telemetry = __esm({
|
|
|
24598
25707
|
|
|
24599
25708
|
// src/api/connected-usage-billing.ts
|
|
24600
25709
|
function hash(value) {
|
|
24601
|
-
return (0,
|
|
25710
|
+
return (0, import_node_crypto15.createHash)("sha256").update(value).digest("hex");
|
|
24602
25711
|
}
|
|
24603
25712
|
function eventKey(idempotencyKey2) {
|
|
24604
25713
|
return `connected-usage:${hash(idempotencyKey2)}`;
|
|
@@ -24870,7 +25979,7 @@ async function settleConnectedUsage(rawInput) {
|
|
|
24870
25979
|
sql: `INSERT OR IGNORE INTO billing_events
|
|
24871
25980
|
(id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
|
|
24872
25981
|
VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
|
|
24873
|
-
args: [(0,
|
|
25982
|
+
args: [(0, import_node_crypto15.randomUUID)(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
|
|
24874
25983
|
});
|
|
24875
25984
|
let event = await readEvent(key);
|
|
24876
25985
|
if (!event) throw new Error("connected usage receipt insert completed without a readable event");
|
|
@@ -24959,11 +26068,11 @@ async function listConnectedUsageHistory(userId, limit = 100) {
|
|
|
24959
26068
|
}
|
|
24960
26069
|
return history;
|
|
24961
26070
|
}
|
|
24962
|
-
var
|
|
26071
|
+
var import_node_crypto15, import_zod14, CONNECTED_USAGE_BILLING_CLASS, CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE, ConnectedUsageSafeMetadataSchema, ConnectedUsageSettlementInputSchema, ConnectedUsagePreflightInputSchema, ConnectedUsageBillingError;
|
|
24963
26072
|
var init_connected_usage_billing = __esm({
|
|
24964
26073
|
"src/api/connected-usage-billing.ts"() {
|
|
24965
26074
|
"use strict";
|
|
24966
|
-
|
|
26075
|
+
import_node_crypto15 = require("crypto");
|
|
24967
26076
|
import_zod14 = require("zod");
|
|
24968
26077
|
init_connected_cost_telemetry();
|
|
24969
26078
|
init_db();
|
|
@@ -27854,7 +28963,7 @@ async function kernelLaunchOptsResidential() {
|
|
|
27854
28963
|
}
|
|
27855
28964
|
return { headless: true, kernelApiKey: browserServiceApiKey(), kernelProxyId: proxyId, viewport: { width: 1280, height: 900 }, locale: "en-US" };
|
|
27856
28965
|
}
|
|
27857
|
-
var import_hono7, import_zod21,
|
|
28966
|
+
var import_hono7, import_zod21, import_client6, FacebookAdBodySchema, FacebookPageIntelBodySchema, FacebookTranscribeBodySchema, FacebookVideoTranscribeBodySchema, FacebookSearchBodySchema, FacebookMediaBodySchema, AD_LIBRARY_APP_CHROME, AUTH_WALL_URL, MIN_RENDERED_BODY_CHARS, facebookAdApp, ALLOWED_MEDIA_HOSTS;
|
|
27858
28967
|
var init_facebook_ad_routes = __esm({
|
|
27859
28968
|
"src/api/facebook-ad-routes.ts"() {
|
|
27860
28969
|
"use strict";
|
|
@@ -27869,7 +28978,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
27869
28978
|
init_FacebookOrganicVideoExtractor();
|
|
27870
28979
|
init_kernel_proxy_resolver();
|
|
27871
28980
|
init_schemas2();
|
|
27872
|
-
|
|
28981
|
+
import_client6 = require("@fal-ai/client");
|
|
27873
28982
|
init_api_auth();
|
|
27874
28983
|
init_url_utils();
|
|
27875
28984
|
init_concurrency_gates();
|
|
@@ -28016,7 +29125,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
28016
29125
|
metadata: { videoUrl }
|
|
28017
29126
|
});
|
|
28018
29127
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
28019
|
-
|
|
29128
|
+
import_client6.fal.config({ credentials: process.env.FAL_KEY });
|
|
28020
29129
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
28021
29130
|
let debited = false;
|
|
28022
29131
|
try {
|
|
@@ -28060,7 +29169,7 @@ var init_facebook_ad_routes = __esm({
|
|
|
28060
29169
|
metadata: { url: sourceUrl.href }
|
|
28061
29170
|
});
|
|
28062
29171
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
28063
|
-
|
|
29172
|
+
import_client6.fal.config({ credentials: process.env.FAL_KEY });
|
|
28064
29173
|
const driver = new BrowserDriver();
|
|
28065
29174
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
28066
29175
|
let debited = false;
|
|
@@ -28512,7 +29621,7 @@ async function kernelLaunchOptsResidential2() {
|
|
|
28512
29621
|
function isBlockedMessage(msg) {
|
|
28513
29622
|
return msg.toLowerCase().includes("blocked") || msg.toLowerCase().includes("captcha");
|
|
28514
29623
|
}
|
|
28515
|
-
var import_hono8, import_zod22,
|
|
29624
|
+
var import_hono8, import_zod22, import_client7, GoogleAdsSearchBodySchema, GoogleAdsPageIntelBodySchema, GoogleAdsTranscribeBodySchema, googleAdsApp;
|
|
28516
29625
|
var init_google_ads_routes = __esm({
|
|
28517
29626
|
"src/api/google-ads-routes.ts"() {
|
|
28518
29627
|
"use strict";
|
|
@@ -28525,7 +29634,7 @@ var init_google_ads_routes = __esm({
|
|
|
28525
29634
|
init_GoogleAdsExtractor();
|
|
28526
29635
|
init_kernel_proxy_resolver();
|
|
28527
29636
|
init_schemas2();
|
|
28528
|
-
|
|
29637
|
+
import_client7 = require("@fal-ai/client");
|
|
28529
29638
|
init_api_auth();
|
|
28530
29639
|
init_url_utils();
|
|
28531
29640
|
init_concurrency_gates();
|
|
@@ -28637,7 +29746,7 @@ var init_google_ads_routes = __esm({
|
|
|
28637
29746
|
const user = c.get("user");
|
|
28638
29747
|
const gate = await acquireConcurrencyGate(user, "google_ads_transcribe", { reuseLockId: c.req.header("x-mcp-scraper-concurrency-lock"), metadata: { videoUrl } });
|
|
28639
29748
|
if (!gate.ok) return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
28640
|
-
|
|
29749
|
+
import_client7.fal.config({ credentials: process.env.FAL_KEY });
|
|
28641
29750
|
const holdMc = MEDIA_TRANSCRIBE_HOLD_MC;
|
|
28642
29751
|
let debited = false;
|
|
28643
29752
|
try {
|
|
@@ -28813,7 +29922,7 @@ async function applyAdjustment(input) {
|
|
|
28813
29922
|
const credits = validateCredits(input.credits, input.confirmLarge === true);
|
|
28814
29923
|
const reason = validateReason(input.reason);
|
|
28815
29924
|
const actor = validateActor(input.actor);
|
|
28816
|
-
const reference = input.reference?.trim() || `adj_${(0,
|
|
29925
|
+
const reference = input.reference?.trim() || `adj_${(0, import_node_crypto16.randomUUID)()}`;
|
|
28817
29926
|
if (reference.length > 120) throw new AdminCreditError(400, "reference must be at most 120 characters");
|
|
28818
29927
|
const db = getDb();
|
|
28819
29928
|
const existing = await db.execute({
|
|
@@ -28958,11 +30067,11 @@ async function lookupAccount(target, limit = 20) {
|
|
|
28958
30067
|
}))
|
|
28959
30068
|
};
|
|
28960
30069
|
}
|
|
28961
|
-
var
|
|
30070
|
+
var import_node_crypto16, MIN_ADJUSTMENT_CREDITS, LARGE_ADJUSTMENT_CREDITS, MAX_ADJUSTMENT_CREDITS, MIN_REASON_LENGTH, MAX_REASON_LENGTH, MAX_ACTOR_LENGTH, AdminCreditError;
|
|
28962
30071
|
var init_admin_credits = __esm({
|
|
28963
30072
|
"src/api/admin-credits.ts"() {
|
|
28964
30073
|
"use strict";
|
|
28965
|
-
|
|
30074
|
+
import_node_crypto16 = require("crypto");
|
|
28966
30075
|
init_db();
|
|
28967
30076
|
init_rates();
|
|
28968
30077
|
MIN_ADJUSTMENT_CREDITS = 1;
|
|
@@ -30487,14 +31596,28 @@ var init_PAAExtractor = __esm({
|
|
|
30487
31596
|
}
|
|
30488
31597
|
return parts.join(" ").replace(/\s+/g, " ").trim();
|
|
30489
31598
|
}
|
|
31599
|
+
function cleanSourceLabel(value) {
|
|
31600
|
+
return (value ?? "").replace(/\s*\(\+\d+\)\s*-\s*View related links.*$/i, "").replace(/\s*-\s*Opens in new tab.*$/i, "").trim();
|
|
31601
|
+
}
|
|
30490
31602
|
return Array.from(document.querySelectorAll(selectors.item)).map((pair) => {
|
|
30491
|
-
|
|
30492
|
-
|
|
30493
|
-
|
|
30494
|
-
|
|
31603
|
+
const clickTarget = pair.querySelector(selectors.clickTarget);
|
|
31604
|
+
const expanded = pair.classList.contains(selectors.expandedClass) || clickTarget?.getAttribute("aria-expanded") === "true";
|
|
31605
|
+
const answerContainer = expanded ? pair.querySelector(selectors.answerContainer) : null;
|
|
31606
|
+
const aiAnswerBody = answerContainer?.querySelector(selectors.aiAnswerBody);
|
|
31607
|
+
const answerElement = aiAnswerBody ?? answerContainer;
|
|
31608
|
+
let sourceTitle = expanded ? pair.querySelector(selectors.sourceTitle)?.innerText?.trim() || void 0 : void 0;
|
|
31609
|
+
let sourceSite = expanded ? pair.querySelector(selectors.sourceSite)?.innerText?.trim() || void 0 : void 0;
|
|
31610
|
+
let sourceCite = expanded ? pair.querySelector(selectors.sourceCite)?.innerText?.trim() || void 0 : void 0;
|
|
31611
|
+
if (expanded && !sourceCite) {
|
|
30495
31612
|
const seenHrefs = /* @__PURE__ */ new Set();
|
|
30496
31613
|
const anchors = [];
|
|
30497
|
-
|
|
31614
|
+
const anchorRoot = answerElement ?? pair;
|
|
31615
|
+
const prioritized = [
|
|
31616
|
+
...Array.from(anchorRoot.querySelectorAll(selectors.aiCitation)),
|
|
31617
|
+
...Array.from(anchorRoot.querySelectorAll(selectors.aiSourceCard)),
|
|
31618
|
+
...Array.from(anchorRoot.querySelectorAll("a[href]"))
|
|
31619
|
+
];
|
|
31620
|
+
for (const a of prioritized) {
|
|
30498
31621
|
const href = a.href;
|
|
30499
31622
|
if (!/^https?:\/\//.test(href) || seenHrefs.has(href)) continue;
|
|
30500
31623
|
let host = "";
|
|
@@ -30505,18 +31628,23 @@ var init_PAAExtractor = __esm({
|
|
|
30505
31628
|
}
|
|
30506
31629
|
if (/(^|\.)google(usercontent)?\.[a-z.]+$/.test(host)) continue;
|
|
30507
31630
|
seenHrefs.add(href);
|
|
30508
|
-
anchors.push({
|
|
31631
|
+
anchors.push({
|
|
31632
|
+
text: (a.textContent || "").trim(),
|
|
31633
|
+
label: cleanSourceLabel(a.getAttribute("aria-label")),
|
|
31634
|
+
href,
|
|
31635
|
+
host
|
|
31636
|
+
});
|
|
30509
31637
|
}
|
|
30510
|
-
const primary = anchors.find((x) => x.text.length > 2) ?? anchors[0];
|
|
31638
|
+
const primary = anchors.find((x) => x.label.length > 2 || x.text.length > 2) ?? anchors[0];
|
|
30511
31639
|
if (primary) {
|
|
30512
|
-
sourceTitle = sourceTitle || primary.text || void 0;
|
|
31640
|
+
sourceTitle = sourceTitle || primary.label || primary.text || void 0;
|
|
30513
31641
|
sourceSite = sourceSite || primary.host.replace(/^www\./, "");
|
|
30514
31642
|
sourceCite = primary.href;
|
|
30515
31643
|
}
|
|
30516
31644
|
}
|
|
30517
31645
|
return {
|
|
30518
31646
|
question: pair.getAttribute(selectors.itemDataQ) || pair.getAttribute(selectors.itemDataInitQ) || pair.querySelector(selectors.itemQuestionEl)?.innerText?.trim() || "",
|
|
30519
|
-
answer: cleanText(
|
|
31647
|
+
answer: cleanText(answerElement) || void 0,
|
|
30520
31648
|
sourceTitle,
|
|
30521
31649
|
sourceSite,
|
|
30522
31650
|
sourceCite
|
|
@@ -33129,12 +34257,15 @@ function mapsErrorResponse(c, err, errorCode) {
|
|
|
33129
34257
|
const msg = err instanceof Error ? err.message : String(err);
|
|
33130
34258
|
const rotationError = err instanceof MapsSearchRotationError ? err : null;
|
|
33131
34259
|
const blocked = msg.includes("CAPTCHA") || msg.includes("blocked");
|
|
33132
|
-
const
|
|
33133
|
-
|
|
34260
|
+
const publicCode = normalizePublicErrorCode(
|
|
34261
|
+
rotationError?.errorCode ?? (blocked ? "captcha_or_blocked" : "service_unavailable")
|
|
34262
|
+
);
|
|
34263
|
+
const retryable = publicCode === "service_unavailable" || rotationError?.retryable === true || blocked;
|
|
34264
|
+
const message = publicErrorMessage(publicCode);
|
|
33134
34265
|
return c.json({
|
|
33135
34266
|
error: message,
|
|
33136
34267
|
message,
|
|
33137
|
-
error_code:
|
|
34268
|
+
error_code: publicCode,
|
|
33138
34269
|
retryable
|
|
33139
34270
|
}, retryable ? 503 : 500);
|
|
33140
34271
|
}
|
|
@@ -33693,7 +34824,7 @@ function retryDelaySeconds(attempts) {
|
|
|
33693
34824
|
}
|
|
33694
34825
|
async function dispatchPendingDirectoryWorkflows(limit = 25) {
|
|
33695
34826
|
const rows = await claimDirectoryWorkflowOutbox({
|
|
33696
|
-
workerId: `directory-dispatch-${process.pid}-${(0,
|
|
34827
|
+
workerId: `directory-dispatch-${process.pid}-${(0, import_node_crypto17.randomUUID)().slice(0, 8)}`,
|
|
33697
34828
|
limit
|
|
33698
34829
|
});
|
|
33699
34830
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
@@ -33715,11 +34846,11 @@ async function dispatchPendingDirectoryWorkflows(limit = 25) {
|
|
|
33715
34846
|
}
|
|
33716
34847
|
return result;
|
|
33717
34848
|
}
|
|
33718
|
-
var
|
|
34849
|
+
var import_node_crypto17;
|
|
33719
34850
|
var init_directory_workflow_dispatch = __esm({
|
|
33720
34851
|
"src/api/directory-workflow-dispatch.ts"() {
|
|
33721
34852
|
"use strict";
|
|
33722
|
-
|
|
34853
|
+
import_node_crypto17 = require("crypto");
|
|
33723
34854
|
init_client();
|
|
33724
34855
|
init_directory_workflow_repository();
|
|
33725
34856
|
}
|
|
@@ -33731,7 +34862,7 @@ function safeOptions(options) {
|
|
|
33731
34862
|
return safe2;
|
|
33732
34863
|
}
|
|
33733
34864
|
function requestFingerprint(options) {
|
|
33734
|
-
return (0,
|
|
34865
|
+
return (0, import_node_crypto18.createHash)("sha256").update(JSON.stringify(safeOptions(options))).digest("hex");
|
|
33735
34866
|
}
|
|
33736
34867
|
function idempotencyKey(raw) {
|
|
33737
34868
|
if (raw !== void 0) {
|
|
@@ -33740,14 +34871,14 @@ function idempotencyKey(raw) {
|
|
|
33740
34871
|
if (trimmed.length > 500) return { ok: false, message: "Idempotency-Key must be 500 characters or fewer." };
|
|
33741
34872
|
return { ok: true, key: trimmed };
|
|
33742
34873
|
}
|
|
33743
|
-
return { ok: true, key: `directory-${(0,
|
|
34874
|
+
return { ok: true, key: `directory-${(0, import_node_crypto18.randomUUID)()}` };
|
|
33744
34875
|
}
|
|
33745
34876
|
function debitKeyFor(userId, responseKey) {
|
|
33746
|
-
const digest = (0,
|
|
34877
|
+
const digest = (0, import_node_crypto18.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
33747
34878
|
return `directory-workflow:${userId}:${digest}`;
|
|
33748
34879
|
}
|
|
33749
34880
|
function jobId() {
|
|
33750
|
-
return `dir_${(0,
|
|
34881
|
+
return `dir_${(0, import_node_crypto18.randomUUID)().replace(/-/g, "")}`;
|
|
33751
34882
|
}
|
|
33752
34883
|
function publicStatus(job, result) {
|
|
33753
34884
|
if (job.status === "failed") return "failed";
|
|
@@ -33856,7 +34987,7 @@ async function runSynchronously(c, user, options, plan, responseKey) {
|
|
|
33856
34987
|
const csv = renderDirectoryWorkflowCsv(result);
|
|
33857
34988
|
const artifact = await createDirectoryCsvArtifact({
|
|
33858
34989
|
ownerId: String(user.id),
|
|
33859
|
-
jobId: (0,
|
|
34990
|
+
jobId: (0, import_node_crypto18.createHash)("sha256").update(debitKey2).digest("hex").slice(0, 32),
|
|
33860
34991
|
createdAt: result.extractedAt,
|
|
33861
34992
|
filename: `${options.state}-${options.query}-directory.csv`,
|
|
33862
34993
|
csv,
|
|
@@ -33912,11 +35043,11 @@ async function runSynchronously(c, user, options, plan, responseKey) {
|
|
|
33912
35043
|
await releaseConcurrencyGate(gate.lockId);
|
|
33913
35044
|
}
|
|
33914
35045
|
}
|
|
33915
|
-
var
|
|
35046
|
+
var import_node_crypto18, import_hono16, directoryApp;
|
|
33916
35047
|
var init_directory_routes = __esm({
|
|
33917
35048
|
"src/api/directory-routes.ts"() {
|
|
33918
35049
|
"use strict";
|
|
33919
|
-
|
|
35050
|
+
import_node_crypto18 = require("crypto");
|
|
33920
35051
|
import_hono16 = require("hono");
|
|
33921
35052
|
init_api_auth();
|
|
33922
35053
|
init_db();
|
|
@@ -34303,6 +35434,67 @@ var init_location_data_routes = __esm({
|
|
|
34303
35434
|
}
|
|
34304
35435
|
});
|
|
34305
35436
|
|
|
35437
|
+
// src/api/paa-harvest-dispatch.ts
|
|
35438
|
+
async function dispatchPaaHarvest(jobId2) {
|
|
35439
|
+
await inngest.send({
|
|
35440
|
+
id: `paa-harvest:${jobId2}`,
|
|
35441
|
+
name: "mcp-scraper/paa.requested",
|
|
35442
|
+
data: { jobId: jobId2 }
|
|
35443
|
+
});
|
|
35444
|
+
}
|
|
35445
|
+
async function redispatchPendingPaaHarvests(limit = 25) {
|
|
35446
|
+
const jobs = await listPendingInngestPaaJobs(limit);
|
|
35447
|
+
let dispatched = 0;
|
|
35448
|
+
let failed = 0;
|
|
35449
|
+
for (const job of jobs) {
|
|
35450
|
+
try {
|
|
35451
|
+
await dispatchPaaHarvest(job.id);
|
|
35452
|
+
dispatched += 1;
|
|
35453
|
+
} catch {
|
|
35454
|
+
failed += 1;
|
|
35455
|
+
}
|
|
35456
|
+
}
|
|
35457
|
+
return { checked: jobs.length, dispatched, failed };
|
|
35458
|
+
}
|
|
35459
|
+
var init_paa_harvest_dispatch = __esm({
|
|
35460
|
+
"src/api/paa-harvest-dispatch.ts"() {
|
|
35461
|
+
"use strict";
|
|
35462
|
+
init_client();
|
|
35463
|
+
init_db();
|
|
35464
|
+
}
|
|
35465
|
+
});
|
|
35466
|
+
|
|
35467
|
+
// src/api/paa-harvest-reconciliation.ts
|
|
35468
|
+
async function reconcilePaaHarvestSettlements(limit = 25) {
|
|
35469
|
+
const stale = await listStaleRunningInngestPaaJobs(limit);
|
|
35470
|
+
let staleFailed = 0;
|
|
35471
|
+
for (const job of stale) {
|
|
35472
|
+
try {
|
|
35473
|
+
await failJob(job.id, "durable PAA invocation exceeded its recovery window; preserved progress remains in result");
|
|
35474
|
+
staleFailed += 1;
|
|
35475
|
+
} catch {
|
|
35476
|
+
}
|
|
35477
|
+
}
|
|
35478
|
+
const jobs = await listRecentTerminalInngestPaaJobs(limit);
|
|
35479
|
+
let settled = 0;
|
|
35480
|
+
let failed = 0;
|
|
35481
|
+
for (const job of jobs) {
|
|
35482
|
+
try {
|
|
35483
|
+
if (await settlePaaHarvestJob(job)) settled += 1;
|
|
35484
|
+
} catch {
|
|
35485
|
+
failed += 1;
|
|
35486
|
+
}
|
|
35487
|
+
}
|
|
35488
|
+
return { checked: jobs.length, staleFailed, settled, failed };
|
|
35489
|
+
}
|
|
35490
|
+
var init_paa_harvest_reconciliation = __esm({
|
|
35491
|
+
"src/api/paa-harvest-reconciliation.ts"() {
|
|
35492
|
+
"use strict";
|
|
35493
|
+
init_db();
|
|
35494
|
+
init_paa_harvest_settlement();
|
|
35495
|
+
}
|
|
35496
|
+
});
|
|
35497
|
+
|
|
34306
35498
|
// src/api/directory-workflow-reconciliation.ts
|
|
34307
35499
|
async function reconcileDirectoryWorkflowSettlements(limit = 25) {
|
|
34308
35500
|
const expiredStarts = await listExpiredDirectoryWorkflowStarts(limit);
|
|
@@ -36337,7 +37529,7 @@ async function readManifestFromSummary(summary) {
|
|
|
36337
37529
|
function webhookSignature(body, timestamp2) {
|
|
36338
37530
|
const secret2 = process.env.MCP_SCRAPER_WEBHOOK_SECRET?.trim();
|
|
36339
37531
|
if (!secret2) return null;
|
|
36340
|
-
return (0,
|
|
37532
|
+
return (0, import_node_crypto19.createHmac)("sha256", secret2).update(`${timestamp2}.${body}`).digest("hex");
|
|
36341
37533
|
}
|
|
36342
37534
|
async function deliverWorkflowWebhook(input) {
|
|
36343
37535
|
if (!input.webhookUrl) return;
|
|
@@ -36544,11 +37736,11 @@ async function dispatchDueWorkflowSchedules(apiUrl, limit = 3) {
|
|
|
36544
37736
|
}
|
|
36545
37737
|
return { dispatched: results.length, results };
|
|
36546
37738
|
}
|
|
36547
|
-
var
|
|
37739
|
+
var import_node_crypto19, import_promises12, import_hono18, import_zod36, workflowApp, WorkflowInputSchema, WorkflowIdSchema, CadenceSchema, ScheduleStatusSchema, RunBodySchema, ScheduleCreateSchema, SchedulePatchSchema, TERMINAL_RUN_STATUSES;
|
|
36548
37740
|
var init_workflow_routes = __esm({
|
|
36549
37741
|
"src/api/workflow-routes.ts"() {
|
|
36550
37742
|
"use strict";
|
|
36551
|
-
|
|
37743
|
+
import_node_crypto19 = require("crypto");
|
|
36552
37744
|
import_promises12 = require("fs/promises");
|
|
36553
37745
|
import_hono18 = require("hono");
|
|
36554
37746
|
import_zod36 = require("zod");
|
|
@@ -36829,7 +38021,7 @@ var init_workflow_routes = __esm({
|
|
|
36829
38021
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
36830
38022
|
function sha256(value) {
|
|
36831
38023
|
if (!value) return null;
|
|
36832
|
-
return (0,
|
|
38024
|
+
return (0, import_node_crypto20.createHash)("sha256").update(value).digest("hex");
|
|
36833
38025
|
}
|
|
36834
38026
|
function countWords(markdown) {
|
|
36835
38027
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -37129,11 +38321,11 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
37129
38321
|
}
|
|
37130
38322
|
};
|
|
37131
38323
|
}
|
|
37132
|
-
var
|
|
38324
|
+
var import_node_crypto20, import_p_limit5, DEFAULT_TIMEOUT_MS, DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_CONTENT_CHARS;
|
|
37133
38325
|
var init_page_snapshot_extractor = __esm({
|
|
37134
38326
|
"src/serp-intelligence/page-snapshot-extractor.ts"() {
|
|
37135
38327
|
"use strict";
|
|
37136
|
-
|
|
38328
|
+
import_node_crypto20 = require("crypto");
|
|
37137
38329
|
import_p_limit5 = __toESM(require("p-limit"), 1);
|
|
37138
38330
|
init_kpo_extractor();
|
|
37139
38331
|
init_url_utils();
|
|
@@ -37160,13 +38352,16 @@ function looksLikeProxyTunnelFailure3(message) {
|
|
|
37160
38352
|
function looksLikeProxyUnavailable3(message) {
|
|
37161
38353
|
return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
|
|
37162
38354
|
}
|
|
38355
|
+
function looksLikeInternalServiceUnavailable(message) {
|
|
38356
|
+
return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
|
|
38357
|
+
}
|
|
37163
38358
|
function classifyHarvestProblem(err) {
|
|
37164
38359
|
const message = errorMessage3(err);
|
|
37165
38360
|
if (err instanceof RequestAbortedError) {
|
|
37166
38361
|
return {
|
|
37167
38362
|
error_code: "request_aborted",
|
|
37168
38363
|
error_type: "request_aborted",
|
|
37169
|
-
message,
|
|
38364
|
+
message: publicErrorMessage("request_aborted"),
|
|
37170
38365
|
retryable: true,
|
|
37171
38366
|
httpStatus: 408,
|
|
37172
38367
|
terminalStatus: "cancelled"
|
|
@@ -37176,7 +38371,7 @@ function classifyHarvestProblem(err) {
|
|
|
37176
38371
|
return {
|
|
37177
38372
|
error_code: "captcha_exhausted",
|
|
37178
38373
|
error_type: "captcha",
|
|
37179
|
-
message,
|
|
38374
|
+
message: publicErrorMessage("captcha_exhausted"),
|
|
37180
38375
|
retryable: true,
|
|
37181
38376
|
httpStatus: 503,
|
|
37182
38377
|
terminalStatus: "failed"
|
|
@@ -37186,7 +38381,7 @@ function classifyHarvestProblem(err) {
|
|
|
37186
38381
|
return {
|
|
37187
38382
|
error_code: "location_mismatch",
|
|
37188
38383
|
error_type: "location_mismatch",
|
|
37189
|
-
message,
|
|
38384
|
+
message: publicErrorMessage("location_mismatch"),
|
|
37190
38385
|
retryable: true,
|
|
37191
38386
|
httpStatus: 503,
|
|
37192
38387
|
terminalStatus: "failed"
|
|
@@ -37196,7 +38391,7 @@ function classifyHarvestProblem(err) {
|
|
|
37196
38391
|
return {
|
|
37197
38392
|
error_code: "proxy_tunnel_failed",
|
|
37198
38393
|
error_type: "proxy_tunnel_failed",
|
|
37199
|
-
message,
|
|
38394
|
+
message: publicErrorMessage("proxy_tunnel_failed"),
|
|
37200
38395
|
retryable: true,
|
|
37201
38396
|
httpStatus: 503,
|
|
37202
38397
|
terminalStatus: "failed"
|
|
@@ -37206,7 +38401,7 @@ function classifyHarvestProblem(err) {
|
|
|
37206
38401
|
return {
|
|
37207
38402
|
error_code: "proxy_unavailable",
|
|
37208
38403
|
error_type: "proxy_unavailable",
|
|
37209
|
-
message,
|
|
38404
|
+
message: publicErrorMessage("proxy_unavailable"),
|
|
37210
38405
|
retryable: true,
|
|
37211
38406
|
httpStatus: 503,
|
|
37212
38407
|
terminalStatus: "failed"
|
|
@@ -37216,16 +38411,26 @@ function classifyHarvestProblem(err) {
|
|
|
37216
38411
|
return {
|
|
37217
38412
|
error_code: "harvest_timeout",
|
|
37218
38413
|
error_type: "timeout",
|
|
37219
|
-
message,
|
|
38414
|
+
message: publicErrorMessage("harvest_timeout"),
|
|
37220
38415
|
retryable: true,
|
|
37221
38416
|
httpStatus: 504,
|
|
37222
38417
|
terminalStatus: "failed"
|
|
37223
38418
|
};
|
|
37224
38419
|
}
|
|
38420
|
+
if (looksLikeInternalServiceUnavailable(message)) {
|
|
38421
|
+
return {
|
|
38422
|
+
error_code: "service_unavailable",
|
|
38423
|
+
error_type: "service_unavailable",
|
|
38424
|
+
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
38425
|
+
retryable: true,
|
|
38426
|
+
httpStatus: 503,
|
|
38427
|
+
terminalStatus: "failed"
|
|
38428
|
+
};
|
|
38429
|
+
}
|
|
37225
38430
|
return {
|
|
37226
38431
|
error_code: "extraction_failed",
|
|
37227
38432
|
error_type: "extraction",
|
|
37228
|
-
message,
|
|
38433
|
+
message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
|
|
37229
38434
|
retryable: false,
|
|
37230
38435
|
httpStatus: 500,
|
|
37231
38436
|
terminalStatus: "failed"
|
|
@@ -37740,21 +38945,21 @@ async function logRequestEventBestEffort(input) {
|
|
|
37740
38945
|
}
|
|
37741
38946
|
}
|
|
37742
38947
|
function captureBillingKeys(userId, suppliedKey, body) {
|
|
37743
|
-
const responseKey = suppliedKey?.trim() || (0,
|
|
37744
|
-
const requestFingerprint2 = (0,
|
|
37745
|
-
const keyDigest = (0,
|
|
38948
|
+
const responseKey = suppliedKey?.trim() || (0, import_node_crypto21.randomUUID)();
|
|
38949
|
+
const requestFingerprint2 = (0, import_node_crypto21.createHash)("sha256").update(JSON.stringify(body)).digest("hex");
|
|
38950
|
+
const keyDigest = (0, import_node_crypto21.createHash)("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
37746
38951
|
return {
|
|
37747
38952
|
responseKey,
|
|
37748
38953
|
debitKey: `serp-capture:${userId}:${keyDigest}`,
|
|
37749
38954
|
debitDescription: `${body.query} [request:${requestFingerprint2}]`
|
|
37750
38955
|
};
|
|
37751
38956
|
}
|
|
37752
|
-
var import_hono19,
|
|
38957
|
+
var import_hono19, import_node_crypto21, SERP_INTELLIGENCE_RATE_LIMIT, SERP_INTELLIGENCE_RATE_WINDOW_SECONDS, POST_CAPTURE_ROUTE_LABEL, POST_PAGE_SNAPSHOTS_ROUTE_LABEL, SERP_CAPTURE_BILLING_SOURCE, serpIntelligenceApp;
|
|
37753
38958
|
var init_serp_intelligence_routes = __esm({
|
|
37754
38959
|
"src/api/serp-intelligence-routes.ts"() {
|
|
37755
38960
|
"use strict";
|
|
37756
38961
|
import_hono19 = require("hono");
|
|
37757
|
-
|
|
38962
|
+
import_node_crypto21 = require("crypto");
|
|
37758
38963
|
init_browser_service_env();
|
|
37759
38964
|
init_page_snapshot_extractor();
|
|
37760
38965
|
init_serp_capture_service();
|
|
@@ -37991,7 +39196,7 @@ var PACKAGE_VERSION;
|
|
|
37991
39196
|
var init_version = __esm({
|
|
37992
39197
|
"src/version.ts"() {
|
|
37993
39198
|
"use strict";
|
|
37994
|
-
PACKAGE_VERSION = "0.40.
|
|
39199
|
+
PACKAGE_VERSION = "0.40.3";
|
|
37995
39200
|
}
|
|
37996
39201
|
});
|
|
37997
39202
|
|
|
@@ -40799,7 +42004,7 @@ var init_editorial_reading_room = __esm({
|
|
|
40799
42004
|
|
|
40800
42005
|
// src/mcp/paa-mcp-server.ts
|
|
40801
42006
|
function hashOwnerId(callerKey) {
|
|
40802
|
-
return (0,
|
|
42007
|
+
return (0, import_node_crypto22.createHash)("sha256").update(callerKey).digest("hex").slice(0, 24);
|
|
40803
42008
|
}
|
|
40804
42009
|
function liveWebToolAnnotations(title) {
|
|
40805
42010
|
return {
|
|
@@ -41347,14 +42552,14 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
41347
42552
|
annotations: { title: "Set Scheduled Action Connections", readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }
|
|
41348
42553
|
}, async (input) => executor.setScheduledActionConnections(input));
|
|
41349
42554
|
}
|
|
41350
|
-
var import_mcp, import_node_fs11, import_node_path15,
|
|
42555
|
+
var import_mcp, import_node_fs11, import_node_path15, import_node_crypto22;
|
|
41351
42556
|
var init_paa_mcp_server = __esm({
|
|
41352
42557
|
"src/mcp/paa-mcp-server.ts"() {
|
|
41353
42558
|
"use strict";
|
|
41354
42559
|
import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
41355
42560
|
import_node_fs11 = require("fs");
|
|
41356
42561
|
import_node_path15 = require("path");
|
|
41357
|
-
|
|
42562
|
+
import_node_crypto22 = require("crypto");
|
|
41358
42563
|
init_version();
|
|
41359
42564
|
init_mcp_response_formatter();
|
|
41360
42565
|
init_server_instructions();
|
|
@@ -41403,7 +42608,20 @@ async function readResponseData(res) {
|
|
|
41403
42608
|
function httpErrorPayload(path6, res, data) {
|
|
41404
42609
|
const objectData = data && typeof data === "object" && !Array.isArray(data) ? data : null;
|
|
41405
42610
|
const rawCode = objectData?.code ?? objectData?.errorCode ?? objectData?.error_code;
|
|
41406
|
-
const
|
|
42611
|
+
const candidateCode = typeof rawCode === "string" && /^[a-z0-9][a-z0-9_-]{0,99}$/.test(rawCode) ? rawCode : "mcp_http_error";
|
|
42612
|
+
if (res.status >= 500) {
|
|
42613
|
+
const safeCode2 = normalizePublicErrorCode(candidateCode);
|
|
42614
|
+
return {
|
|
42615
|
+
error: safeCode2,
|
|
42616
|
+
error_code: safeCode2,
|
|
42617
|
+
error_type: "http",
|
|
42618
|
+
retryable: true,
|
|
42619
|
+
status: res.status,
|
|
42620
|
+
path: path6,
|
|
42621
|
+
message: publicErrorMessage(safeCode2)
|
|
42622
|
+
};
|
|
42623
|
+
}
|
|
42624
|
+
const safeCode = candidateCode;
|
|
41407
42625
|
const bodyMessage = objectData ? objectData.message ?? objectData.error ?? rawCode : data;
|
|
41408
42626
|
return {
|
|
41409
42627
|
...objectData ?? { body: data },
|
|
@@ -41426,13 +42644,15 @@ function withDefaultHostedProfile(input) {
|
|
|
41426
42644
|
...typeof saveProfileChanges === "boolean" ? { saveProfileChanges } : {}
|
|
41427
42645
|
};
|
|
41428
42646
|
}
|
|
41429
|
-
var
|
|
42647
|
+
var import_node_crypto23, HttpMcpToolExecutor;
|
|
41430
42648
|
var init_http_mcp_tool_executor = __esm({
|
|
41431
42649
|
"src/mcp/http-mcp-tool-executor.ts"() {
|
|
41432
42650
|
"use strict";
|
|
41433
|
-
|
|
42651
|
+
import_node_crypto23 = require("crypto");
|
|
41434
42652
|
init_harvest_timeout();
|
|
41435
42653
|
init_browser_service_env();
|
|
42654
|
+
init_errors();
|
|
42655
|
+
init_outbound_sanitize();
|
|
41436
42656
|
HttpMcpToolExecutor = class {
|
|
41437
42657
|
baseUrl;
|
|
41438
42658
|
apiKey;
|
|
@@ -41469,7 +42689,6 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
41469
42689
|
}
|
|
41470
42690
|
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
41471
42691
|
} catch (err) {
|
|
41472
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
41473
42692
|
if (err instanceof DOMException && err.name === "TimeoutError") {
|
|
41474
42693
|
return {
|
|
41475
42694
|
content: [{
|
|
@@ -41486,17 +42705,17 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
41486
42705
|
isError: true
|
|
41487
42706
|
};
|
|
41488
42707
|
}
|
|
41489
|
-
return { content: [{ type: "text", text:
|
|
42708
|
+
return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
|
|
41490
42709
|
}
|
|
41491
42710
|
}
|
|
41492
42711
|
async callConnectedMutation(path6, body, timeoutMs = this.timeoutMs) {
|
|
41493
|
-
const fingerprint = (0,
|
|
42712
|
+
const fingerprint = (0, import_node_crypto23.createHash)("sha256").update("POST").update("\0").update(path6).update("\0").update(JSON.stringify(body)).digest("hex");
|
|
41494
42713
|
const now = Date.now();
|
|
41495
42714
|
for (const [pendingFingerprint, pendingEntry] of this.pendingConnectedMutationKeys) {
|
|
41496
42715
|
if (pendingEntry.expiresAt <= now) this.pendingConnectedMutationKeys.delete(pendingFingerprint);
|
|
41497
42716
|
}
|
|
41498
42717
|
const pending = this.pendingConnectedMutationKeys.get(fingerprint);
|
|
41499
|
-
const idempotencyKey2 = pending && pending.expiresAt > now ? pending.key : (0,
|
|
42718
|
+
const idempotencyKey2 = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto23.randomUUID)();
|
|
41500
42719
|
this.pendingConnectedMutationKeys.set(fingerprint, {
|
|
41501
42720
|
key: idempotencyKey2,
|
|
41502
42721
|
expiresAt: now + 15 * 6e4
|
|
@@ -41510,13 +42729,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
41510
42729
|
return result;
|
|
41511
42730
|
}
|
|
41512
42731
|
async callDirectoryWorkflowStart(body, explicitIdempotencyKey) {
|
|
41513
|
-
const idempotencyKey2 = `mcp-directory-${(0,
|
|
42732
|
+
const idempotencyKey2 = `mcp-directory-${(0, import_node_crypto23.createHash)("sha256").update(explicitIdempotencyKey).digest("hex")}`;
|
|
41514
42733
|
return this.call("/directory/run", body, this.timeoutMs, "POST", {
|
|
41515
42734
|
"Idempotency-Key": idempotencyKey2
|
|
41516
42735
|
});
|
|
41517
42736
|
}
|
|
41518
42737
|
async callSiteExtractStart(toolName, body, explicitIdempotencyKey) {
|
|
41519
|
-
const idempotencyKey2 = `mcp-site-${(0,
|
|
42738
|
+
const idempotencyKey2 = `mcp-site-${(0, import_node_crypto23.createHash)("sha256").update(toolName).update("\0").update(explicitIdempotencyKey).digest("hex")}`;
|
|
41520
42739
|
return this.call("/extract-site", body, this.timeoutMs, "POST", {
|
|
41521
42740
|
"Idempotency-Key": idempotencyKey2
|
|
41522
42741
|
});
|
|
@@ -41535,9 +42754,8 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
41535
42754
|
return { content: [{ type: "text", text: JSON.stringify(httpErrorPayload(path6, res, data)) }], isError: true };
|
|
41536
42755
|
}
|
|
41537
42756
|
return { content: [{ type: "text", text: JSON.stringify(data) }] };
|
|
41538
|
-
} catch
|
|
41539
|
-
|
|
41540
|
-
return { content: [{ type: "text", text: msg }], isError: true };
|
|
42757
|
+
} catch {
|
|
42758
|
+
return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
|
|
41541
42759
|
}
|
|
41542
42760
|
}
|
|
41543
42761
|
async getTextArtifact(path6, maxBytes, timeoutMs = this.timeoutMs) {
|
|
@@ -41551,7 +42769,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
41551
42769
|
});
|
|
41552
42770
|
if (!res.ok) {
|
|
41553
42771
|
const data = await res.json().catch(async () => ({ error: await res.text().catch(() => `HTTP ${res.status}`) }));
|
|
41554
|
-
return {
|
|
42772
|
+
return {
|
|
42773
|
+
content: [{
|
|
42774
|
+
type: "text",
|
|
42775
|
+
text: JSON.stringify(res.status >= 500 ? httpErrorPayload(path6, res, data) : sanitizeOutboundDiagnostics(data))
|
|
42776
|
+
}],
|
|
42777
|
+
isError: true
|
|
42778
|
+
};
|
|
41555
42779
|
}
|
|
41556
42780
|
const bytes = Buffer.from(await res.arrayBuffer());
|
|
41557
42781
|
const sliced = bytes.subarray(0, Math.min(maxBytes, bytes.length));
|
|
@@ -41568,8 +42792,7 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
41568
42792
|
}]
|
|
41569
42793
|
};
|
|
41570
42794
|
} catch (err) {
|
|
41571
|
-
|
|
41572
|
-
return { content: [{ type: "text", text: msg }], isError: true };
|
|
42795
|
+
return { content: [{ type: "text", text: publicErrorMessage("service_unavailable") }], isError: true };
|
|
41573
42796
|
}
|
|
41574
42797
|
}
|
|
41575
42798
|
harvestPaa(input) {
|
|
@@ -41789,13 +43012,13 @@ var init_http_mcp_tool_executor = __esm({
|
|
|
41789
43012
|
return this.call("/editorial-reading-room/renew-download", input);
|
|
41790
43013
|
}
|
|
41791
43014
|
async captureSerpSnapshot(input) {
|
|
41792
|
-
const fingerprint = (0,
|
|
43015
|
+
const fingerprint = (0, import_node_crypto23.createHash)("sha256").update(JSON.stringify(input)).digest("hex");
|
|
41793
43016
|
const now = Date.now();
|
|
41794
43017
|
for (const [pendingFingerprint, pendingEntry] of this.pendingSerpCaptureBillingKeys) {
|
|
41795
43018
|
if (pendingEntry.expiresAt <= now) this.pendingSerpCaptureBillingKeys.delete(pendingFingerprint);
|
|
41796
43019
|
}
|
|
41797
43020
|
const pending = this.pendingSerpCaptureBillingKeys.get(fingerprint);
|
|
41798
|
-
const idempotencyKey2 = pending && pending.expiresAt > now ? pending.key : (0,
|
|
43021
|
+
const idempotencyKey2 = pending && pending.expiresAt > now ? pending.key : (0, import_node_crypto23.randomUUID)();
|
|
41799
43022
|
this.pendingSerpCaptureBillingKeys.set(fingerprint, {
|
|
41800
43023
|
key: idempotencyKey2,
|
|
41801
43024
|
expiresAt: now + 15 * 6e4
|
|
@@ -42632,8 +43855,7 @@ function errorResult(tool, value, sessionId = null, replayId = null) {
|
|
|
42632
43855
|
tool,
|
|
42633
43856
|
session_id: sessionId,
|
|
42634
43857
|
...replayId !== null ? { replay_id: replayId } : {},
|
|
42635
|
-
error: errorMessage4(value)
|
|
42636
|
-
raw: value && typeof value === "object" ? value : { value }
|
|
43858
|
+
error: errorMessage4(value)
|
|
42637
43859
|
}, true);
|
|
42638
43860
|
}
|
|
42639
43861
|
function actionResult(tool, sessionId, ok, data, nextRecommendedTool = "browser_screenshot") {
|
|
@@ -42718,8 +43940,8 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
42718
43940
|
});
|
|
42719
43941
|
const data = await res.json().catch(() => ({}));
|
|
42720
43942
|
return { ok: res.ok, data };
|
|
42721
|
-
} catch
|
|
42722
|
-
return { ok: false, data: { error:
|
|
43943
|
+
} catch {
|
|
43944
|
+
return { ok: false, data: { error: publicErrorMessage("service_unavailable") } };
|
|
42723
43945
|
}
|
|
42724
43946
|
}
|
|
42725
43947
|
async function downloadReplay(sessionId, replayId, filename) {
|
|
@@ -42748,8 +43970,8 @@ function registerBrowserAgentMcpTools(server, opts) {
|
|
|
42748
43970
|
download_url: `${baseUrl}${path6}`
|
|
42749
43971
|
}
|
|
42750
43972
|
};
|
|
42751
|
-
} catch
|
|
42752
|
-
return { ok: false, data: { error:
|
|
43973
|
+
} catch {
|
|
43974
|
+
return { ok: false, data: { error: publicErrorMessage("service_unavailable") } };
|
|
42753
43975
|
}
|
|
42754
43976
|
}
|
|
42755
43977
|
const annotations = (title, readOnly = false) => ({
|
|
@@ -43421,6 +44643,7 @@ var init_browser_agent_mcp_server = __esm({
|
|
|
43421
44643
|
init_browser_agent_tool_schemas();
|
|
43422
44644
|
init_replay_annotator();
|
|
43423
44645
|
init_outbound_sanitize();
|
|
44646
|
+
init_errors();
|
|
43424
44647
|
init_output_schema_registry();
|
|
43425
44648
|
}
|
|
43426
44649
|
});
|
|
@@ -46812,7 +48035,7 @@ async function runBrowserAgentMigration() {
|
|
|
46812
48035
|
}
|
|
46813
48036
|
async function createExtensionRow(input) {
|
|
46814
48037
|
const db = getDb();
|
|
46815
|
-
const id = `bext_${(0,
|
|
48038
|
+
const id = `bext_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
46816
48039
|
await db.execute({
|
|
46817
48040
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
46818
48041
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -46847,7 +48070,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
46847
48070
|
}
|
|
46848
48071
|
async function createAuthConnectionRow(input) {
|
|
46849
48072
|
const db = getDb();
|
|
46850
|
-
const connectionId = `authc_${(0,
|
|
48073
|
+
const connectionId = `authc_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
46851
48074
|
await db.execute({
|
|
46852
48075
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
46853
48076
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -46932,7 +48155,7 @@ async function deleteProfileLabel(userId, profile) {
|
|
|
46932
48155
|
}
|
|
46933
48156
|
async function createSessionRow(input) {
|
|
46934
48157
|
const db = getDb();
|
|
46935
|
-
const id = `bas_${(0,
|
|
48158
|
+
const id = `bas_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "").slice(0, 20)}`;
|
|
46936
48159
|
await db.execute({
|
|
46937
48160
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
46938
48161
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -47001,7 +48224,7 @@ async function recordAction(input) {
|
|
|
47001
48224
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
47002
48225
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
47003
48226
|
args: [
|
|
47004
|
-
`baa_${(0,
|
|
48227
|
+
`baa_${(0, import_node_crypto24.randomUUID)().replace(/-/g, "").slice(0, 20)}`,
|
|
47005
48228
|
input.sessionId,
|
|
47006
48229
|
input.type,
|
|
47007
48230
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -47041,11 +48264,11 @@ async function listReplayRows(sessionId) {
|
|
|
47041
48264
|
});
|
|
47042
48265
|
return res.rows;
|
|
47043
48266
|
}
|
|
47044
|
-
var
|
|
48267
|
+
var import_node_crypto24, _ready2, _migrationPromise2;
|
|
47045
48268
|
var init_browser_agent_db = __esm({
|
|
47046
48269
|
"src/api/browser-agent-db.ts"() {
|
|
47047
48270
|
"use strict";
|
|
47048
|
-
|
|
48271
|
+
import_node_crypto24 = require("crypto");
|
|
47049
48272
|
init_db();
|
|
47050
48273
|
_ready2 = false;
|
|
47051
48274
|
_migrationPromise2 = null;
|
|
@@ -47807,7 +49030,7 @@ function keepHostedBrowserAlive(_browser) {
|
|
|
47807
49030
|
function client() {
|
|
47808
49031
|
const apiKey = browserServiceApiKey();
|
|
47809
49032
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
47810
|
-
return new
|
|
49033
|
+
return new import_sdk9.default({ apiKey });
|
|
47811
49034
|
}
|
|
47812
49035
|
function isProfileConflict(err) {
|
|
47813
49036
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -48165,11 +49388,11 @@ async function replayList(runtimeSessionId) {
|
|
|
48165
49388
|
finishedAt: r.finished_at ?? null
|
|
48166
49389
|
}));
|
|
48167
49390
|
}
|
|
48168
|
-
var
|
|
49391
|
+
var import_sdk9, import_playwright5, DEFAULT_TIMEOUT_SECONDS;
|
|
48169
49392
|
var init_browser_agent_service = __esm({
|
|
48170
49393
|
"src/services/browser-agent/browser-agent-service.ts"() {
|
|
48171
49394
|
"use strict";
|
|
48172
|
-
|
|
49395
|
+
import_sdk9 = __toESM(require("@onkernel/sdk"), 1);
|
|
48173
49396
|
import_playwright5 = require("playwright");
|
|
48174
49397
|
init_browser_service_env();
|
|
48175
49398
|
init_run_capture();
|
|
@@ -48801,7 +50024,7 @@ function buildBrowserAgentRoutes() {
|
|
|
48801
50024
|
}
|
|
48802
50025
|
const existing = await getExtensionRow(user.id, name);
|
|
48803
50026
|
if (existing) return c.json({ error: `an extension named "${name}" already exists \u2014 delete it first or pick another name` }, 409);
|
|
48804
|
-
const backendName = `u${user.id}_${(0,
|
|
50027
|
+
const backendName = `u${user.id}_${(0, import_node_crypto25.randomUUID)().replace(/-/g, "")}`;
|
|
48805
50028
|
try {
|
|
48806
50029
|
const imported = await importExtensionFromStore(storeUrl, backendName);
|
|
48807
50030
|
const row = await createExtensionRow({
|
|
@@ -48860,11 +50083,11 @@ function buildBrowserAgentRoutes() {
|
|
|
48860
50083
|
});
|
|
48861
50084
|
return app2;
|
|
48862
50085
|
}
|
|
48863
|
-
var
|
|
50086
|
+
var import_node_crypto25, import_hono21, auth, DEFAULT_BROWSER_SESSION_LOCK_TTL_SECONDS, EXTENSION_NAME_RE;
|
|
48864
50087
|
var init_browser_agent_routes = __esm({
|
|
48865
50088
|
"src/api/browser-agent-routes.ts"() {
|
|
48866
50089
|
"use strict";
|
|
48867
|
-
|
|
50090
|
+
import_node_crypto25 = require("crypto");
|
|
48868
50091
|
import_hono21 = require("hono");
|
|
48869
50092
|
init_api_auth();
|
|
48870
50093
|
init_errors();
|
|
@@ -49343,7 +50566,7 @@ async function getKeys() {
|
|
|
49343
50566
|
const privateKey = await (0, import_jose2.importPKCS8)(pem, "RS256", { extractable: true });
|
|
49344
50567
|
const full = await (0, import_jose2.exportJWK)(privateKey);
|
|
49345
50568
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
49346
|
-
const kid = (0,
|
|
50569
|
+
const kid = (0, import_node_crypto26.createHash)("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
49347
50570
|
publicJwk.kid = kid;
|
|
49348
50571
|
publicJwk.alg = "RS256";
|
|
49349
50572
|
publicJwk.use = "sig";
|
|
@@ -49522,23 +50745,23 @@ async function validateAuthRequest(p) {
|
|
|
49522
50745
|
}
|
|
49523
50746
|
function pkceMatches(verifier, challenge) {
|
|
49524
50747
|
if (!verifier) return false;
|
|
49525
|
-
const computed = (0,
|
|
50748
|
+
const computed = (0, import_node_crypto26.createHash)("sha256").update(verifier).digest("base64url");
|
|
49526
50749
|
return computed === challenge;
|
|
49527
50750
|
}
|
|
49528
50751
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
49529
50752
|
const { privateKey, kid } = await getKeys();
|
|
49530
|
-
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0,
|
|
50753
|
+
return new import_jose2.SignJWT({ scope, plan }).setProtectedHeader({ alg: "RS256", kid }).setIssuer(ISSUER).setSubject(identity).setAudience(audience).setIssuedAt().setJti((0, import_node_crypto26.randomUUID)()).setExpirationTime(`${ACCESS_TTL_SECONDS}s`).sign(privateKey);
|
|
49531
50754
|
}
|
|
49532
50755
|
function tokenErrorResponse(c, error, description, status) {
|
|
49533
50756
|
return c.json({ error, error_description: description }, status);
|
|
49534
50757
|
}
|
|
49535
|
-
var import_hono23, import_cookie,
|
|
50758
|
+
var import_hono23, import_cookie, import_node_crypto26, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
|
|
49536
50759
|
var init_oauth_routes = __esm({
|
|
49537
50760
|
"src/api/oauth-routes.ts"() {
|
|
49538
50761
|
"use strict";
|
|
49539
50762
|
import_hono23 = require("hono");
|
|
49540
50763
|
import_cookie = require("hono/cookie");
|
|
49541
|
-
|
|
50764
|
+
import_node_crypto26 = require("crypto");
|
|
49542
50765
|
import_jose2 = require("jose");
|
|
49543
50766
|
init_session();
|
|
49544
50767
|
init_db();
|
|
@@ -49624,7 +50847,7 @@ var init_oauth_routes = __esm({
|
|
|
49624
50847
|
}
|
|
49625
50848
|
}
|
|
49626
50849
|
const clientName = typeof body.client_name === "string" ? body.client_name : null;
|
|
49627
|
-
const clientId = `client_${(0,
|
|
50850
|
+
const clientId = `client_${(0, import_node_crypto26.randomBytes)(16).toString("hex")}`;
|
|
49628
50851
|
await registerClient(clientId, redirectUris, clientName);
|
|
49629
50852
|
console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
|
|
49630
50853
|
return c.json({
|
|
@@ -49677,7 +50900,7 @@ var init_oauth_routes = __esm({
|
|
|
49677
50900
|
if (action === "deny") return redirectWithError(p.redirect_uri, p.state, "access_denied");
|
|
49678
50901
|
if (action !== "approve") return c.text("unsupported action", 400);
|
|
49679
50902
|
const scope = negotiateScope(p.scope, user, p.resource);
|
|
49680
|
-
const code = `code_${(0,
|
|
50903
|
+
const code = `code_${(0, import_node_crypto26.randomBytes)(32).toString("base64url")}`;
|
|
49681
50904
|
const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
|
|
49682
50905
|
await putCode({
|
|
49683
50906
|
code,
|
|
@@ -49716,7 +50939,7 @@ var init_oauth_routes = __esm({
|
|
|
49716
50939
|
const plan = user ? resolvePlan(user) : "free";
|
|
49717
50940
|
const audience = record.resource ?? RESOURCE();
|
|
49718
50941
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
49719
|
-
const refreshToken = `rt_${(0,
|
|
50942
|
+
const refreshToken = `rt_${(0, import_node_crypto26.randomBytes)(40).toString("base64url")}`;
|
|
49720
50943
|
await putRefresh({
|
|
49721
50944
|
refresh_token: refreshToken,
|
|
49722
50945
|
client_id: clientId,
|
|
@@ -49746,7 +50969,7 @@ var init_oauth_routes = __esm({
|
|
|
49746
50969
|
const plan = user ? resolvePlan(user) : "free";
|
|
49747
50970
|
const audience = record.resource ?? RESOURCE();
|
|
49748
50971
|
const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
|
|
49749
|
-
const nextRefresh = `rt_${(0,
|
|
50972
|
+
const nextRefresh = `rt_${(0, import_node_crypto26.randomBytes)(40).toString("base64url")}`;
|
|
49750
50973
|
await rotateRefresh(refreshToken, {
|
|
49751
50974
|
refresh_token: nextRefresh,
|
|
49752
50975
|
client_id: record.client_id,
|
|
@@ -51166,7 +52389,7 @@ function quoteUntrusted(value) {
|
|
|
51166
52389
|
return clean2.split("\n").map((line) => `> ${line}`).join("\n");
|
|
51167
52390
|
}
|
|
51168
52391
|
function shortHash(value, length = 24) {
|
|
51169
|
-
return (0,
|
|
52392
|
+
return (0, import_node_crypto27.createHash)("sha256").update(value).digest("hex").slice(0, length);
|
|
51170
52393
|
}
|
|
51171
52394
|
function safePathPart(value) {
|
|
51172
52395
|
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "contact";
|
|
@@ -51654,11 +52877,11 @@ async function captureSupportMessage(memoryKey, email, options) {
|
|
|
51654
52877
|
direction: options.direction
|
|
51655
52878
|
};
|
|
51656
52879
|
}
|
|
51657
|
-
var
|
|
52880
|
+
var import_node_crypto27, SUPPORT_TAGS, ISSUE_TAGS;
|
|
51658
52881
|
var init_resend_support_thread = __esm({
|
|
51659
52882
|
"src/api/resend-support-thread.ts"() {
|
|
51660
52883
|
"use strict";
|
|
51661
|
-
|
|
52884
|
+
import_node_crypto27 = require("crypto");
|
|
51662
52885
|
init_memory();
|
|
51663
52886
|
SUPPORT_TAGS = [
|
|
51664
52887
|
{
|
|
@@ -52243,7 +53466,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
52243
53466
|
) < ?
|
|
52244
53467
|
`,
|
|
52245
53468
|
args: [
|
|
52246
|
-
(0,
|
|
53469
|
+
(0, import_node_crypto28.randomUUID)(),
|
|
52247
53470
|
input.eventSlug,
|
|
52248
53471
|
input.email,
|
|
52249
53472
|
input.firstName,
|
|
@@ -52260,7 +53483,7 @@ async function claimWorkshopRegistration(input) {
|
|
|
52260
53483
|
id, event_slug, email, first_name, last_name, status
|
|
52261
53484
|
) VALUES (?, ?, ?, ?, ?, 'waitlisted')
|
|
52262
53485
|
`,
|
|
52263
|
-
args: [(0,
|
|
53486
|
+
args: [(0, import_node_crypto28.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
52264
53487
|
});
|
|
52265
53488
|
const waitlisted = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
52266
53489
|
if (!waitlisted) throw new Error("workshop registration could not be claimed");
|
|
@@ -52275,7 +53498,7 @@ async function registerWorkshopInterest(input) {
|
|
|
52275
53498
|
id, event_slug, email, first_name, last_name, status
|
|
52276
53499
|
) VALUES (?, ?, ?, ?, ?, 'interested')
|
|
52277
53500
|
`,
|
|
52278
|
-
args: [(0,
|
|
53501
|
+
args: [(0, import_node_crypto28.randomUUID)(), input.eventSlug, input.email, input.firstName, input.lastName || null]
|
|
52279
53502
|
});
|
|
52280
53503
|
const registration = await getWorkshopRegistration(input.eventSlug, input.email);
|
|
52281
53504
|
if (!registration) throw new Error("workshop interest could not be recorded");
|
|
@@ -52305,11 +53528,11 @@ async function updateWorkshopRegistration(id, patch) {
|
|
|
52305
53528
|
args: [...entries.map(([, value]) => value), id]
|
|
52306
53529
|
});
|
|
52307
53530
|
}
|
|
52308
|
-
var
|
|
53531
|
+
var import_node_crypto28;
|
|
52309
53532
|
var init_workshop_registration_repository = __esm({
|
|
52310
53533
|
"src/api/workshop-registration-repository.ts"() {
|
|
52311
53534
|
"use strict";
|
|
52312
|
-
|
|
53535
|
+
import_node_crypto28 = require("crypto");
|
|
52313
53536
|
init_db();
|
|
52314
53537
|
}
|
|
52315
53538
|
});
|
|
@@ -52854,15 +54077,15 @@ function renderEditorialReadingRoom(input, now = /* @__PURE__ */ new Date()) {
|
|
|
52854
54077
|
articleCount: articles.length,
|
|
52855
54078
|
wordCount: totalWordCount,
|
|
52856
54079
|
bytes,
|
|
52857
|
-
sha256: (0,
|
|
54080
|
+
sha256: (0, import_node_crypto29.createHash)("sha256").update(html).digest("hex"),
|
|
52858
54081
|
warnings
|
|
52859
54082
|
};
|
|
52860
54083
|
}
|
|
52861
|
-
var
|
|
54084
|
+
var import_node_crypto29, import_node_fs14, import_node_path19, import_marked, runtimeEntryDir, assetCache;
|
|
52862
54085
|
var init_render = __esm({
|
|
52863
54086
|
"src/editorial-reading-room/render.ts"() {
|
|
52864
54087
|
"use strict";
|
|
52865
|
-
|
|
54088
|
+
import_node_crypto29 = require("crypto");
|
|
52866
54089
|
import_node_fs14 = require("fs");
|
|
52867
54090
|
import_node_path19 = require("path");
|
|
52868
54091
|
import_marked = require("marked");
|
|
@@ -53137,7 +54360,7 @@ var init_site_extract_reconciliation = __esm({
|
|
|
53137
54360
|
|
|
53138
54361
|
// src/api/page-diff.ts
|
|
53139
54362
|
function sha256Hex(value) {
|
|
53140
|
-
return (0,
|
|
54363
|
+
return (0, import_node_crypto30.createHash)("sha256").update(value).digest("hex");
|
|
53141
54364
|
}
|
|
53142
54365
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
53143
54366
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -53194,11 +54417,11 @@ function diffPageContent(oldContent, newContent) {
|
|
|
53194
54417
|
totalChangedLineCount
|
|
53195
54418
|
};
|
|
53196
54419
|
}
|
|
53197
|
-
var
|
|
54420
|
+
var import_node_crypto30, import_diff, MAX_SNAPSHOT_CONTENT_CHARS, MAX_DIFF_HUNKS, MAX_DIFF_LINES_PER_RESPONSE;
|
|
53198
54421
|
var init_page_diff = __esm({
|
|
53199
54422
|
"src/api/page-diff.ts"() {
|
|
53200
54423
|
"use strict";
|
|
53201
|
-
|
|
54424
|
+
import_node_crypto30 = require("crypto");
|
|
53202
54425
|
import_diff = require("diff");
|
|
53203
54426
|
MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
53204
54427
|
MAX_DIFF_HUNKS = 200;
|
|
@@ -53697,7 +54920,7 @@ function canonicalJson(value) {
|
|
|
53697
54920
|
return JSON.stringify(value);
|
|
53698
54921
|
}
|
|
53699
54922
|
function sha2562(value) {
|
|
53700
|
-
return (0,
|
|
54923
|
+
return (0, import_node_crypto31.createHash)("sha256").update(value).digest("hex");
|
|
53701
54924
|
}
|
|
53702
54925
|
function sensitiveKey(key) {
|
|
53703
54926
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -53912,11 +55135,11 @@ async function importServiceConnectionToMemory(identity, input, dependencies) {
|
|
|
53912
55135
|
...!searchReady ? { warning: "The snapshot was stored, but no search chunks were indexed yet." } : {}
|
|
53913
55136
|
};
|
|
53914
55137
|
}
|
|
53915
|
-
var
|
|
55138
|
+
var import_node_crypto31, CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES, CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES, CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS, CONNECTION_MEMORY_IMPORT_MAX_DEPTH, ConnectionMemoryImportError;
|
|
53916
55139
|
var init_connection_memory_import = __esm({
|
|
53917
55140
|
"src/api/connection-memory-import.ts"() {
|
|
53918
55141
|
"use strict";
|
|
53919
|
-
|
|
55142
|
+
import_node_crypto31 = require("crypto");
|
|
53920
55143
|
init_slugify();
|
|
53921
55144
|
CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
53922
55145
|
CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
@@ -54090,7 +55313,7 @@ function finitePositive(value, fallback) {
|
|
|
54090
55313
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
54091
55314
|
}
|
|
54092
55315
|
async function collectConnectedDataExport(args) {
|
|
54093
|
-
const exportId = (0,
|
|
55316
|
+
const exportId = (0, import_node_crypto32.randomUUID)();
|
|
54094
55317
|
const now = args.now ?? Date.now;
|
|
54095
55318
|
const startedAt = now();
|
|
54096
55319
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -54200,11 +55423,11 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
54200
55423
|
untrustedContent: true
|
|
54201
55424
|
};
|
|
54202
55425
|
}
|
|
54203
|
-
var
|
|
55426
|
+
var import_node_crypto32, CONNECTED_DATA_INLINE_BUDGET_BYTES, CONNECTED_DATA_MAX_EXPORT_BYTES, CONNECTED_DATA_EXPORT_BUDGET_MS, CONNECTED_DATA_PAGE_START_HEADROOM_MS, CONNECTED_DATA_DATASETS, ConnectedDataExportValidationError;
|
|
54204
55427
|
var init_connected_data_export = __esm({
|
|
54205
55428
|
"src/api/connected-data-export.ts"() {
|
|
54206
55429
|
"use strict";
|
|
54207
|
-
|
|
55430
|
+
import_node_crypto32 = require("crypto");
|
|
54208
55431
|
CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
54209
55432
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
54210
55433
|
);
|
|
@@ -54317,7 +55540,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
54317
55540
|
offset += rows.length;
|
|
54318
55541
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
54319
55542
|
}
|
|
54320
|
-
const exportId = (0,
|
|
55543
|
+
const exportId = (0, import_node_crypto33.randomUUID)();
|
|
54321
55544
|
const artifact = await args.writeArtifact({
|
|
54322
55545
|
ownerId: args.ownerId,
|
|
54323
55546
|
exportId,
|
|
@@ -54339,11 +55562,11 @@ async function exportSearchConsoleTableData(args) {
|
|
|
54339
55562
|
warnings
|
|
54340
55563
|
};
|
|
54341
55564
|
}
|
|
54342
|
-
var
|
|
55565
|
+
var import_node_crypto33, SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS, SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE, SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES, SEARCH_CONSOLE_TABLE_COLUMNS, SearchConsoleTableExportValidationError;
|
|
54343
55566
|
var init_search_console_table_export = __esm({
|
|
54344
55567
|
"src/api/search-console-table-export.ts"() {
|
|
54345
55568
|
"use strict";
|
|
54346
|
-
|
|
55569
|
+
import_node_crypto33 = require("crypto");
|
|
54347
55570
|
SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
54348
55571
|
SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
54349
55572
|
SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -54771,7 +55994,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
|
54771
55994
|
updated_at = excluded.updated_at
|
|
54772
55995
|
`,
|
|
54773
55996
|
args: [
|
|
54774
|
-
(0,
|
|
55997
|
+
(0, import_node_crypto34.randomUUID)(),
|
|
54775
55998
|
userId,
|
|
54776
55999
|
connection.providerConfigKey,
|
|
54777
56000
|
connection.provider,
|
|
@@ -54842,7 +56065,7 @@ async function recordServiceConnectionHealth(args) {
|
|
|
54842
56065
|
});
|
|
54843
56066
|
await getDb().execute({
|
|
54844
56067
|
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
54845
|
-
args: [(0,
|
|
56068
|
+
args: [(0, import_node_crypto34.randomUUID)(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
54846
56069
|
});
|
|
54847
56070
|
}
|
|
54848
56071
|
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
@@ -54882,7 +56105,7 @@ async function claimServiceConnectionAction(args) {
|
|
|
54882
56105
|
if (!connection) throw new Error("service_connection_not_found");
|
|
54883
56106
|
const inserted = await getDb().execute({
|
|
54884
56107
|
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status) VALUES (?, ?, ?, ?, ?, 'started')`,
|
|
54885
|
-
args: [(0,
|
|
56108
|
+
args: [(0, import_node_crypto34.randomUUID)(), connection.id, connection.userId, args.tool, args.requestId]
|
|
54886
56109
|
});
|
|
54887
56110
|
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
54888
56111
|
const existing = await getDb().execute({
|
|
@@ -54895,11 +56118,11 @@ async function claimServiceConnectionAction(args) {
|
|
|
54895
56118
|
errorCode: typeof existing.rows[0]?.error_code === "string" ? existing.rows[0].error_code : null
|
|
54896
56119
|
};
|
|
54897
56120
|
}
|
|
54898
|
-
var
|
|
56121
|
+
var import_node_crypto34, schemaReady2, schemaDb3;
|
|
54899
56122
|
var init_service_connections = __esm({
|
|
54900
56123
|
"src/api/service-connections.ts"() {
|
|
54901
56124
|
"use strict";
|
|
54902
|
-
|
|
56125
|
+
import_node_crypto34 = require("crypto");
|
|
54903
56126
|
init_db();
|
|
54904
56127
|
schemaReady2 = null;
|
|
54905
56128
|
schemaDb3 = null;
|
|
@@ -55163,7 +56386,7 @@ async function withNangoClient(connection, run) {
|
|
|
55163
56386
|
}
|
|
55164
56387
|
}
|
|
55165
56388
|
});
|
|
55166
|
-
const client2 = new
|
|
56389
|
+
const client2 = new import_client11.Client({ name: "mcp-scraper-integrations", version: "1.0.0" }, { capabilities: {} });
|
|
55167
56390
|
try {
|
|
55168
56391
|
await client2.connect(transport);
|
|
55169
56392
|
const value = await run(client2);
|
|
@@ -55216,7 +56439,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
55216
56439
|
});
|
|
55217
56440
|
const readTools = tools.filter((tool) => classifyTool(tool) === "read").map((tool) => tool.name);
|
|
55218
56441
|
const actionTools = tools.filter((tool) => classifyTool(tool) === "action").map((tool) => tool.name);
|
|
55219
|
-
const revision = (0,
|
|
56442
|
+
const revision = (0, import_node_crypto35.createHash)("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
55220
56443
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
55221
56444
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
55222
56445
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -55236,8 +56459,8 @@ async function callNangoToolDirect(args) {
|
|
|
55236
56459
|
identity: args.identity,
|
|
55237
56460
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
55238
56461
|
});
|
|
55239
|
-
const requestId = args.requestId?.trim() || (0,
|
|
55240
|
-
const idempotencyKey2 = `main-nango:${(0,
|
|
56462
|
+
const requestId = args.requestId?.trim() || (0, import_node_crypto35.randomUUID)();
|
|
56463
|
+
const idempotencyKey2 = `main-nango:${(0, import_node_crypto35.createHash)("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
55241
56464
|
const startedAt = /* @__PURE__ */ new Date();
|
|
55242
56465
|
const started = performance.now();
|
|
55243
56466
|
let result;
|
|
@@ -55262,7 +56485,7 @@ async function callNangoToolDirect(args) {
|
|
|
55262
56485
|
toolName: args.tool,
|
|
55263
56486
|
operationKind: args.operationKind ?? args.classification,
|
|
55264
56487
|
outcome: providerError ? "error" : "partial",
|
|
55265
|
-
requestId: requestId.length <= 200 ? requestId : (0,
|
|
56488
|
+
requestId: requestId.length <= 200 ? requestId : (0, import_node_crypto35.createHash)("sha256").update(requestId).digest("hex"),
|
|
55266
56489
|
startedAt: startedAt.toISOString(),
|
|
55267
56490
|
completedAt: completedAt.toISOString()
|
|
55268
56491
|
}
|
|
@@ -55299,16 +56522,16 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
55299
56522
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
55300
56523
|
protocolVersion: null,
|
|
55301
56524
|
schemaSource: "live_tools_list",
|
|
55302
|
-
schemaHash: (0,
|
|
56525
|
+
schemaHash: (0, import_node_crypto35.createHash)("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
55303
56526
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
55304
56527
|
};
|
|
55305
56528
|
}
|
|
55306
|
-
var
|
|
56529
|
+
var import_node_crypto35, import_client11, import_streamableHttp, DEFAULT_NANGO_MCP_URL, NANGO_TIMEOUT_MS, NANGO_CONNECTION_PAGE_SIZE, NANGO_CONNECTION_MAX_PAGES, MAIN_INTEGRATION_CONTRACT_VERSION, MAIN_INTEGRATION_CONTRACT_HASH, MainNangoTransportError;
|
|
55307
56530
|
var init_main_nango_transport = __esm({
|
|
55308
56531
|
"src/api/main-nango-transport.ts"() {
|
|
55309
56532
|
"use strict";
|
|
55310
|
-
|
|
55311
|
-
|
|
56533
|
+
import_node_crypto35 = require("crypto");
|
|
56534
|
+
import_client11 = require("@modelcontextprotocol/sdk/client/index.js");
|
|
55312
56535
|
import_streamableHttp = require("@modelcontextprotocol/sdk/client/streamableHttp.js");
|
|
55313
56536
|
init_service_connections();
|
|
55314
56537
|
init_connected_usage_billing();
|
|
@@ -55929,7 +57152,7 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
55929
57152
|
return data.connection.actionsEnabled === true;
|
|
55930
57153
|
}
|
|
55931
57154
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey2) {
|
|
55932
|
-
const requestId = `main-connected-action:${(0,
|
|
57155
|
+
const requestId = `main-connected-action:${(0, import_node_crypto36.createHash)("sha256").update(identity).update("\0").update(idempotencyKey2?.trim() || (0, import_node_crypto36.randomUUID)()).digest("hex")}`;
|
|
55933
57156
|
if (mainOwnsIntegrations()) {
|
|
55934
57157
|
const selectedTool = tool?.trim();
|
|
55935
57158
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -56071,7 +57294,7 @@ function canonicalJson2(value) {
|
|
|
56071
57294
|
return JSON.stringify(value);
|
|
56072
57295
|
}
|
|
56073
57296
|
function projectedToolSchemaHash(tool) {
|
|
56074
|
-
return (0,
|
|
57297
|
+
return (0, import_node_crypto36.createHash)("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
56075
57298
|
}
|
|
56076
57299
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
56077
57300
|
if (mainOwnsIntegrations()) {
|
|
@@ -56332,11 +57555,11 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
56332
57555
|
untrustedContent: true
|
|
56333
57556
|
};
|
|
56334
57557
|
}
|
|
56335
|
-
var
|
|
57558
|
+
var import_node_crypto36, DEFAULT_NANGO_CONTROL_URL, DISABLED_NANGO_TOOLS, CONNECTION_SYNC_REQUIRED_TOOLS, CONNECTION_SYNC_OPTIONAL_TOOLS, NangoControlError, ScheduleConnectionValidationError, SAFE_CONTROL_ERROR_CODES, FIXED_CONTROL_ERROR_MESSAGES, CONTROL_ERROR_CODE_ALIASES;
|
|
56336
57559
|
var init_nango_control = __esm({
|
|
56337
57560
|
"src/api/nango-control.ts"() {
|
|
56338
57561
|
"use strict";
|
|
56339
|
-
|
|
57562
|
+
import_node_crypto36 = require("crypto");
|
|
56340
57563
|
init_connected_data_export();
|
|
56341
57564
|
init_main_nango_transport();
|
|
56342
57565
|
init_service_connections();
|
|
@@ -56804,8 +58027,8 @@ function signingSecret() {
|
|
|
56804
58027
|
return secret2;
|
|
56805
58028
|
}
|
|
56806
58029
|
function schedulerIntegrationSignature(args) {
|
|
56807
|
-
const bodyHash = (0,
|
|
56808
|
-
return (0,
|
|
58030
|
+
const bodyHash = (0, import_node_crypto37.createHash)("sha256").update(args.body).digest("hex");
|
|
58031
|
+
return (0, import_node_crypto37.createHmac)("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
56809
58032
|
${args.path}
|
|
56810
58033
|
${args.timestamp}
|
|
56811
58034
|
${args.nonce}
|
|
@@ -56852,17 +58075,17 @@ async function verifySchedulerIntegrationRequest(request, rawBody) {
|
|
|
56852
58075
|
});
|
|
56853
58076
|
const suppliedBytes = Buffer.from(signature, "hex");
|
|
56854
58077
|
const expectedBytes = Buffer.from(expected, "hex");
|
|
56855
|
-
if (suppliedBytes.length !== expectedBytes.length || !(0,
|
|
58078
|
+
if (suppliedBytes.length !== expectedBytes.length || !(0, import_node_crypto37.timingSafeEqual)(suppliedBytes, expectedBytes)) {
|
|
56856
58079
|
throw new SchedulerIntegrationAuthError("invalid_signature");
|
|
56857
58080
|
}
|
|
56858
58081
|
await claimNonce(nonce, timestampMs);
|
|
56859
58082
|
return { requestId };
|
|
56860
58083
|
}
|
|
56861
|
-
var
|
|
58084
|
+
var import_node_crypto37, MAX_CLOCK_SKEW_MS, SchedulerIntegrationAuthError;
|
|
56862
58085
|
var init_scheduler_integration_auth = __esm({
|
|
56863
58086
|
"src/api/scheduler-integration-auth.ts"() {
|
|
56864
58087
|
"use strict";
|
|
56865
|
-
|
|
58088
|
+
import_node_crypto37 = require("crypto");
|
|
56866
58089
|
init_db();
|
|
56867
58090
|
init_service_connections();
|
|
56868
58091
|
MAX_CLOCK_SKEW_MS = 5 * 60 * 1e3;
|
|
@@ -56877,31 +58100,6 @@ var init_scheduler_integration_auth = __esm({
|
|
|
56877
58100
|
}
|
|
56878
58101
|
});
|
|
56879
58102
|
|
|
56880
|
-
// src/api/webhook.ts
|
|
56881
|
-
async function deliverWebhook(url, payload, retries = 3) {
|
|
56882
|
-
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
56883
|
-
try {
|
|
56884
|
-
const res = await fetch(url, {
|
|
56885
|
-
method: "POST",
|
|
56886
|
-
headers: { "content-type": "application/json" },
|
|
56887
|
-
body: JSON.stringify(payload),
|
|
56888
|
-
signal: AbortSignal.timeout(1e4)
|
|
56889
|
-
});
|
|
56890
|
-
if (res.ok) return;
|
|
56891
|
-
console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
|
|
56892
|
-
} catch (err) {
|
|
56893
|
-
console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
|
|
56894
|
-
}
|
|
56895
|
-
if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
|
|
56896
|
-
}
|
|
56897
|
-
console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
|
|
56898
|
-
}
|
|
56899
|
-
var init_webhook = __esm({
|
|
56900
|
-
"src/api/webhook.ts"() {
|
|
56901
|
-
"use strict";
|
|
56902
|
-
}
|
|
56903
|
-
});
|
|
56904
|
-
|
|
56905
58103
|
// src/api/worker.ts
|
|
56906
58104
|
var worker_exports = {};
|
|
56907
58105
|
__export(worker_exports, {
|
|
@@ -56915,8 +58113,8 @@ function countPaaQuestions(result) {
|
|
|
56915
58113
|
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
56916
58114
|
return Array.isArray(value.flat) ? value.flat.length : 0;
|
|
56917
58115
|
}
|
|
56918
|
-
function paaCostForQuestionCount(
|
|
56919
|
-
return MC_COSTS.paa_base + Math.max(1,
|
|
58116
|
+
function paaCostForQuestionCount(questionCount2) {
|
|
58117
|
+
return MC_COSTS.paa_base + Math.max(1, questionCount2) * MC_COSTS.paa;
|
|
56920
58118
|
}
|
|
56921
58119
|
async function processJob(job) {
|
|
56922
58120
|
running++;
|
|
@@ -56948,9 +58146,18 @@ async function processJob(job) {
|
|
|
56948
58146
|
else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
|
|
56949
58147
|
}
|
|
56950
58148
|
if (job.callback_url) {
|
|
56951
|
-
await deliverWebhook(job.callback_url, {
|
|
58149
|
+
await deliverWebhook(job.callback_url, {
|
|
58150
|
+
job_id: job.id,
|
|
58151
|
+
status: "done",
|
|
58152
|
+
result: sanitizeHarvestResult(result),
|
|
58153
|
+
attempts: sanitizeAttempts(attempts)
|
|
58154
|
+
});
|
|
56952
58155
|
}
|
|
56953
58156
|
} catch (err) {
|
|
58157
|
+
console.error("[harvest/worker] failed", {
|
|
58158
|
+
jobId: job.id,
|
|
58159
|
+
error: err instanceof Error ? err.message : String(err)
|
|
58160
|
+
});
|
|
56954
58161
|
const problem = classifyHarvestProblem(err);
|
|
56955
58162
|
await failJob(job.id, serializeHarvestProblem(problem));
|
|
56956
58163
|
const attempts = await listHarvestAttempts(job.id, job.user_id);
|
|
@@ -56962,7 +58169,12 @@ async function processJob(job) {
|
|
|
56962
58169
|
} catch {
|
|
56963
58170
|
}
|
|
56964
58171
|
if (job.callback_url) {
|
|
56965
|
-
await deliverWebhook(job.callback_url, {
|
|
58172
|
+
await deliverWebhook(job.callback_url, {
|
|
58173
|
+
job_id: job.id,
|
|
58174
|
+
status: "failed",
|
|
58175
|
+
...harvestProblemResponse(problem),
|
|
58176
|
+
attempts: sanitizeAttempts(attempts)
|
|
58177
|
+
});
|
|
56966
58178
|
}
|
|
56967
58179
|
} finally {
|
|
56968
58180
|
running--;
|
|
@@ -57005,6 +58217,7 @@ var init_worker = __esm({
|
|
|
57005
58217
|
init_harvest_problems();
|
|
57006
58218
|
init_harvest_attempt_events();
|
|
57007
58219
|
init_cost_context();
|
|
58220
|
+
init_outbound_sanitize();
|
|
57008
58221
|
MAX_CONCURRENT = 2;
|
|
57009
58222
|
running = 0;
|
|
57010
58223
|
}
|
|
@@ -57324,8 +58537,8 @@ function countPaaQuestions2(result) {
|
|
|
57324
58537
|
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
57325
58538
|
return Array.isArray(value.flat) ? value.flat.length : 0;
|
|
57326
58539
|
}
|
|
57327
|
-
function paaCostForQuestionCount2(
|
|
57328
|
-
return MC_COSTS.paa_base + Math.max(1,
|
|
58540
|
+
function paaCostForQuestionCount2(questionCount2) {
|
|
58541
|
+
return MC_COSTS.paa_base + Math.max(1, questionCount2) * MC_COSTS.paa;
|
|
57329
58542
|
}
|
|
57330
58543
|
async function checkHarvestLimits(user, reuseLockId) {
|
|
57331
58544
|
if (reuseLockId && await reuseExistingConcurrencyGate(user.id, reuseLockId, 30 * 60)) return null;
|
|
@@ -57359,7 +58572,7 @@ async function chargeTierChangeNow(stripeClient, subscriptionId, customerId) {
|
|
|
57359
58572
|
return { ok: false, amountDue: 0, error: err instanceof Error ? err.message : "Unable to charge the plan change immediately." };
|
|
57360
58573
|
}
|
|
57361
58574
|
}
|
|
57362
|
-
var import_resend3, import_hono30, import_hono31, import_factory7, import_cookie2, import_stripe2, secureCookies2, isProduction2, sessionCookieOptions2, requireAllowedOrigin, auth2, sessionAuth, requireIntegrationsTier, requirePaidSchedulingTier, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
58575
|
+
var import_resend3, import_node_crypto38, import_hono30, import_hono31, import_factory7, import_cookie2, import_stripe2, secureCookies2, isProduction2, sessionCookieOptions2, requireAllowedOrigin, auth2, sessionAuth, requireIntegrationsTier, requirePaidSchedulingTier, app, STRIPE_API_VERSION, SYNC_HARVEST_TIMEOUT_OVERRIDE_MS;
|
|
57363
58576
|
var init_server = __esm({
|
|
57364
58577
|
"src/api/server.ts"() {
|
|
57365
58578
|
"use strict";
|
|
@@ -57372,6 +58585,7 @@ var init_server = __esm({
|
|
|
57372
58585
|
init_og();
|
|
57373
58586
|
import_resend3 = require("resend");
|
|
57374
58587
|
init_url_utils();
|
|
58588
|
+
import_node_crypto38 = require("crypto");
|
|
57375
58589
|
init_kpo_extractor();
|
|
57376
58590
|
init_screenshot();
|
|
57377
58591
|
init_media_extractor();
|
|
@@ -57383,6 +58597,7 @@ var init_server = __esm({
|
|
|
57383
58597
|
init_site_audit();
|
|
57384
58598
|
init_site_extract();
|
|
57385
58599
|
init_directory_workflow2();
|
|
58600
|
+
init_paa_harvest();
|
|
57386
58601
|
init_site_extract_repository();
|
|
57387
58602
|
init_site_extract_start();
|
|
57388
58603
|
init_catalog_seed();
|
|
@@ -57407,6 +58622,8 @@ var init_server = __esm({
|
|
|
57407
58622
|
init_directory_routes();
|
|
57408
58623
|
init_location_data_routes();
|
|
57409
58624
|
init_directory_workflow_dispatch();
|
|
58625
|
+
init_paa_harvest_dispatch();
|
|
58626
|
+
init_paa_harvest_reconciliation();
|
|
57410
58627
|
init_directory_workflow_reconciliation();
|
|
57411
58628
|
init_directory_artifacts();
|
|
57412
58629
|
init_workflow_routes();
|
|
@@ -57516,6 +58733,18 @@ var init_server = __esm({
|
|
|
57516
58733
|
STRIPE_API_VERSION = "2026-02-25.clover";
|
|
57517
58734
|
app.use("*", async (c, next) => {
|
|
57518
58735
|
await next();
|
|
58736
|
+
if (c.res.status >= 500 && c.res.headers.get("content-type")?.includes("application/json")) {
|
|
58737
|
+
try {
|
|
58738
|
+
const payload = await c.res.clone().json();
|
|
58739
|
+
const headers = new Headers(c.res.headers);
|
|
58740
|
+
c.res = new Response(JSON.stringify(sanitizeOutboundDiagnostics(payload)), {
|
|
58741
|
+
status: c.res.status,
|
|
58742
|
+
statusText: c.res.statusText,
|
|
58743
|
+
headers
|
|
58744
|
+
});
|
|
58745
|
+
} catch {
|
|
58746
|
+
}
|
|
58747
|
+
}
|
|
57519
58748
|
c.header("Cache-Control", "no-store");
|
|
57520
58749
|
c.header("X-Content-Type-Options", "nosniff");
|
|
57521
58750
|
c.header("Referrer-Policy", "same-origin");
|
|
@@ -58828,9 +60057,50 @@ var init_server = __esm({
|
|
|
58828
60057
|
pages: Math.min(2, Math.max(1, body.pages ?? 1))
|
|
58829
60058
|
};
|
|
58830
60059
|
const harvestCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount2(options.maxQuestions);
|
|
58831
|
-
|
|
58832
|
-
if (
|
|
58833
|
-
|
|
60060
|
+
let jobId2;
|
|
60061
|
+
if (options.serpOnly) {
|
|
60062
|
+
const { ok: harvestOk, balance_mc: harvestBal } = await debitMc(
|
|
60063
|
+
user.id,
|
|
60064
|
+
harvestCost,
|
|
60065
|
+
LedgerOperation.SERP,
|
|
60066
|
+
options.query
|
|
60067
|
+
);
|
|
60068
|
+
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
60069
|
+
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
60070
|
+
} else {
|
|
60071
|
+
jobId2 = (0, import_node_crypto38.randomUUID)();
|
|
60072
|
+
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
60073
|
+
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
60074
|
+
const hold = await debitMcIdempotent(
|
|
60075
|
+
user.id,
|
|
60076
|
+
harvestCost,
|
|
60077
|
+
LedgerOperation.PAA,
|
|
60078
|
+
description,
|
|
60079
|
+
billingDebitKey
|
|
60080
|
+
);
|
|
60081
|
+
if (!hold.ok) return c.json(insufficientBalanceResponse(hold.balance_mc, harvestCost), 402);
|
|
60082
|
+
try {
|
|
60083
|
+
await createJobWithId(jobId2, user.id, options.query, {
|
|
60084
|
+
...options,
|
|
60085
|
+
executionOwner: "inngest",
|
|
60086
|
+
billingHoldMc: harvestCost,
|
|
60087
|
+
billingDebitKey
|
|
60088
|
+
}, body.callback_url);
|
|
60089
|
+
} catch (error) {
|
|
60090
|
+
await settleDebitMcIdempotent(
|
|
60091
|
+
user.id,
|
|
60092
|
+
billingDebitKey,
|
|
60093
|
+
0,
|
|
60094
|
+
LedgerOperation.PAA_REFUND,
|
|
60095
|
+
"PAA job creation failed",
|
|
60096
|
+
"paa_harvest"
|
|
60097
|
+
).catch(() => void 0);
|
|
60098
|
+
throw error;
|
|
60099
|
+
}
|
|
60100
|
+
await dispatchPaaHarvest(jobId2).catch((error) => {
|
|
60101
|
+
console.error("[harvest] durable PAA dispatch pending reconciliation:", error instanceof Error ? error.message : String(error));
|
|
60102
|
+
});
|
|
60103
|
+
}
|
|
58834
60104
|
if (process.env.CRON_SECRET) {
|
|
58835
60105
|
const url = new URL(c.req.url);
|
|
58836
60106
|
void fetch(`${url.origin}/cron/tick`, {
|
|
@@ -58905,6 +60175,10 @@ var init_server = __esm({
|
|
|
58905
60175
|
}
|
|
58906
60176
|
return c.json({ job_id: jobId2, status: "done", result: sanitizeHarvestResult(result), attempts: sanitizeAttempts(attempts) });
|
|
58907
60177
|
} catch (err) {
|
|
60178
|
+
console.error("[harvest/sync] failed", {
|
|
60179
|
+
jobId: jobId2,
|
|
60180
|
+
error: err instanceof Error ? err.message : String(err)
|
|
60181
|
+
});
|
|
58908
60182
|
const problem = classifyHarvestProblem(err);
|
|
58909
60183
|
const response = harvestProblemResponse(problem);
|
|
58910
60184
|
const attempts = await listHarvestAttempts(jobId2, user.id);
|
|
@@ -59935,7 +61209,7 @@ var init_server = __esm({
|
|
|
59935
61209
|
const { drainQueue: drainQueue2 } = await Promise.resolve().then(() => (init_worker(), worker_exports));
|
|
59936
61210
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
59937
61211
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
59938
|
-
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation] = await Promise.all([
|
|
61212
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation] = await Promise.all([
|
|
59939
61213
|
drainQueue2(budget),
|
|
59940
61214
|
runMonthlyRefreshSweep(),
|
|
59941
61215
|
reapIdleBrowserSessions(120).catch(() => ({ reaped: 0 })),
|
|
@@ -59948,7 +61222,9 @@ var init_server = __esm({
|
|
|
59948
61222
|
cleanupExpiredDirectoryArtifacts().catch(() => ({ deleted: 0, store: "none" })),
|
|
59949
61223
|
cleanupExpiredSiteExtractArtifacts().catch(() => ({ deleted: 0, store: "none" })),
|
|
59950
61224
|
redispatchFundedSiteExtractJobs(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
|
|
59951
|
-
reconcileSiteExtractSettlements(25).catch(() => ({ checked: 0, settled: 0, failed: 0 }))
|
|
61225
|
+
reconcileSiteExtractSettlements(25).catch(() => ({ checked: 0, settled: 0, failed: 0 })),
|
|
61226
|
+
redispatchPendingPaaHarvests(25).catch(() => ({ checked: 0, dispatched: 0, failed: 0 })),
|
|
61227
|
+
reconcilePaaHarvestSettlements(25).catch(() => ({ checked: 0, staleFailed: 0, settled: 0, failed: 0 }))
|
|
59952
61228
|
]);
|
|
59953
61229
|
return c.json({
|
|
59954
61230
|
drained: results.length,
|
|
@@ -59965,6 +61241,8 @@ var init_server = __esm({
|
|
|
59965
61241
|
siteExtractArtifactCleanup,
|
|
59966
61242
|
siteExtractRedispatch,
|
|
59967
61243
|
siteExtractReconciliation,
|
|
61244
|
+
paaHarvestRedispatch,
|
|
61245
|
+
paaHarvestReconciliation,
|
|
59968
61246
|
workflowDispatch: workflowDispatchResult
|
|
59969
61247
|
});
|
|
59970
61248
|
});
|
|
@@ -60024,7 +61302,10 @@ var init_server = __esm({
|
|
|
60024
61302
|
const bundle = stored.find((a) => a.contentType === "application/zip" || a.filename?.endsWith(".zip"));
|
|
60025
61303
|
return c.json({ ok: true, jobId: jobId2, artifacts: stored.length, settlement, bundleUrl: bundle?.url ?? null, bundleBytes: bundle?.bytes ?? null });
|
|
60026
61304
|
});
|
|
60027
|
-
app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono31.serve)({
|
|
61305
|
+
app.on(["GET", "POST", "PUT"], "/api/inngest", (0, import_hono31.serve)({
|
|
61306
|
+
client: inngest,
|
|
61307
|
+
functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, paaHarvestFn]
|
|
61308
|
+
}));
|
|
60028
61309
|
app.route("/admin/credits", adminCreditsApp);
|
|
60029
61310
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
60030
61311
|
app.route("/api/internal/memory", internalMemoryApp);
|