mcp-scraper 0.66.0 → 0.66.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -1
- package/README.md +2 -2
- package/dist/{analytics-repository-IKE53S3E.js → analytics-repository-RUXBFSFX.js} +1 -2
- package/dist/bin/api-server.cjs +2898 -2075
- package/dist/bin/api-server.js +4 -5
- package/dist/bin/mcp-scraper-cli.cjs +1 -2
- package/dist/bin/mcp-scraper-cli.js +6 -7
- package/dist/bin/mcp-scraper-install.cjs +7 -3
- package/dist/bin/mcp-scraper-install.js +2 -3
- package/dist/bin/mcp-stdio-server.cjs +1669 -97
- package/dist/bin/mcp-stdio-server.js +11 -12
- package/dist/bin/paa-harvest.cjs +0 -1
- package/dist/bin/paa-harvest.js +4 -5
- package/dist/{chunk-VIWT35S6.js → chunk-3EYALCYK.js} +1 -2
- package/dist/{chunk-4LINSJNI.js → chunk-3TKKJQB4.js} +1 -2
- package/dist/{chunk-4VG3NUUY.js → chunk-55B3T5L2.js} +23 -28
- package/dist/{chunk-FG6LJGJ7.js → chunk-77LQV73B.js} +0 -1
- package/dist/{chunk-N54JS5WP.js → chunk-ABYBPODM.js} +1 -2
- package/dist/chunk-ALEBMPOW.js +6 -0
- package/dist/{chunk-2XUXVKT4.js → chunk-AYJHI2FK.js} +0 -1
- package/dist/{chunk-AAGVOPP5.js → chunk-CETDPNMZ.js} +24 -25
- package/dist/{chunk-5BEPOFKG.js → chunk-CNLLTD5B.js} +0 -1
- package/dist/{chunk-P7YFUNKR.js → chunk-DBWWIK7O.js} +3 -4
- package/dist/{chunk-EKJ65E3Z.js → chunk-DNM65UCK.js} +0 -1
- package/dist/{chunk-3YSEJTMC.js → chunk-GGZEC22A.js} +0 -1
- package/dist/{chunk-AJ4YGHG2.js → chunk-K7Y27ND3.js} +3 -14
- package/dist/{chunk-XPFRP5QT.js → chunk-KBTUPRTF.js} +3 -4
- package/dist/{chunk-5PIO7QBG.js → chunk-KXRWY7DZ.js} +3 -4
- package/dist/{chunk-DCWXVAQT.js → chunk-OZDCTNUV.js} +6 -2
- package/dist/{chunk-BIUQ6O2Y.js → chunk-OZJMVCDK.js} +0 -1
- package/dist/{chunk-X7UG2JCF.js → chunk-PGJQDMC2.js} +1 -2
- package/dist/{chunk-DKIG2TEE.js → chunk-PMCT2YQM.js} +2 -3
- package/dist/{chunk-G3P3ZDB4.js → chunk-RJ7JVYKU.js} +0 -1
- package/dist/{chunk-WD73OFL3.js → chunk-VXLU74YZ.js} +1 -2
- package/dist/{chunk-SRRI2HJL.js → chunk-YKL3O7IU.js} +1742 -104
- package/dist/{db-BRI465XQ.js → db-4RVZ3NYZ.js} +9 -10
- package/dist/{extract-bundle-QAI7JYV2.js → extract-bundle-EA5URLXN.js} +8 -9
- package/dist/index.cjs +0 -1
- package/dist/index.js +4 -5
- package/dist/{lead-list-enrichment-repository-ODWGRVCQ.js → lead-list-enrichment-repository-IBXNWM6H.js} +2 -3
- package/dist/{location-data-repository-67MMEVZQ.js → location-data-repository-WHHQUCUI.js} +3 -4
- package/dist/{server-MTKWBSAR.js → server-Q7VYGSGQ.js} +450 -1264
- package/dist/{site-extract-repository-J3RH3MOS.js → site-extract-repository-NTM4MDQN.js} +4 -5
- package/dist/{worker-K7MPPQ7I.js → worker-UTBUWEXQ.js} +8 -9
- package/package.json +4 -1
- package/dist/analytics-repository-IKE53S3E.js.map +0 -1
- package/dist/bin/api-server.cjs.map +0 -1
- package/dist/bin/api-server.js.map +0 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +0 -1
- package/dist/bin/mcp-scraper-cli.js.map +0 -1
- package/dist/bin/mcp-scraper-install.cjs.map +0 -1
- package/dist/bin/mcp-scraper-install.js.map +0 -1
- package/dist/bin/mcp-stdio-server.cjs.map +0 -1
- package/dist/bin/mcp-stdio-server.js.map +0 -1
- package/dist/bin/paa-harvest.cjs.map +0 -1
- package/dist/bin/paa-harvest.js.map +0 -1
- package/dist/chunk-2XUXVKT4.js.map +0 -1
- package/dist/chunk-3YSEJTMC.js.map +0 -1
- package/dist/chunk-4LINSJNI.js.map +0 -1
- package/dist/chunk-4VG3NUUY.js.map +0 -1
- package/dist/chunk-5BEPOFKG.js.map +0 -1
- package/dist/chunk-5PIO7QBG.js.map +0 -1
- package/dist/chunk-AAGVOPP5.js.map +0 -1
- package/dist/chunk-AJ4YGHG2.js.map +0 -1
- package/dist/chunk-BIUQ6O2Y.js.map +0 -1
- package/dist/chunk-DCWXVAQT.js.map +0 -1
- package/dist/chunk-DKIG2TEE.js.map +0 -1
- package/dist/chunk-EKJ65E3Z.js.map +0 -1
- package/dist/chunk-FG6LJGJ7.js.map +0 -1
- package/dist/chunk-G3P3ZDB4.js.map +0 -1
- package/dist/chunk-HDYOVGC6.js +0 -7
- package/dist/chunk-HDYOVGC6.js.map +0 -1
- package/dist/chunk-N54JS5WP.js.map +0 -1
- package/dist/chunk-P7YFUNKR.js.map +0 -1
- package/dist/chunk-SRRI2HJL.js.map +0 -1
- package/dist/chunk-VIWT35S6.js.map +0 -1
- package/dist/chunk-WD73OFL3.js.map +0 -1
- package/dist/chunk-X7UG2JCF.js.map +0 -1
- package/dist/chunk-XPFRP5QT.js.map +0 -1
- package/dist/db-BRI465XQ.js.map +0 -1
- package/dist/extract-bundle-QAI7JYV2.js.map +0 -1
- package/dist/index.cjs.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lead-list-enrichment-repository-ODWGRVCQ.js.map +0 -1
- package/dist/location-data-repository-67MMEVZQ.js.map +0 -1
- package/dist/server-MTKWBSAR.js.map +0 -1
- package/dist/site-extract-repository-J3RH3MOS.js.map +0 -1
- package/dist/worker-K7MPPQ7I.js.map +0 -1
|
@@ -10,13 +10,13 @@ import {
|
|
|
10
10
|
queuedCommonsDocumentCount,
|
|
11
11
|
semanticCommonsEntityScores,
|
|
12
12
|
typeFromMime
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-DBWWIK7O.js";
|
|
14
14
|
import {
|
|
15
15
|
REQUIRED_LOCATION_STATE_CODES,
|
|
16
16
|
importHostedCensusPlacesCsv,
|
|
17
17
|
importHostedZipGroupsCsv,
|
|
18
18
|
listActiveHostedLocationDatasets
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-PMCT2YQM.js";
|
|
20
20
|
import {
|
|
21
21
|
cancelPendingLeadListEnrichmentRows,
|
|
22
22
|
claimLeadListEnrichmentOutbox,
|
|
@@ -43,7 +43,7 @@ import {
|
|
|
43
43
|
saveLeadListEnrichmentRowResult,
|
|
44
44
|
setInitializingLeadListEnrichmentConcurrencyLock,
|
|
45
45
|
stableCanonicalJson
|
|
46
|
-
} from "./chunk-
|
|
46
|
+
} from "./chunk-ABYBPODM.js";
|
|
47
47
|
import {
|
|
48
48
|
ANALYTICS_CONTENT_SORTS,
|
|
49
49
|
AnalyticsRepositoryError,
|
|
@@ -99,7 +99,7 @@ import {
|
|
|
99
99
|
updateAnalyticsPixel,
|
|
100
100
|
upsertAnalyticsAdSpend,
|
|
101
101
|
verifySession
|
|
102
|
-
} from "./chunk-
|
|
102
|
+
} from "./chunk-CNLLTD5B.js";
|
|
103
103
|
import {
|
|
104
104
|
classifyHarvestProblem,
|
|
105
105
|
createHarvestAttemptRecorder,
|
|
@@ -108,20 +108,12 @@ import {
|
|
|
108
108
|
harvestProblemResponse,
|
|
109
109
|
publicErrorBoundary,
|
|
110
110
|
serializeHarvestProblem
|
|
111
|
-
} from "./chunk-
|
|
112
|
-
import "./chunk-
|
|
111
|
+
} from "./chunk-CETDPNMZ.js";
|
|
112
|
+
import "./chunk-OZJMVCDK.js";
|
|
113
113
|
import {
|
|
114
|
-
AIModeSelectors,
|
|
115
|
-
AIOverviewSelectors,
|
|
116
114
|
BACKUP_LAST_RESORT_ATTEMPTS,
|
|
117
115
|
BrowserDriver,
|
|
118
|
-
ForumSelectors,
|
|
119
|
-
LocalPackSelectors,
|
|
120
116
|
MapsSelectors,
|
|
121
|
-
OrganicSelectors,
|
|
122
|
-
PAASelectors,
|
|
123
|
-
VideoSelectors,
|
|
124
|
-
WhatPeopleSayingSelectors,
|
|
125
117
|
backupProxyAvailable,
|
|
126
118
|
brightDataSerpEnabled,
|
|
127
119
|
buildYouTubeChannelVideosUrl,
|
|
@@ -130,14 +122,12 @@ import {
|
|
|
130
122
|
deleteKernelProxyId,
|
|
131
123
|
detectPageUserAgent,
|
|
132
124
|
detectRuntimeUserAgent,
|
|
133
|
-
encodeUule,
|
|
134
125
|
harvest,
|
|
135
|
-
normalizeLocation,
|
|
136
126
|
resolveBackupKernelProxyId,
|
|
137
127
|
resolveKernelProxyId,
|
|
138
128
|
serpArrivalUrl,
|
|
139
129
|
withRuntimeUserAgent
|
|
140
|
-
} from "./chunk-
|
|
130
|
+
} from "./chunk-K7Y27ND3.js";
|
|
141
131
|
import {
|
|
142
132
|
ApiErrorResponseSchema,
|
|
143
133
|
ArtifactTemplatePresetKeySchema,
|
|
@@ -209,18 +199,18 @@ import {
|
|
|
209
199
|
resolveDeploymentProfile,
|
|
210
200
|
resolveLocalSourcebookSchemaType,
|
|
211
201
|
transcribeMediaUrl
|
|
212
|
-
} from "./chunk-
|
|
202
|
+
} from "./chunk-YKL3O7IU.js";
|
|
213
203
|
import {
|
|
214
204
|
auditImageUrls,
|
|
215
205
|
auditImages,
|
|
216
206
|
buildLinkReport,
|
|
217
207
|
computeIssues
|
|
218
|
-
} from "./chunk-
|
|
208
|
+
} from "./chunk-PGJQDMC2.js";
|
|
219
209
|
import {
|
|
220
210
|
sanitizeAttempts,
|
|
221
211
|
sanitizeHarvestResult,
|
|
222
212
|
sanitizeOutboundDiagnostics
|
|
223
|
-
} from "./chunk-
|
|
213
|
+
} from "./chunk-3EYALCYK.js";
|
|
224
214
|
import {
|
|
225
215
|
browserServiceApiKey,
|
|
226
216
|
browserServiceProfileName,
|
|
@@ -230,7 +220,7 @@ import {
|
|
|
230
220
|
recordKernelSession,
|
|
231
221
|
recordVendorUsage,
|
|
232
222
|
runWithCostContext
|
|
233
|
-
} from "./chunk-
|
|
223
|
+
} from "./chunk-3TKKJQB4.js";
|
|
234
224
|
import {
|
|
235
225
|
listWorkflowDefinitions,
|
|
236
226
|
runWorkflow,
|
|
@@ -240,12 +230,12 @@ import {
|
|
|
240
230
|
workflowOutputBaseDir,
|
|
241
231
|
workflowStepCount,
|
|
242
232
|
workflowSupportsSteps
|
|
243
|
-
} from "./chunk-
|
|
233
|
+
} from "./chunk-KBTUPRTF.js";
|
|
244
234
|
import {
|
|
245
235
|
csvRecords,
|
|
246
236
|
parseCsv,
|
|
247
237
|
rowsToCsv
|
|
248
|
-
} from "./chunk-
|
|
238
|
+
} from "./chunk-RJ7JVYKU.js";
|
|
249
239
|
import {
|
|
250
240
|
DEFAULT_MAPS_PROXY_MODE,
|
|
251
241
|
DEFAULT_PROXY_MODE,
|
|
@@ -255,18 +245,18 @@ import {
|
|
|
255
245
|
RawMapsHoursRowSchema,
|
|
256
246
|
RawMapsOverviewSchema,
|
|
257
247
|
RawMapsReviewStatsSchema
|
|
258
|
-
} from "./chunk-
|
|
248
|
+
} from "./chunk-77LQV73B.js";
|
|
259
249
|
import {
|
|
260
250
|
extractContactEvidence
|
|
261
|
-
} from "./chunk-
|
|
251
|
+
} from "./chunk-VXLU74YZ.js";
|
|
262
252
|
import {
|
|
263
253
|
isPrivateIpAddress,
|
|
264
254
|
loadHtmlDocument,
|
|
265
255
|
validatePublicHttpUrl
|
|
266
|
-
} from "./chunk-
|
|
256
|
+
} from "./chunk-GGZEC22A.js";
|
|
267
257
|
import {
|
|
268
258
|
PACKAGE_VERSION
|
|
269
|
-
} from "./chunk-
|
|
259
|
+
} from "./chunk-ALEBMPOW.js";
|
|
270
260
|
import {
|
|
271
261
|
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
272
262
|
abandonExtractSettlement,
|
|
@@ -297,14 +287,14 @@ import {
|
|
|
297
287
|
setExtractJobTotal,
|
|
298
288
|
settleExtractJob,
|
|
299
289
|
terminalExtractJobStatus
|
|
300
|
-
} from "./chunk-
|
|
290
|
+
} from "./chunk-KXRWY7DZ.js";
|
|
301
291
|
import {
|
|
302
292
|
createPrivateArtifact,
|
|
303
293
|
privateArtifactOwnerId,
|
|
304
294
|
readPrivateArtifactBuffer,
|
|
305
295
|
readPrivateArtifactWindow,
|
|
306
296
|
renewPrivateArtifactDownload
|
|
307
|
-
} from "./chunk-
|
|
297
|
+
} from "./chunk-DNM65UCK.js";
|
|
308
298
|
import {
|
|
309
299
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
310
300
|
CONCURRENCY_PACK_INVOICE_DESCRIPTION,
|
|
@@ -357,7 +347,7 @@ import {
|
|
|
357
347
|
tierChangeLeavesPrivateOffer,
|
|
358
348
|
tierCreditsPerInterval,
|
|
359
349
|
vendorCostUsdToMc
|
|
360
|
-
} from "./chunk-
|
|
350
|
+
} from "./chunk-AYJHI2FK.js";
|
|
361
351
|
import {
|
|
362
352
|
CaptchaError,
|
|
363
353
|
MapsSearchIncompletePageError,
|
|
@@ -369,7 +359,7 @@ import {
|
|
|
369
359
|
cancelJob,
|
|
370
360
|
checkRateLimit,
|
|
371
361
|
checkpointJob,
|
|
372
|
-
|
|
362
|
+
claimDurablePaaJob,
|
|
373
363
|
claimStripeEvent,
|
|
374
364
|
completeJob,
|
|
375
365
|
completeStripeEvent,
|
|
@@ -392,9 +382,9 @@ import {
|
|
|
392
382
|
deleteWorkflowSchedule,
|
|
393
383
|
expireOldLots,
|
|
394
384
|
failJob,
|
|
385
|
+
failRunningJob,
|
|
395
386
|
failStripeEvent,
|
|
396
387
|
failWorkflowRunRecord,
|
|
397
|
-
finishHarvestAttempt,
|
|
398
388
|
getBillingLedgerSummary,
|
|
399
389
|
getClient,
|
|
400
390
|
getDb,
|
|
@@ -424,10 +414,9 @@ import {
|
|
|
424
414
|
listHarvestAttempts,
|
|
425
415
|
listInboxMessages,
|
|
426
416
|
listJobs,
|
|
427
|
-
|
|
428
|
-
listRecentTerminalInngestPaaJobs,
|
|
417
|
+
listRecentTerminalDurablePaaJobs,
|
|
429
418
|
listRequestEvents,
|
|
430
|
-
|
|
419
|
+
listStaleRunningDurablePaaJobs,
|
|
431
420
|
listUsers,
|
|
432
421
|
listUsersDueForConnectedAccountBillingReconciliation,
|
|
433
422
|
listWorkflowArtifacts,
|
|
@@ -460,10 +449,9 @@ import {
|
|
|
460
449
|
setUserMemoryCreds,
|
|
461
450
|
settleDebitMcIdempotent,
|
|
462
451
|
softDeleteInboxMessage,
|
|
463
|
-
startHarvestAttempt,
|
|
464
452
|
upsertPageSnapshot,
|
|
465
453
|
verifyPassword
|
|
466
|
-
} from "./chunk-
|
|
454
|
+
} from "./chunk-55B3T5L2.js";
|
|
467
455
|
|
|
468
456
|
// src/blog/registry.ts
|
|
469
457
|
var posts = [
|
|
@@ -7795,12 +7783,12 @@ function buildIngestValidatePrompt(input) {
|
|
|
7795
7783
|
`{"error_severity":"blocking","error_code":"dep-001","message":"Required Python package missing: {package}. Install with: pip install networkx pandas lxml","file":null,"remediation":"Run: pip install networkx pandas lxml"}`,
|
|
7796
7784
|
`\`\`\``,
|
|
7797
7785
|
``,
|
|
7798
|
-
`Also
|
|
7786
|
+
`Also check whether the optional uszips.csv classifier dataset is accessible. When unavailable, Phase 3 remains functional and falls back to LLM classification:`,
|
|
7799
7787
|
``,
|
|
7800
7788
|
`\`\`\`bash`,
|
|
7801
7789
|
`python3 -c "`,
|
|
7802
7790
|
`import csv, os`,
|
|
7803
|
-
`path = '/
|
|
7791
|
+
`path = os.environ.get('US_ZIPS_CSV_PATH') or os.environ.get('MCP_SCRAPER_USZIPS_CSV_PATH') or '/tmp/uszips.csv'`,
|
|
7804
7792
|
`if not os.path.exists(path):`,
|
|
7805
7793
|
` print('uszips:missing')`,
|
|
7806
7794
|
`else:`,
|
|
@@ -7814,12 +7802,12 @@ function buildIngestValidatePrompt(input) {
|
|
|
7814
7802
|
`If \`uszips:missing\`, write a **warning** (non-blocking) to \`ingestion_error_log.jsonl\`:`,
|
|
7815
7803
|
``,
|
|
7816
7804
|
`\`\`\`json`,
|
|
7817
|
-
`{"error_severity":"warning","error_code":"dep-002","message":"uszips.csv not found at
|
|
7805
|
+
`{"error_severity":"warning","error_code":"dep-002","message":"uszips.csv was not found at the resolved path. The location classifier is disabled and Phase 3 will use LLM classification, which may increase cost.","file":"{resolved_uszips_path}","remediation":"Optionally set US_ZIPS_CSV_PATH or MCP_SCRAPER_USZIPS_CSV_PATH to a readable uszips.csv file with the expected schema."}`,
|
|
7818
7806
|
`\`\`\``,
|
|
7819
7807
|
``,
|
|
7820
7808
|
`If \`uszips:schema-mismatch\`, write the same warning with the mismatch detail in the message. The audit continues without the location classifier \u2014 all URLs fall through to Claude classification.`,
|
|
7821
7809
|
``,
|
|
7822
|
-
`
|
|
7810
|
+
`Continue after recording either warning; missing or mismatched uszips.csv data is not a blocking gate.`,
|
|
7823
7811
|
``,
|
|
7824
7812
|
`---`,
|
|
7825
7813
|
``,
|
|
@@ -8733,6 +8721,9 @@ var CompareRecommendOutputSchema = z9.object({
|
|
|
8733
8721
|
).optional(),
|
|
8734
8722
|
newPageCreationList: z9.array(z9.unknown()).optional()
|
|
8735
8723
|
});
|
|
8724
|
+
function resolveSiteAuditUsZipsCsvPath(env = process.env) {
|
|
8725
|
+
return env["US_ZIPS_CSV_PATH"]?.trim() || env["MCP_SCRAPER_USZIPS_CSV_PATH"]?.trim() || "/tmp/uszips.csv";
|
|
8726
|
+
}
|
|
8736
8727
|
var SiteAuditService = class {
|
|
8737
8728
|
constructor(deps) {
|
|
8738
8729
|
this.deps = deps;
|
|
@@ -8900,7 +8891,7 @@ var SiteAuditService = class {
|
|
|
8900
8891
|
currentPayloadChars += itemChars;
|
|
8901
8892
|
}
|
|
8902
8893
|
if (currentBatch.length > 0) batches.push(currentBatch);
|
|
8903
|
-
const usZipsCsvPath =
|
|
8894
|
+
const usZipsCsvPath = resolveSiteAuditUsZipsCsvPath();
|
|
8904
8895
|
await this.runLlmBilled(job.user_id, `${jobId2}:phase3-classify`, async () => {
|
|
8905
8896
|
for (const batch of batches) {
|
|
8906
8897
|
let classified = [];
|
|
@@ -9554,7 +9545,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9554
9545
|
});
|
|
9555
9546
|
}) : null;
|
|
9556
9547
|
const finalized = await step.run("finalize", async () => {
|
|
9557
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9548
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-EA5URLXN.js");
|
|
9558
9549
|
const current = await getExtractJob(jobId2);
|
|
9559
9550
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
9560
9551
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -10514,7 +10505,7 @@ async function resolveUsZipsPath(requestedPath) {
|
|
|
10514
10505
|
async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
|
|
10515
10506
|
const mode = resolvedLocationDataMode({ zipSource: sourceMode });
|
|
10516
10507
|
if (mode === "hosted") {
|
|
10517
|
-
const { getHostedZipGroups } = await import("./location-data-repository-
|
|
10508
|
+
const { getHostedZipGroups } = await import("./location-data-repository-WHHQUCUI.js");
|
|
10518
10509
|
const hosted = await getHostedZipGroups(stateAbbr);
|
|
10519
10510
|
if (!hosted.dataset) {
|
|
10520
10511
|
throw new DirectoryLocationError(
|
|
@@ -10577,7 +10568,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
|
|
|
10577
10568
|
const sourceUrl = censusStateUrl(state.fips);
|
|
10578
10569
|
const warnings = [];
|
|
10579
10570
|
if (resolvedLocationDataMode(config) === "hosted") {
|
|
10580
|
-
const { queryHostedLocationMarkets } = await import("./location-data-repository-
|
|
10571
|
+
const { queryHostedLocationMarkets } = await import("./location-data-repository-WHHQUCUI.js");
|
|
10581
10572
|
const hosted = await queryHostedLocationMarkets({
|
|
10582
10573
|
state: state.abbr,
|
|
10583
10574
|
minPopulation: options.minPopulation,
|
|
@@ -12064,7 +12055,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
|
|
|
12064
12055
|
async function settlePersistedLeadListEnrichment(jobId2) {
|
|
12065
12056
|
const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
|
|
12066
12057
|
const job = jobs.find((candidate) => candidate.id === jobId2);
|
|
12067
|
-
const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-
|
|
12058
|
+
const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-IBXNWM6H.js");
|
|
12068
12059
|
const current = job ?? await getLeadListEnrichmentJob2(jobId2);
|
|
12069
12060
|
if (!current) return null;
|
|
12070
12061
|
const rows = await listLeadListEnrichmentRows(jobId2);
|
|
@@ -12120,7 +12111,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
|
|
|
12120
12111
|
const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
|
|
12121
12112
|
const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
|
|
12122
12113
|
if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
|
|
12123
|
-
const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-
|
|
12114
|
+
const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-IBXNWM6H.js");
|
|
12124
12115
|
await failLeadListEnrichmentJob2({
|
|
12125
12116
|
jobId: job.id,
|
|
12126
12117
|
errorCode: "lead_list_all_rows_failed",
|
|
@@ -12421,1067 +12412,8 @@ var leadListEnrichmentFn = inngest.createFunction(
|
|
|
12421
12412
|
}
|
|
12422
12413
|
);
|
|
12423
12414
|
|
|
12424
|
-
// src/api/paa-capture-artifacts.ts
|
|
12425
|
-
var DAY_MS = 24 * 60 * 60 * 1e3;
|
|
12426
|
-
var PAA_CAPTURE_ARTIFACT_PREFIX = "paa-captures";
|
|
12427
|
-
function paaCaptureArtifactPolicy() {
|
|
12428
|
-
return {
|
|
12429
|
-
prefix: PAA_CAPTURE_ARTIFACT_PREFIX,
|
|
12430
|
-
artifactTtlMs: 7 * DAY_MS,
|
|
12431
|
-
downloadTtlMs: 15 * 60 * 1e3,
|
|
12432
|
-
token: process.env.PAA_CAPTURE_BLOB_READ_WRITE_TOKEN ?? process.env.BLOB_READ_WRITE_TOKEN ?? null
|
|
12433
|
-
};
|
|
12434
|
-
}
|
|
12435
|
-
async function createPaaRawDomArtifact(input) {
|
|
12436
|
-
return createPrivateArtifact({
|
|
12437
|
-
policy: paaCaptureArtifactPolicy(),
|
|
12438
|
-
ownerId: String(input.userId),
|
|
12439
|
-
artifactKey: `${input.jobId}-attempt-${input.attempt}.html.gz`,
|
|
12440
|
-
createdAt: input.createdAt ?? /* @__PURE__ */ new Date(),
|
|
12441
|
-
filename: `${input.jobId}-serp.html.gz`,
|
|
12442
|
-
contentType: "application/gzip",
|
|
12443
|
-
content: input.gzip
|
|
12444
|
-
});
|
|
12445
|
-
}
|
|
12446
|
-
async function tryCreatePaaRawDomArtifact(input) {
|
|
12447
|
-
try {
|
|
12448
|
-
return {
|
|
12449
|
-
artifact: await createPaaRawDomArtifact(input),
|
|
12450
|
-
status: "stored"
|
|
12451
|
-
};
|
|
12452
|
-
} catch (error) {
|
|
12453
|
-
console.error("[paa-capture-artifact] private DOM preservation unavailable:", error instanceof Error ? error.message : String(error));
|
|
12454
|
-
return { artifact: null, status: "unavailable" };
|
|
12455
|
-
}
|
|
12456
|
-
}
|
|
12457
|
-
|
|
12458
|
-
// src/api/paa-harvest-settlement.ts
|
|
12459
|
-
function capturedQuestionCount(result) {
|
|
12460
|
-
if (!result || typeof result !== "object") return 0;
|
|
12461
|
-
const value = result;
|
|
12462
|
-
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
12463
|
-
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
12464
|
-
}
|
|
12465
|
-
function finalCost(result, heldMc) {
|
|
12466
|
-
const questions = capturedQuestionCount(result);
|
|
12467
|
-
if (questions <= 0) return 0;
|
|
12468
|
-
return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
|
|
12469
|
-
}
|
|
12470
|
-
async function settlePaaHarvestJob(jobOrId) {
|
|
12471
|
-
const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
|
|
12472
|
-
if (!job) return false;
|
|
12473
|
-
const options = job.options;
|
|
12474
|
-
const debitKey2 = options.billingDebitKey;
|
|
12475
|
-
const heldMc = Number(options.billingHoldMc ?? 0);
|
|
12476
|
-
if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
|
|
12477
|
-
const settlement = await settleDebitMcIdempotent(
|
|
12478
|
-
job.user_id,
|
|
12479
|
-
debitKey2,
|
|
12480
|
-
finalCost(job.result, heldMc),
|
|
12481
|
-
LedgerOperation.PAA_REFUND,
|
|
12482
|
-
"durable PAA harvest settlement",
|
|
12483
|
-
"paa_harvest"
|
|
12484
|
-
);
|
|
12485
|
-
await markJobBillingSettled(job.id, settlement.final_amount_mc);
|
|
12486
|
-
return true;
|
|
12487
|
-
}
|
|
12488
|
-
|
|
12489
|
-
// src/api/serp-identity-db.ts
|
|
12490
|
-
import { randomUUID as randomUUID3 } from "crypto";
|
|
12491
|
-
async function createSerpIdentityRow(input) {
|
|
12492
|
-
const id = `serpi_${randomUUID3().replace(/-/g, "").slice(0, 20)}`;
|
|
12493
|
-
await getDb().execute({
|
|
12494
|
-
sql: `INSERT INTO serp_identities
|
|
12495
|
-
(id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
|
|
12496
|
-
VALUES (?, ?, ?, ?, ?, 'isp', ?, 'ready')`,
|
|
12497
|
-
args: [id, input.userId, input.name, input.kernelProfileName, input.kernelProxyId, input.country]
|
|
12498
|
-
});
|
|
12499
|
-
const row = await getSerpIdentityRow(input.userId, input.name);
|
|
12500
|
-
if (!row) throw new Error("SERP identity insert failed");
|
|
12501
|
-
return row;
|
|
12502
|
-
}
|
|
12503
|
-
async function getSerpIdentityRow(userId, name) {
|
|
12504
|
-
const result = await getDb().execute({
|
|
12505
|
-
sql: `SELECT * FROM serp_identities WHERE user_id = ? AND name = ? LIMIT 1`,
|
|
12506
|
-
args: [userId, name]
|
|
12507
|
-
});
|
|
12508
|
-
return result.rows[0] ?? null;
|
|
12509
|
-
}
|
|
12510
|
-
async function listSerpIdentityRows(userId) {
|
|
12511
|
-
const result = await getDb().execute({
|
|
12512
|
-
sql: `SELECT * FROM serp_identities WHERE user_id = ? ORDER BY created_at DESC`,
|
|
12513
|
-
args: [userId]
|
|
12514
|
-
});
|
|
12515
|
-
return result.rows;
|
|
12516
|
-
}
|
|
12517
|
-
async function touchSerpIdentity(userId, name) {
|
|
12518
|
-
await getDb().execute({
|
|
12519
|
-
sql: `UPDATE serp_identities
|
|
12520
|
-
SET last_used_at = datetime('now'), updated_at = datetime('now')
|
|
12521
|
-
WHERE user_id = ? AND name = ?`,
|
|
12522
|
-
args: [userId, name]
|
|
12523
|
-
});
|
|
12524
|
-
}
|
|
12525
|
-
async function setSerpIdentityStatus(userId, name, status) {
|
|
12526
|
-
await getDb().execute({
|
|
12527
|
-
sql: `UPDATE serp_identities SET status = ?, updated_at = datetime('now') WHERE user_id = ? AND name = ?`,
|
|
12528
|
-
args: [status, userId, name]
|
|
12529
|
-
});
|
|
12530
|
-
}
|
|
12531
|
-
async function deleteSerpIdentityRow(userId, name) {
|
|
12532
|
-
await getDb().execute({
|
|
12533
|
-
sql: `DELETE FROM serp_identities WHERE user_id = ? AND name = ?`,
|
|
12534
|
-
args: [userId, name]
|
|
12535
|
-
});
|
|
12536
|
-
}
|
|
12537
|
-
|
|
12538
|
-
// src/paa/durable-capture.ts
|
|
12539
|
-
import Kernel4 from "@onkernel/sdk";
|
|
12540
|
-
import { createHash as createHash2 } from "crypto";
|
|
12541
|
-
var PAA_INVOCATION_BUDGET_MS = 28e4;
|
|
12542
|
-
var PAA_BROWSER_WORK_BUDGET_MS = 25e4;
|
|
12543
|
-
var PAA_SOLVER_BUDGET_MS = 6e4;
|
|
12544
|
-
function remainingSeconds(deadlineMs, ceiling) {
|
|
12545
|
-
return Math.max(1, Math.min(ceiling, Math.floor((deadlineMs - Date.now()) / 1e3)));
|
|
12546
|
-
}
|
|
12547
|
-
async function execute(kernel, sessionId, code, deadlineMs, ceilingSeconds) {
|
|
12548
|
-
if (Date.now() >= deadlineMs) throw new Error("paa_work_deadline_exhausted");
|
|
12549
|
-
const response = await kernel.browsers.playwright.execute(sessionId, {
|
|
12550
|
-
code,
|
|
12551
|
-
timeout_sec: remainingSeconds(deadlineMs, ceilingSeconds)
|
|
12552
|
-
});
|
|
12553
|
-
if (!response.success || response.result === null || response.result === void 0) {
|
|
12554
|
-
throw new Error(response.error || response.stderr || "co-located browser execution failed");
|
|
12555
|
-
}
|
|
12556
|
-
return response.result;
|
|
12557
|
-
}
|
|
12558
|
-
function navigationCode(searchUrl) {
|
|
12559
|
-
return `
|
|
12560
|
-
const wait = ms => page.waitForTimeout(ms);
|
|
12561
|
-
const block = async () => {
|
|
12562
|
-
if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
|
|
12563
|
-
if (await page.locator(${JSON.stringify(PAASelectors.captchaMarker)}).count().catch(() => 0)) return 'captcha';
|
|
12564
|
-
const title = await page.title().catch(() => '');
|
|
12565
|
-
const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
|
|
12566
|
-
if (/recaptcha|unusual traffic|are you a robot|about this page|detected unusual traffic/i.test(text)) return 'captcha';
|
|
12567
|
-
if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(title + '\\n' + text.slice(0, 3000))) return 'soft_block';
|
|
12568
|
-
return null;
|
|
12569
|
-
};
|
|
12570
|
-
await page.goto(${JSON.stringify(searchUrl)}, { waitUntil: 'domcontentloaded', timeout: 45000 });
|
|
12571
|
-
let captchaObserved = false;
|
|
12572
|
-
let solverCleared = false;
|
|
12573
|
-
let blocked = await block();
|
|
12574
|
-
const solverDeadline = Date.now() + ${PAA_SOLVER_BUDGET_MS};
|
|
12575
|
-
while (blocked === 'captcha' && Date.now() < solverDeadline) {
|
|
12576
|
-
captchaObserved = true;
|
|
12577
|
-
await wait(4000 + Math.floor(Math.random() * 1500));
|
|
12578
|
-
blocked = await block();
|
|
12579
|
-
if (!blocked) solverCleared = true;
|
|
12580
|
-
}
|
|
12581
|
-
let count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
|
|
12582
|
-
for (let scan = 0; !blocked && count === 0 && scan < 7; scan += 1) {
|
|
12583
|
-
await page.mouse.wheel(0, 520 + Math.floor(Math.random() * 420));
|
|
12584
|
-
await wait(350 + Math.floor(Math.random() * 500));
|
|
12585
|
-
count = await page.locator(${JSON.stringify(PAASelectors.item)}).count().catch(() => 0);
|
|
12586
|
-
}
|
|
12587
|
-
await page.evaluate((itemSelector) => {
|
|
12588
|
-
const state = window;
|
|
12589
|
-
if (state.__mcpTrustedListenerInstalled) return;
|
|
12590
|
-
document.addEventListener('click', event => {
|
|
12591
|
-
if (event.target instanceof Element && event.target.closest(itemSelector)) {
|
|
12592
|
-
state.__mcpLastPaaClickTrusted = event.isTrusted;
|
|
12593
|
-
}
|
|
12594
|
-
}, { capture: true });
|
|
12595
|
-
state.__mcpTrustedListenerInstalled = true;
|
|
12596
|
-
}, ${JSON.stringify(PAASelectors.item)}).catch(() => {});
|
|
12597
|
-
return {
|
|
12598
|
-
outcome: blocked || (count > 0 ? 'ready' : 'no_paa'),
|
|
12599
|
-
finalUrl: page.url(),
|
|
12600
|
-
title: await page.title().catch(() => ''),
|
|
12601
|
-
captchaObserved,
|
|
12602
|
-
solverCleared,
|
|
12603
|
-
environment: await page.evaluate(() => ({
|
|
12604
|
-
userAgent: navigator.userAgent,
|
|
12605
|
-
language: navigator.language,
|
|
12606
|
-
platform: navigator.platform,
|
|
12607
|
-
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
12608
|
-
innerWidth: window.innerWidth,
|
|
12609
|
-
innerHeight: window.innerHeight,
|
|
12610
|
-
outerWidth: window.outerWidth,
|
|
12611
|
-
outerHeight: window.outerHeight,
|
|
12612
|
-
screenWidth: window.screen.width,
|
|
12613
|
-
screenHeight: window.screen.height,
|
|
12614
|
-
devicePixelRatio: window.devicePixelRatio,
|
|
12615
|
-
})).catch(() => null),
|
|
12616
|
-
};
|
|
12617
|
-
`;
|
|
12618
|
-
}
|
|
12619
|
-
function recordReaderSource() {
|
|
12620
|
-
return `
|
|
12621
|
-
const clean = value => (value || '').replace(/\\u00a0/g, ' ').replace(/[ \\t]+/g, ' ').replace(/\\n{3,}/g, '\\n\\n').trim();
|
|
12622
|
-
const externalHref = anchor => {
|
|
12623
|
-
const href = anchor && anchor.href || '';
|
|
12624
|
-
if (!/^https?:\\/\\//i.test(href)) return '';
|
|
12625
|
-
try {
|
|
12626
|
-
const url = new URL(href);
|
|
12627
|
-
if (/(^|\\.)google(usercontent)?\\.[a-z.]+$/i.test(url.hostname)) {
|
|
12628
|
-
const redirected = url.searchParams.get('q') || url.searchParams.get('url');
|
|
12629
|
-
return redirected && redirected.startsWith('http') ? redirected : '';
|
|
12630
|
-
}
|
|
12631
|
-
} catch {}
|
|
12632
|
-
return href;
|
|
12633
|
-
};
|
|
12634
|
-
const sourceLabel = value => clean(value)
|
|
12635
|
-
.replace(/\\s*\\(\\+\\d+\\)\\s*-\\s*View related links.*$/i, '')
|
|
12636
|
-
.replace(/\\s*-\\s*Opens in new tab.*$/i, '').trim();
|
|
12637
|
-
const readRecords = () => Array.from(document.querySelectorAll(SEL.item)).map(item => {
|
|
12638
|
-
const question = clean(item.getAttribute(SEL.itemDataQ)
|
|
12639
|
-
|| item.getAttribute(SEL.itemDataInitQ)
|
|
12640
|
-
|| item.querySelector(SEL.itemQuestionEl)?.innerText);
|
|
12641
|
-
const expanded = item.classList.contains(SEL.expandedClass)
|
|
12642
|
-
|| item.querySelector(SEL.clickTarget)?.getAttribute('aria-expanded') === 'true';
|
|
12643
|
-
const container = expanded ? item.querySelector(SEL.answerContainer) : null;
|
|
12644
|
-
const answerElement = container?.querySelector(SEL.aiAnswerBody) || container;
|
|
12645
|
-
const anchors = expanded ? Array.from(answerElement?.querySelectorAll(
|
|
12646
|
-
SEL.aiCitation + ',' + SEL.aiSourceCard + ',a[href]'
|
|
12647
|
-
) || []) : [];
|
|
12648
|
-
const byUrl = new Map();
|
|
12649
|
-
for (const anchor of anchors) {
|
|
12650
|
-
const url = externalHref(anchor);
|
|
12651
|
-
if (!url || byUrl.has(url)) continue;
|
|
12652
|
-
let site = '';
|
|
12653
|
-
try { site = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
|
|
12654
|
-
byUrl.set(url, { title: sourceLabel(anchor.getAttribute('aria-label')) || clean(anchor.textContent), site, url });
|
|
12655
|
-
}
|
|
12656
|
-
const sources = Array.from(byUrl.values());
|
|
12657
|
-
let sourceTitle = expanded ? clean(item.querySelector(SEL.sourceTitle)?.textContent) : '';
|
|
12658
|
-
let sourceSite = expanded ? clean(item.querySelector(SEL.sourceSite)?.textContent) : '';
|
|
12659
|
-
let sourceCite = expanded ? externalHref(item.querySelector(SEL.sourceCite)) : '';
|
|
12660
|
-
const primary = sources.find(source => source.title.length > 2) || sources[0];
|
|
12661
|
-
if (!sourceCite && primary) {
|
|
12662
|
-
sourceTitle ||= primary.title;
|
|
12663
|
-
sourceSite ||= primary.site;
|
|
12664
|
-
sourceCite = primary.url;
|
|
12665
|
-
}
|
|
12666
|
-
return {
|
|
12667
|
-
question,
|
|
12668
|
-
answer: clean(answerElement?.innerText || answerElement?.textContent),
|
|
12669
|
-
sourceTitle,
|
|
12670
|
-
sourceSite,
|
|
12671
|
-
sourceCite,
|
|
12672
|
-
sources,
|
|
12673
|
-
depth: 1,
|
|
12674
|
-
parentQuestion: null,
|
|
12675
|
-
};
|
|
12676
|
-
}).filter(record => record.question);
|
|
12677
|
-
`;
|
|
12678
|
-
}
|
|
12679
|
-
function expansionCode(clickedQuestions) {
|
|
12680
|
-
return `
|
|
12681
|
-
const SEL = ${JSON.stringify(PAASelectors)};
|
|
12682
|
-
${recordReaderSource()}
|
|
12683
|
-
const deliveryBlock = async () => {
|
|
12684
|
-
if (/google\\.[^/]+\\/sorry\\//i.test(page.url())) return 'captcha';
|
|
12685
|
-
if (await page.locator(SEL.captchaMarker).count().catch(() => 0)) return 'captcha';
|
|
12686
|
-
const text = await page.locator('body').innerText({ timeout: 2000 }).catch(() => '');
|
|
12687
|
-
if (/recaptcha|unusual traffic|are you a robot|about this page/i.test(text)) return 'captcha';
|
|
12688
|
-
if (/\\b403\\b.{0,120}forbidden|access denied|permission to access/i.test(text.slice(0, 3000))) return 'soft_block';
|
|
12689
|
-
return null;
|
|
12690
|
-
};
|
|
12691
|
-
const blocked = await deliveryBlock();
|
|
12692
|
-
const records = await page.evaluate(({ SEL }) => {
|
|
12693
|
-
${recordReaderSource()}
|
|
12694
|
-
return readRecords();
|
|
12695
|
-
}, { SEL });
|
|
12696
|
-
if (blocked) return {
|
|
12697
|
-
outcome: blocked, targetQuestion: null, records, newQuestions: [],
|
|
12698
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
12699
|
-
interactionTrusted: null, clickError: null,
|
|
12700
|
-
};
|
|
12701
|
-
const clicked = new Set(${JSON.stringify(clickedQuestions)});
|
|
12702
|
-
const targetQuestion = records.map(record => record.question).find(question => !clicked.has(question)) || null;
|
|
12703
|
-
if (!targetQuestion) return {
|
|
12704
|
-
outcome: 'exhausted', targetQuestion: null, records, newQuestions: [],
|
|
12705
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
12706
|
-
interactionTrusted: null, clickError: null,
|
|
12707
|
-
};
|
|
12708
|
-
const items = page.locator(SEL.item);
|
|
12709
|
-
let targetIndex = -1;
|
|
12710
|
-
for (let index = 0, count = await items.count().catch(() => 0); index < count; index += 1) {
|
|
12711
|
-
const question = await items.nth(index).evaluate((item, SEL) =>
|
|
12712
|
-
item.getAttribute(SEL.itemDataQ)
|
|
12713
|
-
|| item.getAttribute(SEL.itemDataInitQ)
|
|
12714
|
-
|| item.querySelector(SEL.itemQuestionEl)?.innerText?.trim()
|
|
12715
|
-
|| '', SEL).catch(() => '');
|
|
12716
|
-
if (question.replace(/\\s+/g, ' ').trim() === targetQuestion) { targetIndex = index; break; }
|
|
12717
|
-
}
|
|
12718
|
-
if (targetIndex < 0) return {
|
|
12719
|
-
outcome: 'exhausted', targetQuestion, records, newQuestions: [],
|
|
12720
|
-
finalUrl: page.url(), title: await page.title().catch(() => ''),
|
|
12721
|
-
interactionTrusted: null, clickError: 'question disappeared before click',
|
|
12722
|
-
};
|
|
12723
|
-
const beforeQuestions = new Set(records.map(record => record.question));
|
|
12724
|
-
const item = items.nth(targetIndex);
|
|
12725
|
-
await item.scrollIntoViewIfNeeded({ timeout: 5000 }).catch(() => {});
|
|
12726
|
-
await page.waitForTimeout(180 + Math.floor(Math.random() * 420));
|
|
12727
|
-
await page.evaluate(() => { window.__mcpLastPaaClickTrusted = null; }).catch(() => {});
|
|
12728
|
-
let clickError = null;
|
|
12729
|
-
try {
|
|
12730
|
-
await item.locator(SEL.clickTarget).first().click({
|
|
12731
|
-
timeout: 6000,
|
|
12732
|
-
delay: 55 + Math.floor(Math.random() * 120),
|
|
12733
|
-
});
|
|
12734
|
-
} catch (error) {
|
|
12735
|
-
clickError = String(error && error.message || error).slice(0, 240);
|
|
12736
|
-
}
|
|
12737
|
-
const interactionTrusted = await page.evaluate(() => window.__mcpLastPaaClickTrusted ?? null).catch(() => null);
|
|
12738
|
-
await page.waitForTimeout(650 + Math.floor(Math.random() * 750));
|
|
12739
|
-
const afterRecords = await page.evaluate(({ SEL }) => {
|
|
12740
|
-
${recordReaderSource()}
|
|
12741
|
-
return readRecords();
|
|
12742
|
-
}, { SEL });
|
|
12743
|
-
return {
|
|
12744
|
-
outcome: clickError ? 'exhausted' : 'clicked',
|
|
12745
|
-
targetQuestion,
|
|
12746
|
-
records: afterRecords,
|
|
12747
|
-
newQuestions: afterRecords.map(record => record.question).filter(question => !beforeQuestions.has(question)),
|
|
12748
|
-
finalUrl: page.url(),
|
|
12749
|
-
title: await page.title().catch(() => ''),
|
|
12750
|
-
interactionTrusted,
|
|
12751
|
-
clickError,
|
|
12752
|
-
};
|
|
12753
|
-
`;
|
|
12754
|
-
}
|
|
12755
|
-
function finalCaptureCode(maxQuestions, parentByQuestion) {
|
|
12756
|
-
const surfaces = {
|
|
12757
|
-
aiOverview: AIOverviewSelectors,
|
|
12758
|
-
aiMode: AIModeSelectors,
|
|
12759
|
-
organic: OrganicSelectors,
|
|
12760
|
-
localPack: LocalPackSelectors,
|
|
12761
|
-
video: VideoSelectors,
|
|
12762
|
-
forum: ForumSelectors,
|
|
12763
|
-
whatPeopleSaying: WhatPeopleSayingSelectors
|
|
12764
|
-
};
|
|
12765
|
-
return `
|
|
12766
|
-
const SEL = ${JSON.stringify(PAASelectors)};
|
|
12767
|
-
const SURF = ${JSON.stringify(surfaces)};
|
|
12768
|
-
const PARENTS = ${JSON.stringify(parentByQuestion)};
|
|
12769
|
-
${recordReaderSource()}
|
|
12770
|
-
const capture = await page.evaluate(({ SEL, SURF, PARENTS, maxQuestions }) => {
|
|
12771
|
-
${recordReaderSource()}
|
|
12772
|
-
const unique = (items, key) => {
|
|
12773
|
-
const seen = new Set();
|
|
12774
|
-
return items.filter(item => { const value = key(item); if (!value || seen.has(value)) return false; seen.add(value); return true; });
|
|
12775
|
-
};
|
|
12776
|
-
const paa = readRecords().map(record => {
|
|
12777
|
-
const parentQuestion = PARENTS[record.question] || null;
|
|
12778
|
-
let depth = 1, parent = parentQuestion;
|
|
12779
|
-
const seen = new Set();
|
|
12780
|
-
while (parent && !seen.has(parent)) { seen.add(parent); depth += 1; parent = PARENTS[parent] || null; }
|
|
12781
|
-
return { ...record, depth, parentQuestion };
|
|
12782
|
-
}).slice(0, maxQuestions);
|
|
12783
|
-
const organicResults = [];
|
|
12784
|
-
document.querySelectorAll(SURF.organic.result).forEach(card => {
|
|
12785
|
-
const titleElement = card.querySelector(SURF.organic.title);
|
|
12786
|
-
const title = clean(titleElement?.textContent);
|
|
12787
|
-
const link = titleElement?.closest('a');
|
|
12788
|
-
const url = externalHref(link);
|
|
12789
|
-
if (!title || !url) return;
|
|
12790
|
-
let domain = '';
|
|
12791
|
-
try { domain = new URL(url).hostname.replace(/^www\\./, ''); } catch {}
|
|
12792
|
-
const rating = card.querySelector(SURF.organic.ratingWrap);
|
|
12793
|
-
organicResults.push({
|
|
12794
|
-
position: organicResults.length + 1, title, url, domain,
|
|
12795
|
-
cite: clean(card.querySelector(SURF.organic.cite)?.textContent) || null,
|
|
12796
|
-
snippet: clean(card.querySelector(SURF.organic.snippet)?.textContent) || null,
|
|
12797
|
-
isRedditStyle: Boolean(card.querySelector(SURF.organic.redditCite)),
|
|
12798
|
-
inlineRating: rating ? {
|
|
12799
|
-
value: clean(rating.querySelector(SURF.organic.ratingValue)?.textContent),
|
|
12800
|
-
count: clean(rating.querySelector(SURF.organic.reviewCount)?.textContent),
|
|
12801
|
-
} : null,
|
|
12802
|
-
});
|
|
12803
|
-
});
|
|
12804
|
-
const localPack = [];
|
|
12805
|
-
document.querySelectorAll(SURF.localPack.card).forEach(card => {
|
|
12806
|
-
const name = clean(card.querySelector(SURF.localPack.name)?.textContent);
|
|
12807
|
-
if (!name) return;
|
|
12808
|
-
const links = Array.from(card.querySelectorAll('a[href]'));
|
|
12809
|
-
let cid = card.querySelector('a[data-cid]')?.getAttribute('data-cid') || null;
|
|
12810
|
-
for (const link of links) {
|
|
12811
|
-
if (cid) break;
|
|
12812
|
-
const match = link.href.match(/[?&]cid=(\\d+)/);
|
|
12813
|
-
if (match) cid = match[1];
|
|
12814
|
-
}
|
|
12815
|
-
localPack.push({
|
|
12816
|
-
position: localPack.length + 1, name, cid,
|
|
12817
|
-
rating: clean(card.querySelector(SURF.localPack.ratingValue)?.textContent) || null,
|
|
12818
|
-
reviewCount: clean(card.querySelector(SURF.localPack.reviewCount)?.textContent).replace(/[()]/g, '') || null,
|
|
12819
|
-
metadata: [], websiteUrl: links.map(externalHref).find(Boolean) || null,
|
|
12820
|
-
directionsUrl: links.find(link => link.href.includes('google.com/maps'))?.href || null,
|
|
12821
|
-
});
|
|
12822
|
-
});
|
|
12823
|
-
const videos = unique(Array.from(document.querySelectorAll(SURF.video.item)).map(link => {
|
|
12824
|
-
const url = externalHref(link);
|
|
12825
|
-
const raw = clean(link.textContent);
|
|
12826
|
-
const platform = /youtu/i.test(url) ? 'YouTube' : /tiktok/i.test(url) ? 'TikTok' : /instagram/i.test(url) ? 'Instagram' : /facebook/i.test(url) ? 'Facebook' : '';
|
|
12827
|
-
return { type: 'video', title: raw, channel: '', platform, duration: raw.match(/\\b\\d{1,2}:\\d{2}\\b/)?.[0] || '', url };
|
|
12828
|
-
}).filter(item => item.title && item.url), item => item.url);
|
|
12829
|
-
const forums = unique(Array.from(document.querySelectorAll(SURF.forum.item)).map(link => ({
|
|
12830
|
-
title: clean(link.querySelector(SURF.forum.title)?.textContent),
|
|
12831
|
-
source: clean(link.querySelector(SURF.forum.source)?.textContent),
|
|
12832
|
-
url: externalHref(link),
|
|
12833
|
-
})).filter(item => item.title && item.url), item => item.url);
|
|
12834
|
-
const socialCards = Array.from(document.querySelectorAll(SURF.whatPeopleSaying.card));
|
|
12835
|
-
const whatPeopleSaying = unique(socialCards.map(card => {
|
|
12836
|
-
const link = card.querySelector(SURF.whatPeopleSaying.cardLink);
|
|
12837
|
-
const url = externalHref(link);
|
|
12838
|
-
const title = clean(card.querySelector(SURF.whatPeopleSaying.titleH1)?.textContent || card.querySelector(SURF.whatPeopleSaying.titleDiv)?.textContent);
|
|
12839
|
-
const source = clean(card.querySelector(SURF.whatPeopleSaying.ytChannel)?.textContent || card.querySelector(SURF.whatPeopleSaying.source)?.textContent);
|
|
12840
|
-
const platform = clean(card.querySelector(SURF.whatPeopleSaying.platformBadge)?.textContent) || (/youtube/i.test(url) ? 'YouTube' : '');
|
|
12841
|
-
const identity = (platform + source + url).toLowerCase();
|
|
12842
|
-
const type = identity.includes('reddit') ? 'reddit' : identity.includes('facebook') ? 'facebook' : identity.includes('instagram') ? 'instagram' : identity.includes('tiktok') ? 'tiktok' : identity.includes('youtube') ? 'youtube' : 'unknown';
|
|
12843
|
-
return { type, title, url, source, platform, popularComment: null, engagement: '', date: '', duration: null, authorNote: null };
|
|
12844
|
-
}).filter(item => item.title || item.url), item => item.url || item.type + ':' + item.title);
|
|
12845
|
-
const aioRoot = document.querySelector(SURF.aiOverview.root) || document.querySelector(SURF.aiOverview.legacyRoot);
|
|
12846
|
-
const aimRoot = document.querySelector(SURF.aiMode.root);
|
|
12847
|
-
const citations = root => unique(Array.from(root?.querySelectorAll('a[href]') || []).map(link => ({
|
|
12848
|
-
text: clean(link.textContent || link.getAttribute('aria-label')), href: externalHref(link),
|
|
12849
|
-
})).filter(item => item.href), item => item.href);
|
|
12850
|
-
const aiOverview = { detected: Boolean(aioRoot), text: aioRoot ? clean(aioRoot.innerText || aioRoot.textContent) : null, citations: citations(aioRoot), expanded: true, fullyExpanded: true, sections: [] };
|
|
12851
|
-
const aiMode = { detected: Boolean(aimRoot), text: aimRoot ? clean(aimRoot.innerText || aimRoot.textContent) : null, citations: citations(aimRoot) };
|
|
12852
|
-
const html = document.documentElement.outerHTML;
|
|
12853
|
-
const kgIds = unique(Array.from(html.matchAll(/\\/g\\/[a-zA-Z0-9_-]{5,20}/g)).map(match => match[0]), value => value);
|
|
12854
|
-
const cids = unique([
|
|
12855
|
-
...localPack.map(item => item.cid).filter(Boolean),
|
|
12856
|
-
...Array.from(html.matchAll(/[?&]cid=(\\d+)/g)).map(match => match[1]),
|
|
12857
|
-
], value => value);
|
|
12858
|
-
const gcids = unique(Array.from(html.matchAll(/gcid:[a-zA-Z0-9_]+/g)).map(match => match[0]), value => value);
|
|
12859
|
-
const entities = localPack.filter(item => item.cid).map(item => ({ name: item.name, kgId: null, cid: item.cid, gcid: null }));
|
|
12860
|
-
const headings = unique(Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"]')).map(node => clean(node.textContent)).filter(text => text.length > 1 && text.length < 160), value => value);
|
|
12861
|
-
const relatedSearches = unique(Array.from(document.querySelectorAll('a[href*="/search?"]')).map(link => ({
|
|
12862
|
-
text: clean(link.textContent), url: link.href,
|
|
12863
|
-
})).filter(item => item.text && /related/i.test(item.url + ' ' + item.text)), item => item.url);
|
|
12864
|
-
const counts = {
|
|
12865
|
-
peopleAlsoAsk: paa.length, organic: organicResults.length, localPack: localPack.length,
|
|
12866
|
-
videos: videos.length, discussions: forums.length, whatPeopleSaying: whatPeopleSaying.length,
|
|
12867
|
-
aiOverview: aiOverview.detected ? 1 : 0, aiMode: aiMode.detected ? 1 : 0,
|
|
12868
|
-
relatedSearches: relatedSearches.length,
|
|
12869
|
-
};
|
|
12870
|
-
return {
|
|
12871
|
-
paa, organicResults: unique(organicResults, item => item.url), localPack: unique(localPack, item => item.cid || item.name),
|
|
12872
|
-
videos, forums, whatPeopleSaying, aiOverview, aiMode,
|
|
12873
|
-
surface: window.google?.sn === 'aim' ? 'aim' : window.google?.sn === 'web' ? 'web' : 'unknown',
|
|
12874
|
-
entityIds: { entities, kgIds, cids, gcids },
|
|
12875
|
-
serpFeatures: { sectionHeadings: headings, detected: Object.fromEntries(Object.entries(counts).map(([key, count]) => [key, count > 0])), counts },
|
|
12876
|
-
relatedSearches,
|
|
12877
|
-
};
|
|
12878
|
-
}, { SEL, SURF, PARENTS, maxQuestions: ${maxQuestions} });
|
|
12879
|
-
return capture;
|
|
12880
|
-
`;
|
|
12881
|
-
}
|
|
12882
|
-
var RAW_PREPARE_CODE = `
|
|
12883
|
-
const result = await page.evaluate(async () => {
|
|
12884
|
-
const source = new TextEncoder().encode(document.documentElement.outerHTML);
|
|
12885
|
-
const compressed = typeof CompressionStream === 'undefined'
|
|
12886
|
-
? source
|
|
12887
|
-
: new Uint8Array(await new Response(new Blob([source]).stream().pipeThrough(new CompressionStream('gzip'))).arrayBuffer());
|
|
12888
|
-
const pieces = [];
|
|
12889
|
-
for (let offset = 0; offset < compressed.length; offset += 32768) {
|
|
12890
|
-
pieces.push(String.fromCharCode(...compressed.subarray(offset, offset + 32768)));
|
|
12891
|
-
}
|
|
12892
|
-
window.__mcpPaaRawDom = btoa(pieces.join(''));
|
|
12893
|
-
return { encoding: typeof CompressionStream === 'undefined' ? 'plain' : 'gzip', chars: window.__mcpPaaRawDom.length };
|
|
12894
|
-
});
|
|
12895
|
-
return result;
|
|
12896
|
-
`;
|
|
12897
|
-
function rawChunkCode(offset, length) {
|
|
12898
|
-
return `return await page.evaluate(({ offset, length }) => ({ offset, chunk: window.__mcpPaaRawDom.slice(offset, offset + length) }), { offset: ${offset}, length: ${length} });`;
|
|
12899
|
-
}
|
|
12900
|
-
function buildPublicResult(input) {
|
|
12901
|
-
const extractedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
12902
|
-
const selected = input.records;
|
|
12903
|
-
const flat = selected.map((record) => ({
|
|
12904
|
-
seed_query: input.query,
|
|
12905
|
-
question: record.question,
|
|
12906
|
-
answer: record.answer,
|
|
12907
|
-
source_title: record.sourceTitle,
|
|
12908
|
-
source_site: record.sourceSite,
|
|
12909
|
-
source_cite: record.sourceCite,
|
|
12910
|
-
depth: record.depth,
|
|
12911
|
-
parent_question: record.parentQuestion ?? "",
|
|
12912
|
-
extracted_at: extractedAt
|
|
12913
|
-
}));
|
|
12914
|
-
const tree = selected.map((record) => ({
|
|
12915
|
-
question: record.question,
|
|
12916
|
-
answer: record.answer || null,
|
|
12917
|
-
sourceTitle: record.sourceTitle || null,
|
|
12918
|
-
sourceSite: record.sourceSite || null,
|
|
12919
|
-
sourceCite: record.sourceCite || null,
|
|
12920
|
-
depth: record.depth,
|
|
12921
|
-
parentQuestion: record.parentQuestion,
|
|
12922
|
-
children: []
|
|
12923
|
-
}));
|
|
12924
|
-
return {
|
|
12925
|
-
seed: input.query,
|
|
12926
|
-
location: input.location,
|
|
12927
|
-
extractedAt,
|
|
12928
|
-
diagnostics: {
|
|
12929
|
-
completionStatus: selected.length > 0 ? "paa_found" : "no_paa",
|
|
12930
|
-
problem: null,
|
|
12931
|
-
resultQuality: input.partial ? "partial" : "complete",
|
|
12932
|
-
degradedResult: input.partial,
|
|
12933
|
-
retryRecommended: false,
|
|
12934
|
-
completeness: {
|
|
12935
|
-
paaWithoutAnswer: selected.filter((record) => !record.answer).length,
|
|
12936
|
-
paaWithoutSource: selected.filter((record) => !record.sourceCite).length,
|
|
12937
|
-
paaAnswersRecovered: 0,
|
|
12938
|
-
aioShareCaptured: null
|
|
12939
|
-
}
|
|
12940
|
-
},
|
|
12941
|
-
totalQuestions: selected.length,
|
|
12942
|
-
surface: input.capture.surface,
|
|
12943
|
-
aiOverview: input.capture.aiOverview,
|
|
12944
|
-
aiMode: input.capture.aiMode,
|
|
12945
|
-
whatPeopleSaying: input.capture.whatPeopleSaying,
|
|
12946
|
-
tree,
|
|
12947
|
-
flat,
|
|
12948
|
-
videos: input.capture.videos,
|
|
12949
|
-
forums: input.capture.forums,
|
|
12950
|
-
organicResults: input.capture.organicResults,
|
|
12951
|
-
localPack: input.capture.localPack,
|
|
12952
|
-
entityIds: input.capture.entityIds,
|
|
12953
|
-
serpFeatures: input.capture.serpFeatures,
|
|
12954
|
-
relatedSearches: input.capture.relatedSearches,
|
|
12955
|
-
stats: {
|
|
12956
|
-
seed: input.query,
|
|
12957
|
-
totalQuestions: selected.length,
|
|
12958
|
-
maxDepthReached: selected.reduce((max, item) => Math.max(max, item.depth), 0),
|
|
12959
|
-
durationMs: input.durationMs,
|
|
12960
|
-
errorCount: input.partial ? 1 : 0
|
|
12961
|
-
},
|
|
12962
|
-
capture: {
|
|
12963
|
-
execution: "kernel_colocated_playwright",
|
|
12964
|
-
stealth: true,
|
|
12965
|
-
managedProxy: input.persistentIdentity ? "serp_identity" : "kernel_default",
|
|
12966
|
-
metadataOverrides: false,
|
|
12967
|
-
locationSignal: input.location ? "uule_only" : "none",
|
|
12968
|
-
attempt: input.attempt,
|
|
12969
|
-
finalUrl: input.navigation.finalUrl,
|
|
12970
|
-
captchaObserved: input.navigation.captchaObserved,
|
|
12971
|
-
solverCleared: input.navigation.solverCleared,
|
|
12972
|
-
environment: input.navigation.environment,
|
|
12973
|
-
rawDomSha256: input.rawDomSha256
|
|
12974
|
-
}
|
|
12975
|
-
};
|
|
12976
|
-
}
|
|
12977
|
-
async function runDurablePaaCapture(input) {
|
|
12978
|
-
const startedAt = input.startedAtMs ?? Date.now();
|
|
12979
|
-
const workDeadline = startedAt + PAA_BROWSER_WORK_BUDGET_MS;
|
|
12980
|
-
const location2 = input.location?.trim() || null;
|
|
12981
|
-
const params = new URLSearchParams({
|
|
12982
|
-
q: input.query,
|
|
12983
|
-
gl: input.gl ?? "us",
|
|
12984
|
-
hl: input.hl ?? "en",
|
|
12985
|
-
pws: "0"
|
|
12986
|
-
});
|
|
12987
|
-
if (location2) params.set("uule", encodeUule(normalizeLocation(location2)));
|
|
12988
|
-
const searchUrl = `https://www.google.com/search?${params.toString()}`;
|
|
12989
|
-
const kernel = new Kernel4({ apiKey: input.apiKey });
|
|
12990
|
-
let lastCheckpoint = {
|
|
12991
|
-
phase: "navigating",
|
|
12992
|
-
attempt: 1,
|
|
12993
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
12994
|
-
records: [],
|
|
12995
|
-
clickedQuestions: [],
|
|
12996
|
-
finalUrl: null,
|
|
12997
|
-
captchaObserved: false,
|
|
12998
|
-
solverCleared: false,
|
|
12999
|
-
error: null
|
|
13000
|
-
};
|
|
13001
|
-
const persist = async (patch) => {
|
|
13002
|
-
lastCheckpoint = { ...lastCheckpoint, ...patch, observedAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
13003
|
-
await input.onCheckpoint?.(lastCheckpoint);
|
|
13004
|
-
};
|
|
13005
|
-
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
|
13006
|
-
if (Date.now() >= workDeadline) break;
|
|
13007
|
-
let sessionId = null;
|
|
13008
|
-
const records = /* @__PURE__ */ new Map();
|
|
13009
|
-
const clicked = /* @__PURE__ */ new Set();
|
|
13010
|
-
const parents = {};
|
|
13011
|
-
let navigation = null;
|
|
13012
|
-
try {
|
|
13013
|
-
await persist({ phase: "navigating", attempt, records: [], clickedQuestions: [], error: null });
|
|
13014
|
-
const browser = await kernel.browsers.create({
|
|
13015
|
-
stealth: true,
|
|
13016
|
-
timeout_seconds: Math.max(60, Math.min(280, Math.ceil((workDeadline - Date.now()) / 1e3) + 20)),
|
|
13017
|
-
...input.proxyId ? { proxy_id: input.proxyId } : {},
|
|
13018
|
-
...input.profileName ? { profile: { name: input.profileName, save_changes: false } } : {}
|
|
13019
|
-
});
|
|
13020
|
-
sessionId = browser.session_id;
|
|
13021
|
-
navigation = await execute(kernel, sessionId, navigationCode(searchUrl), workDeadline, 110);
|
|
13022
|
-
await persist({
|
|
13023
|
-
phase: navigation.outcome === "ready" ? "expanding" : "blocked",
|
|
13024
|
-
finalUrl: navigation.finalUrl,
|
|
13025
|
-
captchaObserved: navigation.captchaObserved,
|
|
13026
|
-
solverCleared: navigation.solverCleared
|
|
13027
|
-
});
|
|
13028
|
-
if (navigation.outcome === "captcha" || navigation.outcome === "soft_block") {
|
|
13029
|
-
if (attempt === 1 && records.size === 0) continue;
|
|
13030
|
-
throw new Error(navigation.outcome);
|
|
13031
|
-
}
|
|
13032
|
-
let stagnant = 0;
|
|
13033
|
-
for (let step = 0; step < input.maxQuestions * 2 && Date.now() < workDeadline - 35e3; step += 1) {
|
|
13034
|
-
const observed = await execute(
|
|
13035
|
-
kernel,
|
|
13036
|
-
sessionId,
|
|
13037
|
-
expansionCode(Array.from(clicked)),
|
|
13038
|
-
workDeadline,
|
|
13039
|
-
25
|
|
13040
|
-
);
|
|
13041
|
-
const beforeSize = records.size;
|
|
13042
|
-
for (const record of observed.records) {
|
|
13043
|
-
const existing = records.get(record.question);
|
|
13044
|
-
records.set(record.question, {
|
|
13045
|
-
...record,
|
|
13046
|
-
answer: record.answer || existing?.answer || "",
|
|
13047
|
-
sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
|
|
13048
|
-
sourceSite: record.sourceSite || existing?.sourceSite || "",
|
|
13049
|
-
sourceCite: record.sourceCite || existing?.sourceCite || "",
|
|
13050
|
-
sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
|
|
13051
|
-
parentQuestion: parents[record.question] ?? null,
|
|
13052
|
-
depth: parents[record.question] ? 2 : 1
|
|
13053
|
-
});
|
|
13054
|
-
}
|
|
13055
|
-
if (observed.outcome === "clicked" && observed.targetQuestion) {
|
|
13056
|
-
clicked.add(observed.targetQuestion);
|
|
13057
|
-
for (const question of observed.newQuestions) {
|
|
13058
|
-
if (!parents[question]) parents[question] = observed.targetQuestion;
|
|
13059
|
-
}
|
|
13060
|
-
}
|
|
13061
|
-
const selected2 = Array.from(records.values()).slice(0, input.maxQuestions);
|
|
13062
|
-
await persist({
|
|
13063
|
-
phase: "expanding",
|
|
13064
|
-
records: selected2,
|
|
13065
|
-
clickedQuestions: Array.from(clicked),
|
|
13066
|
-
finalUrl: observed.finalUrl,
|
|
13067
|
-
captchaObserved: lastCheckpoint.captchaObserved || observed.outcome === "captcha"
|
|
13068
|
-
});
|
|
13069
|
-
if (observed.outcome === "captcha" || observed.outcome === "soft_block") break;
|
|
13070
|
-
if (selected2.length >= input.maxQuestions && selected2.every((record) => clicked.has(record.question))) break;
|
|
13071
|
-
stagnant = records.size === beforeSize || observed.outcome === "exhausted" ? stagnant + 1 : 0;
|
|
13072
|
-
if (stagnant >= 4) break;
|
|
13073
|
-
}
|
|
13074
|
-
const finalCapture = await execute(
|
|
13075
|
-
kernel,
|
|
13076
|
-
sessionId,
|
|
13077
|
-
finalCaptureCode(input.maxQuestions, parents),
|
|
13078
|
-
workDeadline,
|
|
13079
|
-
45
|
|
13080
|
-
);
|
|
13081
|
-
for (const record of finalCapture.paa) {
|
|
13082
|
-
const existing = records.get(record.question);
|
|
13083
|
-
records.set(record.question, {
|
|
13084
|
-
...record,
|
|
13085
|
-
answer: record.answer || existing?.answer || "",
|
|
13086
|
-
sourceTitle: record.sourceTitle || existing?.sourceTitle || "",
|
|
13087
|
-
sourceSite: record.sourceSite || existing?.sourceSite || "",
|
|
13088
|
-
sourceCite: record.sourceCite || existing?.sourceCite || "",
|
|
13089
|
-
sources: Array.from(new Map([...existing?.sources ?? [], ...record.sources].map((source) => [source.url, source])).values()),
|
|
13090
|
-
parentQuestion: parents[record.question] ?? record.parentQuestion,
|
|
13091
|
-
depth: record.depth
|
|
13092
|
-
});
|
|
13093
|
-
}
|
|
13094
|
-
let rawDomGzip = null;
|
|
13095
|
-
let rawDomSha256 = null;
|
|
13096
|
-
if (Date.now() < workDeadline - 12e3) {
|
|
13097
|
-
const prepared = await execute(
|
|
13098
|
-
kernel,
|
|
13099
|
-
sessionId,
|
|
13100
|
-
RAW_PREPARE_CODE,
|
|
13101
|
-
workDeadline,
|
|
13102
|
-
35
|
|
13103
|
-
);
|
|
13104
|
-
const chunks = [];
|
|
13105
|
-
for (let offset = 0; offset < prepared.chars && Date.now() < workDeadline - 5e3; offset += 18e4) {
|
|
13106
|
-
const part = await execute(
|
|
13107
|
-
kernel,
|
|
13108
|
-
sessionId,
|
|
13109
|
-
rawChunkCode(offset, 18e4),
|
|
13110
|
-
workDeadline,
|
|
13111
|
-
15
|
|
13112
|
-
);
|
|
13113
|
-
if (part.offset !== offset) throw new Error("raw_dom_chunk_offset_mismatch");
|
|
13114
|
-
chunks.push(part.chunk);
|
|
13115
|
-
}
|
|
13116
|
-
if (chunks.join("").length === prepared.chars) {
|
|
13117
|
-
const transported = Buffer.from(chunks.join(""), "base64");
|
|
13118
|
-
if (prepared.encoding === "gzip") rawDomGzip = transported;
|
|
13119
|
-
else {
|
|
13120
|
-
const { gzipSync } = await import("zlib");
|
|
13121
|
-
rawDomGzip = gzipSync(transported);
|
|
13122
|
-
}
|
|
13123
|
-
rawDomSha256 = createHash2("sha256").update(rawDomGzip).digest("hex");
|
|
13124
|
-
}
|
|
13125
|
-
}
|
|
13126
|
-
const selected = Array.from(records.values()).slice(0, input.maxQuestions);
|
|
13127
|
-
const partial = selected.length < input.maxQuestions || selected.some((record) => !record.answer || !record.sourceCite);
|
|
13128
|
-
await persist({ phase: "captured", records: selected, clickedQuestions: Array.from(clicked) });
|
|
13129
|
-
return {
|
|
13130
|
-
result: buildPublicResult({
|
|
13131
|
-
query: input.query,
|
|
13132
|
-
location: location2,
|
|
13133
|
-
capture: finalCapture,
|
|
13134
|
-
records: selected,
|
|
13135
|
-
navigation,
|
|
13136
|
-
durationMs: Date.now() - startedAt,
|
|
13137
|
-
partial,
|
|
13138
|
-
attempt,
|
|
13139
|
-
rawDomSha256,
|
|
13140
|
-
persistentIdentity: Boolean(input.proxyId && input.profileName)
|
|
13141
|
-
}),
|
|
13142
|
-
rawDomGzip,
|
|
13143
|
-
rawDomSha256,
|
|
13144
|
-
attempt,
|
|
13145
|
-
partial,
|
|
13146
|
-
checkpoint: lastCheckpoint
|
|
13147
|
-
};
|
|
13148
|
-
} catch (error) {
|
|
13149
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
13150
|
-
await persist({
|
|
13151
|
-
phase: records.size > 0 ? "captured" : "failed",
|
|
13152
|
-
records: Array.from(records.values()).slice(0, input.maxQuestions),
|
|
13153
|
-
clickedQuestions: Array.from(clicked),
|
|
13154
|
-
error: message.slice(0, 500)
|
|
13155
|
-
});
|
|
13156
|
-
if (records.size > 0) {
|
|
13157
|
-
const emptyCapture = {
|
|
13158
|
-
paa: [],
|
|
13159
|
-
organicResults: [],
|
|
13160
|
-
localPack: [],
|
|
13161
|
-
videos: [],
|
|
13162
|
-
forums: [],
|
|
13163
|
-
whatPeopleSaying: [],
|
|
13164
|
-
aiOverview: { detected: false, text: null, citations: [] },
|
|
13165
|
-
aiMode: { detected: false, text: null, citations: [] },
|
|
13166
|
-
surface: "unknown",
|
|
13167
|
-
entityIds: { entities: [], kgIds: [], cids: [], gcids: [] },
|
|
13168
|
-
serpFeatures: { sectionHeadings: [], detected: {}, counts: {} },
|
|
13169
|
-
relatedSearches: []
|
|
13170
|
-
};
|
|
13171
|
-
return {
|
|
13172
|
-
result: buildPublicResult({
|
|
13173
|
-
query: input.query,
|
|
13174
|
-
location: location2,
|
|
13175
|
-
capture: emptyCapture,
|
|
13176
|
-
records: Array.from(records.values()).slice(0, input.maxQuestions),
|
|
13177
|
-
navigation: navigation ?? {
|
|
13178
|
-
outcome: "soft_block",
|
|
13179
|
-
finalUrl: lastCheckpoint.finalUrl ?? "",
|
|
13180
|
-
title: "",
|
|
13181
|
-
captchaObserved: lastCheckpoint.captchaObserved,
|
|
13182
|
-
solverCleared: lastCheckpoint.solverCleared,
|
|
13183
|
-
environment: null
|
|
13184
|
-
},
|
|
13185
|
-
durationMs: Date.now() - startedAt,
|
|
13186
|
-
partial: true,
|
|
13187
|
-
attempt,
|
|
13188
|
-
rawDomSha256: null,
|
|
13189
|
-
persistentIdentity: Boolean(input.proxyId && input.profileName)
|
|
13190
|
-
}),
|
|
13191
|
-
rawDomGzip: null,
|
|
13192
|
-
rawDomSha256: null,
|
|
13193
|
-
attempt,
|
|
13194
|
-
partial: true,
|
|
13195
|
-
checkpoint: lastCheckpoint
|
|
13196
|
-
};
|
|
13197
|
-
}
|
|
13198
|
-
if (attempt === 2 || Date.now() >= workDeadline - 35e3) throw error;
|
|
13199
|
-
} finally {
|
|
13200
|
-
if (sessionId) {
|
|
13201
|
-
await kernel.browsers.deleteByID(sessionId).catch(() => void 0);
|
|
13202
|
-
}
|
|
13203
|
-
}
|
|
13204
|
-
}
|
|
13205
|
-
throw new Error("paa_work_deadline_exhausted_before_capture");
|
|
13206
|
-
}
|
|
13207
|
-
|
|
13208
|
-
// src/inngest/functions/paa-harvest.ts
|
|
13209
|
-
var BRIGHTDATA_INVOCATION_BUDGET_MS = 76e4;
|
|
13210
|
-
var BRIGHTDATA_WORK_BUDGET_MS = 72e4;
|
|
13211
|
-
function durableRecordsFromResult(result) {
|
|
13212
|
-
const flat = Array.isArray(result.flat) ? result.flat : [];
|
|
13213
|
-
return flat.flatMap((value) => {
|
|
13214
|
-
if (!value || typeof value !== "object") return [];
|
|
13215
|
-
const row = value;
|
|
13216
|
-
const question = typeof row.question === "string" ? row.question : "";
|
|
13217
|
-
if (!question) return [];
|
|
13218
|
-
const sourceCite = typeof row.source_cite === "string" ? row.source_cite : "";
|
|
13219
|
-
const sourceTitle = typeof row.source_title === "string" ? row.source_title : "";
|
|
13220
|
-
const sourceSite = typeof row.source_site === "string" ? row.source_site : "";
|
|
13221
|
-
return [{
|
|
13222
|
-
question,
|
|
13223
|
-
answer: typeof row.answer === "string" ? row.answer : "",
|
|
13224
|
-
sourceTitle,
|
|
13225
|
-
sourceSite,
|
|
13226
|
-
sourceCite,
|
|
13227
|
-
sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
|
|
13228
|
-
depth: typeof row.depth === "number" ? row.depth : 1,
|
|
13229
|
-
parentQuestion: typeof row.parent_question === "string" && row.parent_question ? row.parent_question : null
|
|
13230
|
-
}];
|
|
13231
|
-
});
|
|
13232
|
-
}
|
|
13233
|
-
function questionCount(result) {
|
|
13234
|
-
if (!result || typeof result !== "object") return 0;
|
|
13235
|
-
const value = result;
|
|
13236
|
-
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
13237
|
-
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
13238
|
-
}
|
|
13239
|
-
var paaHarvestFn = inngest.createFunction(
|
|
13240
|
-
{
|
|
13241
|
-
id: "paa-harvest",
|
|
13242
|
-
retries: 0,
|
|
13243
|
-
triggers: [{ event: "mcp-scraper/paa.requested" }],
|
|
13244
|
-
onFailure: async ({ event: event2 }) => {
|
|
13245
|
-
const jobId2 = event2?.data?.event?.data?.jobId;
|
|
13246
|
-
if (!jobId2) return;
|
|
13247
|
-
const current = await getJob(jobId2);
|
|
13248
|
-
if (!current) return;
|
|
13249
|
-
if (current.status === "done" || current.status === "failed" || current.status === "cancelled") {
|
|
13250
|
-
await settlePaaHarvestJob(jobId2).catch((error) => {
|
|
13251
|
-
console.error("[paa-harvest/onFailure] terminal settlement pending:", error instanceof Error ? error.message : String(error));
|
|
13252
|
-
});
|
|
13253
|
-
return;
|
|
13254
|
-
}
|
|
13255
|
-
const rawError = String(event2?.data?.error?.message ?? "durable PAA invocation failed").slice(0, 1e3);
|
|
13256
|
-
const problem = classifyHarvestProblem(new Error(rawError));
|
|
13257
|
-
await failJob(jobId2, rawError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
13258
|
-
await settlePaaHarvestJob(jobId2).then(async (settled) => {
|
|
13259
|
-
if (!settled) return;
|
|
13260
|
-
const settledJob = await getJob(jobId2);
|
|
13261
|
-
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
13262
|
-
await failJob(jobId2, rawError, harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
13263
|
-
}).catch((error) => {
|
|
13264
|
-
console.error("[paa-harvest/onFailure] settlement pending:", error instanceof Error ? error.message : String(error));
|
|
13265
|
-
});
|
|
13266
|
-
}
|
|
13267
|
-
},
|
|
13268
|
-
async ({ event: event2 }) => {
|
|
13269
|
-
const invocationStartedAt = Date.now();
|
|
13270
|
-
const job = await claimInngestPaaJob(event2.data.jobId);
|
|
13271
|
-
if (!job) return { jobId: event2.data.jobId, status: "missing_or_not_paa" };
|
|
13272
|
-
if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
|
|
13273
|
-
await settlePaaHarvestJob(job.id);
|
|
13274
|
-
return { jobId: job.id, status: job.status };
|
|
13275
|
-
}
|
|
13276
|
-
const options = job.options;
|
|
13277
|
-
const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
|
|
13278
|
-
const serpIdentity = options.serpIdentity ? await getSerpIdentityRow(job.user_id, options.serpIdentity) : null;
|
|
13279
|
-
if (options.serpIdentity && !serpIdentity) throw new Error("SERP identity not found");
|
|
13280
|
-
if (serpIdentity && serpIdentity.status !== "ready") throw new Error("SERP identity is not ready");
|
|
13281
|
-
const useBrightData = brightDataSerpEnabled() && !serpIdentity;
|
|
13282
|
-
const apiKey = browserServiceApiKey();
|
|
13283
|
-
if (!useBrightData && !apiKey) throw new Error("browser service is not configured");
|
|
13284
|
-
const attemptStartedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
13285
|
-
await startHarvestAttempt({
|
|
13286
|
-
jobId: job.id,
|
|
13287
|
-
userId: job.user_id,
|
|
13288
|
-
attemptNumber: 1,
|
|
13289
|
-
maxAttempts: 2,
|
|
13290
|
-
query: options.query ?? job.query,
|
|
13291
|
-
location: options.location ?? null,
|
|
13292
|
-
maxQuestions,
|
|
13293
|
-
startedAt: attemptStartedAt
|
|
13294
|
-
});
|
|
13295
|
-
const checkpointState = { value: null };
|
|
13296
|
-
try {
|
|
13297
|
-
const persistCheckpoint = async (checkpoint, brightData = false) => {
|
|
13298
|
-
checkpointState.value = checkpoint;
|
|
13299
|
-
const persisted = await checkpointJob(job.id, {
|
|
13300
|
-
progress: checkpoint,
|
|
13301
|
-
durable: {
|
|
13302
|
-
invocationBudgetMs: brightData ? BRIGHTDATA_INVOCATION_BUDGET_MS : PAA_INVOCATION_BUDGET_MS,
|
|
13303
|
-
workBudgetMs: brightData ? BRIGHTDATA_WORK_BUDGET_MS : 25e4,
|
|
13304
|
-
cleanupReserveMs: brightData ? 4e4 : 3e4
|
|
13305
|
-
}
|
|
13306
|
-
});
|
|
13307
|
-
if (!persisted) throw new Error("job_left_running_state_during_capture");
|
|
13308
|
-
};
|
|
13309
|
-
const query = options.query ?? job.query;
|
|
13310
|
-
const capture = useBrightData ? await (async () => {
|
|
13311
|
-
const navigating = {
|
|
13312
|
-
phase: "navigating",
|
|
13313
|
-
attempt: 1,
|
|
13314
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13315
|
-
records: [],
|
|
13316
|
-
clickedQuestions: [],
|
|
13317
|
-
finalUrl: null,
|
|
13318
|
-
captchaObserved: false,
|
|
13319
|
-
solverCleared: false,
|
|
13320
|
-
error: null
|
|
13321
|
-
};
|
|
13322
|
-
await persistCheckpoint(navigating, true);
|
|
13323
|
-
const result2 = await harvest({
|
|
13324
|
-
query,
|
|
13325
|
-
location: options.location ?? void 0,
|
|
13326
|
-
gl: options.gl ?? "us",
|
|
13327
|
-
hl: options.hl ?? "en",
|
|
13328
|
-
maxQuestions,
|
|
13329
|
-
serpOnly: options.serpOnly ?? false,
|
|
13330
|
-
maxAttempts: 1,
|
|
13331
|
-
headless: true,
|
|
13332
|
-
format: "json",
|
|
13333
|
-
outputDir: "/tmp/paa-output-inngest",
|
|
13334
|
-
includeAllSerpFeatures: options.includeAllSerpFeatures ?? false,
|
|
13335
|
-
includeLocalPack: options.includeLocalPack ?? false,
|
|
13336
|
-
includeForums: options.includeForums ?? false,
|
|
13337
|
-
includeVideos: options.includeVideos ?? false,
|
|
13338
|
-
includeAiOverview: options.includeAiOverview ?? false,
|
|
13339
|
-
includeWhatPeopleSaying: options.includeWhatPeopleSaying ?? false,
|
|
13340
|
-
softDeadlineMs: invocationStartedAt + BRIGHTDATA_WORK_BUDGET_MS
|
|
13341
|
-
});
|
|
13342
|
-
const records = durableRecordsFromResult(result2);
|
|
13343
|
-
const captured = {
|
|
13344
|
-
...navigating,
|
|
13345
|
-
phase: "captured",
|
|
13346
|
-
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13347
|
-
records,
|
|
13348
|
-
clickedQuestions: records.map((record) => record.question)
|
|
13349
|
-
};
|
|
13350
|
-
await persistCheckpoint(captured, true);
|
|
13351
|
-
return {
|
|
13352
|
-
result: {
|
|
13353
|
-
...result2,
|
|
13354
|
-
capture: {
|
|
13355
|
-
execution: "brightdata_browser_api_controller_fulfilled",
|
|
13356
|
-
provider: "bright_data_browser_api",
|
|
13357
|
-
sameQueryClickGraph: true
|
|
13358
|
-
}
|
|
13359
|
-
},
|
|
13360
|
-
rawDomGzip: null,
|
|
13361
|
-
rawDomSha256: null,
|
|
13362
|
-
attempt: 1,
|
|
13363
|
-
partial: records.length < maxQuestions || records.some((record) => !record.answer || !record.sourceCite),
|
|
13364
|
-
checkpoint: captured
|
|
13365
|
-
};
|
|
13366
|
-
})() : await runDurablePaaCapture({
|
|
13367
|
-
apiKey,
|
|
13368
|
-
query,
|
|
13369
|
-
location: options.location ?? null,
|
|
13370
|
-
gl: options.gl ?? "us",
|
|
13371
|
-
hl: options.hl ?? "en",
|
|
13372
|
-
maxQuestions,
|
|
13373
|
-
...serpIdentity ? {
|
|
13374
|
-
proxyId: serpIdentity.kernel_proxy_id,
|
|
13375
|
-
profileName: serpIdentity.kernel_profile_name
|
|
13376
|
-
} : {},
|
|
13377
|
-
startedAtMs: invocationStartedAt,
|
|
13378
|
-
onCheckpoint: (checkpoint) => persistCheckpoint(checkpoint, false)
|
|
13379
|
-
});
|
|
13380
|
-
const artifactResult = capture.rawDomGzip ? await tryCreatePaaRawDomArtifact({
|
|
13381
|
-
userId: job.user_id,
|
|
13382
|
-
jobId: job.id,
|
|
13383
|
-
attempt: capture.attempt,
|
|
13384
|
-
gzip: capture.rawDomGzip
|
|
13385
|
-
}) : { artifact: null, status: "unavailable" };
|
|
13386
|
-
const artifact = artifactResult.artifact;
|
|
13387
|
-
const artifactMissing = capture.rawDomGzip !== null && artifactResult.status !== "stored";
|
|
13388
|
-
const captureResult = capture.result;
|
|
13389
|
-
const result = {
|
|
13390
|
-
...captureResult,
|
|
13391
|
-
diagnostics: {
|
|
13392
|
-
...captureResult.diagnostics,
|
|
13393
|
-
...artifactMissing ? {
|
|
13394
|
-
resultQuality: "partial",
|
|
13395
|
-
degradedResult: true,
|
|
13396
|
-
retryRecommended: false
|
|
13397
|
-
} : {}
|
|
13398
|
-
},
|
|
13399
|
-
capture: {
|
|
13400
|
-
...captureResult.capture,
|
|
13401
|
-
rawDomArtifact: artifact,
|
|
13402
|
-
rawDomArtifactStatus: artifactResult.status
|
|
13403
|
-
}
|
|
13404
|
-
};
|
|
13405
|
-
await completeJob(job.id, result);
|
|
13406
|
-
if (options.serpIdentity) await touchSerpIdentity(job.user_id, options.serpIdentity).catch(() => void 0);
|
|
13407
|
-
await finishHarvestAttempt({
|
|
13408
|
-
jobId: job.id,
|
|
13409
|
-
attemptNumber: 1,
|
|
13410
|
-
outcome: capture.partial ? "partial" : "paa_found",
|
|
13411
|
-
kernelSessionId: null,
|
|
13412
|
-
questionCount: questionCount(result),
|
|
13413
|
-
durationMs: Date.now() - invocationStartedAt,
|
|
13414
|
-
error: capture.checkpoint.error,
|
|
13415
|
-
willRetry: false,
|
|
13416
|
-
kernelDeleteStarted: true,
|
|
13417
|
-
kernelDeleteSucceeded: true,
|
|
13418
|
-
kernelDeleteError: null,
|
|
13419
|
-
browserCloseSucceeded: true,
|
|
13420
|
-
browserCloseError: null,
|
|
13421
|
-
debug: {
|
|
13422
|
-
execution: useBrightData ? "brightdata_browser_api_controller_fulfilled" : "kernel_colocated_playwright",
|
|
13423
|
-
stealth: true,
|
|
13424
|
-
attemptUsed: capture.attempt,
|
|
13425
|
-
rawDomStored: Boolean(artifact)
|
|
13426
|
-
},
|
|
13427
|
-
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13428
|
-
});
|
|
13429
|
-
await settlePaaHarvestJob(job.id);
|
|
13430
|
-
if (job.callback_url) {
|
|
13431
|
-
await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
|
|
13432
|
-
}
|
|
13433
|
-
return {
|
|
13434
|
-
jobId: job.id,
|
|
13435
|
-
status: "done",
|
|
13436
|
-
totalQuestions: questionCount(result),
|
|
13437
|
-
partial: capture.partial,
|
|
13438
|
-
artifactStored: Boolean(artifact)
|
|
13439
|
-
};
|
|
13440
|
-
} catch (error) {
|
|
13441
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
13442
|
-
const problem = classifyHarvestProblem(error);
|
|
13443
|
-
const latestCheckpoint = checkpointState.value;
|
|
13444
|
-
await finishHarvestAttempt({
|
|
13445
|
-
jobId: job.id,
|
|
13446
|
-
attemptNumber: 1,
|
|
13447
|
-
outcome: latestCheckpoint?.records.length ? "partial_session_lost" : "failed_before_capture",
|
|
13448
|
-
kernelSessionId: null,
|
|
13449
|
-
questionCount: latestCheckpoint?.records.length ?? 0,
|
|
13450
|
-
durationMs: Date.now() - invocationStartedAt,
|
|
13451
|
-
error: message.slice(0, 1e3),
|
|
13452
|
-
willRetry: false,
|
|
13453
|
-
kernelDeleteStarted: true,
|
|
13454
|
-
kernelDeleteSucceeded: null,
|
|
13455
|
-
kernelDeleteError: null,
|
|
13456
|
-
browserCloseSucceeded: null,
|
|
13457
|
-
browserCloseError: null,
|
|
13458
|
-
debug: latestCheckpoint,
|
|
13459
|
-
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13460
|
-
});
|
|
13461
|
-
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
13462
|
-
await settlePaaHarvestJob(job.id).then(async (settled) => {
|
|
13463
|
-
if (!settled) return;
|
|
13464
|
-
const settledJob = await getJob(job.id);
|
|
13465
|
-
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
13466
|
-
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
13467
|
-
}).catch((settlementError) => {
|
|
13468
|
-
console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
|
|
13469
|
-
});
|
|
13470
|
-
if (job.callback_url) {
|
|
13471
|
-
await deliverWebhook(job.callback_url, {
|
|
13472
|
-
job_id: job.id,
|
|
13473
|
-
status: "failed",
|
|
13474
|
-
...harvestProblemResponse(problem),
|
|
13475
|
-
progress: latestCheckpoint
|
|
13476
|
-
});
|
|
13477
|
-
}
|
|
13478
|
-
throw error;
|
|
13479
|
-
}
|
|
13480
|
-
}
|
|
13481
|
-
);
|
|
13482
|
-
|
|
13483
12415
|
// src/api/local-sourcebook-repository.ts
|
|
13484
|
-
import { randomUUID as
|
|
12416
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
13485
12417
|
var LOCAL_SOURCEBOOK_CATEGORIES = ["home", "professional", "restaurants", "financial", "realestate", "auto", "wellness"];
|
|
13486
12418
|
var schemaPromise2 = null;
|
|
13487
12419
|
var schemaDb2 = null;
|
|
@@ -13596,7 +12528,7 @@ function mapSubmission(row) {
|
|
|
13596
12528
|
async function event(submissionId, eventType, actorKind, actorId, metadata = {}) {
|
|
13597
12529
|
await getDb().execute({
|
|
13598
12530
|
sql: `INSERT INTO local_sourcebook_events (id, submission_id, event_type, actor_kind, actor_id, metadata_json) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
13599
|
-
args: [
|
|
12531
|
+
args: [randomUUID3(), submissionId, eventType, actorKind, actorId, JSON.stringify(metadata)]
|
|
13600
12532
|
});
|
|
13601
12533
|
}
|
|
13602
12534
|
async function createLocalSourcebookSubmission(input) {
|
|
@@ -13610,7 +12542,7 @@ async function createLocalSourcebookSubmission(input) {
|
|
|
13610
12542
|
return submission;
|
|
13611
12543
|
}
|
|
13612
12544
|
}
|
|
13613
|
-
const id = `lsb_${
|
|
12545
|
+
const id = `lsb_${randomUUID3().replace(/-/g, "")}`;
|
|
13614
12546
|
const coverage = {
|
|
13615
12547
|
requested: ["website_crawl", "structured_data", "services_products", "service_areas", "genuine_images", "staff_team", "review_sources"],
|
|
13616
12548
|
crawl: { state: "queued", pagesDiscovered: 0, pagesCaptured: 0 },
|
|
@@ -14700,7 +13632,7 @@ async function refundPersistedListingWrite(input) {
|
|
|
14700
13632
|
}
|
|
14701
13633
|
|
|
14702
13634
|
// src/api/local-sourcebook-compiler.ts
|
|
14703
|
-
import { createHash as
|
|
13635
|
+
import { createHash as createHash2 } from "crypto";
|
|
14704
13636
|
|
|
14705
13637
|
// src/api/local-sourcebook-public-urls.ts
|
|
14706
13638
|
var DEFAULT_ROOT_DOMAIN = "localsourcebook.com";
|
|
@@ -14812,7 +13744,7 @@ var CREDENTIAL_PATH = /\/(certifications?|credentials?|licenses?|accreditation|a
|
|
|
14812
13744
|
var BLOG_PATH = /\/(blog|articles?|resources?)\//i;
|
|
14813
13745
|
var LOCATION_PATH = /\/(?:locations?|service-areas?|areas-we-serve|communities|offices?)(?:\/|$|-)|(?:company|in|near|serving)-[a-z0-9-]+-[a-z]{2}$/i;
|
|
14814
13746
|
function digest(value) {
|
|
14815
|
-
return
|
|
13747
|
+
return createHash2("sha256").update(value).digest("hex").slice(0, 20);
|
|
14816
13748
|
}
|
|
14817
13749
|
function unique(values, limit = 100) {
|
|
14818
13750
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -16129,7 +15061,7 @@ import { Hono as Hono3 } from "hono";
|
|
|
16129
15061
|
import { z as z13 } from "zod";
|
|
16130
15062
|
|
|
16131
15063
|
// src/api/unified-billing.ts
|
|
16132
|
-
import { createHash as
|
|
15064
|
+
import { createHash as createHash3, createHmac, randomUUID as randomUUID4, timingSafeEqual } from "crypto";
|
|
16133
15065
|
|
|
16134
15066
|
// src/api/scheduling-access.ts
|
|
16135
15067
|
var PAID_PLAN_REQUIRED_CODE = "paid_plan_required";
|
|
@@ -16230,7 +15162,7 @@ function tokenFor(id, idempotencyKey4) {
|
|
|
16230
15162
|
return createHmac("sha256", billingSecret()).update(id).update(":").update(idempotencyKey4).digest("base64url");
|
|
16231
15163
|
}
|
|
16232
15164
|
function tokenHash(token6) {
|
|
16233
|
-
return
|
|
15165
|
+
return createHash3("sha256").update(token6).digest("hex");
|
|
16234
15166
|
}
|
|
16235
15167
|
function verifyToken(row, token6) {
|
|
16236
15168
|
const expected = Buffer.from(row.token_hash, "hex");
|
|
@@ -16350,7 +15282,7 @@ async function authorizeScheduledRun(args) {
|
|
|
16350
15282
|
{ balanceMc, requiredMc: SCHEDULED_RUN_BASE_MC }
|
|
16351
15283
|
);
|
|
16352
15284
|
}
|
|
16353
|
-
const id =
|
|
15285
|
+
const id = randomUUID4();
|
|
16354
15286
|
const token6 = tokenFor(id, idempotencyKey4);
|
|
16355
15287
|
const expiresAt = new Date(Date.now() + AUTHORIZATION_TTL_MS).toISOString();
|
|
16356
15288
|
const inserted = await getDb().execute({
|
|
@@ -16418,7 +15350,7 @@ async function startScheduledRun(args) {
|
|
|
16418
15350
|
if (!authorization || !["starting", "started"].includes(authorization.status)) {
|
|
16419
15351
|
throw new UnifiedBillingError("authorization_closed", "billing authorization is no longer startable", 409);
|
|
16420
15352
|
}
|
|
16421
|
-
const eventId = existing?.id ??
|
|
15353
|
+
const eventId = existing?.id ?? randomUUID4();
|
|
16422
15354
|
if (!existing) {
|
|
16423
15355
|
await getDb().execute({
|
|
16424
15356
|
sql: "INSERT OR IGNORE INTO billing_events (id, user_id, authorization_id, idempotency_key, billing_class, source_surface, status, amount_mc, multiplier_bps, metadata) VALUES (?, ?, ?, ?, ?, ?, 'capturing', ?, ?, ?)",
|
|
@@ -16557,7 +15489,7 @@ async function settleScheduledRun(args) {
|
|
|
16557
15489
|
...modelCostUnreported ? { modelCostUnreported: true } : {},
|
|
16558
15490
|
...pendingReason ? { reason: pendingReason } : {}
|
|
16559
15491
|
});
|
|
16560
|
-
const eventId = existing?.id ??
|
|
15492
|
+
const eventId = existing?.id ?? randomUUID4();
|
|
16561
15493
|
const status = pendingReason ? "cost_pending" : "settling";
|
|
16562
15494
|
const eventValues = [
|
|
16563
15495
|
modelMc,
|
|
@@ -16828,7 +15760,7 @@ async function syncScheduledActionCredentials(user) {
|
|
|
16828
15760
|
}
|
|
16829
15761
|
|
|
16830
15762
|
// src/api/connected-usage-billing.ts
|
|
16831
|
-
import { createHash as
|
|
15763
|
+
import { createHash as createHash4, randomUUID as randomUUID5 } from "crypto";
|
|
16832
15764
|
import { z as z12 } from "zod";
|
|
16833
15765
|
|
|
16834
15766
|
// src/api/connected-cost-telemetry.ts
|
|
@@ -16903,7 +15835,7 @@ var ConnectedUsageBillingError = class extends Error {
|
|
|
16903
15835
|
details;
|
|
16904
15836
|
};
|
|
16905
15837
|
function hash(value) {
|
|
16906
|
-
return
|
|
15838
|
+
return createHash4("sha256").update(value).digest("hex");
|
|
16907
15839
|
}
|
|
16908
15840
|
function eventKey(idempotencyKey4) {
|
|
16909
15841
|
return `connected-usage:${hash(idempotencyKey4)}`;
|
|
@@ -17175,7 +16107,7 @@ async function settleConnectedUsage(rawInput) {
|
|
|
17175
16107
|
sql: `INSERT OR IGNORE INTO billing_events
|
|
17176
16108
|
(id, user_id, idempotency_key, billing_class, source_surface, status, amount_mc, metadata)
|
|
17177
16109
|
VALUES (?, ?, ?, ?, ?, 'settling', ?, ?)`,
|
|
17178
|
-
args: [
|
|
16110
|
+
args: [randomUUID5(), user.id, key, CONNECTED_USAGE_BILLING_CLASS, input.sourceSurface, charge2.amountMc, encodeMetadata(storedMetadata)]
|
|
17179
16111
|
});
|
|
17180
16112
|
let event2 = await readEvent(key);
|
|
17181
16113
|
if (!event2) throw new Error("connected usage receipt insert completed without a readable event");
|
|
@@ -20793,7 +19725,7 @@ var adminAuth = createMiddleware3(async (c, next) => {
|
|
|
20793
19725
|
import { Hono as Hono9 } from "hono";
|
|
20794
19726
|
|
|
20795
19727
|
// src/api/admin-credits.ts
|
|
20796
|
-
import { randomUUID as
|
|
19728
|
+
import { randomUUID as randomUUID6 } from "crypto";
|
|
20797
19729
|
var MIN_ADJUSTMENT_CREDITS = 1;
|
|
20798
19730
|
var LARGE_ADJUSTMENT_CREDITS = 1e5;
|
|
20799
19731
|
var MAX_ADJUSTMENT_CREDITS = 1e7;
|
|
@@ -20943,7 +19875,7 @@ async function applyAdjustment(input) {
|
|
|
20943
19875
|
const credits = validateCredits(input.credits, input.confirmLarge === true);
|
|
20944
19876
|
const reason = validateReason(input.reason);
|
|
20945
19877
|
const actor = validateActor(input.actor);
|
|
20946
|
-
const reference = input.reference?.trim() || `adj_${
|
|
19878
|
+
const reference = input.reference?.trim() || `adj_${randomUUID6()}`;
|
|
20947
19879
|
if (reference.length > 120) throw new AdminCreditError(400, "reference must be at most 120 characters");
|
|
20948
19880
|
const db = getDb();
|
|
20949
19881
|
const existing = await db.execute({
|
|
@@ -23550,17 +22482,17 @@ g2App.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
23550
22482
|
});
|
|
23551
22483
|
|
|
23552
22484
|
// src/api/directory-routes.ts
|
|
23553
|
-
import { createHash as
|
|
22485
|
+
import { createHash as createHash5, randomUUID as randomUUID8 } from "crypto";
|
|
23554
22486
|
import { Hono as Hono16 } from "hono";
|
|
23555
22487
|
|
|
23556
22488
|
// src/api/directory-workflow-dispatch.ts
|
|
23557
|
-
import { randomUUID as
|
|
22489
|
+
import { randomUUID as randomUUID7 } from "crypto";
|
|
23558
22490
|
function retryDelaySeconds(attempts) {
|
|
23559
22491
|
return Math.min(60 * 60, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
|
|
23560
22492
|
}
|
|
23561
22493
|
async function dispatchPendingDirectoryWorkflows(limit = 25) {
|
|
23562
22494
|
const rows = await claimDirectoryWorkflowOutbox({
|
|
23563
|
-
workerId: `directory-dispatch-${process.pid}-${
|
|
22495
|
+
workerId: `directory-dispatch-${process.pid}-${randomUUID7().slice(0, 8)}`,
|
|
23564
22496
|
limit
|
|
23565
22497
|
});
|
|
23566
22498
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
@@ -23590,7 +22522,7 @@ function safeOptions(options) {
|
|
|
23590
22522
|
return safe;
|
|
23591
22523
|
}
|
|
23592
22524
|
function requestFingerprint(options) {
|
|
23593
|
-
return
|
|
22525
|
+
return createHash5("sha256").update(JSON.stringify(safeOptions(options))).digest("hex");
|
|
23594
22526
|
}
|
|
23595
22527
|
function idempotencyKey(raw) {
|
|
23596
22528
|
if (raw !== void 0) {
|
|
@@ -23599,14 +22531,14 @@ function idempotencyKey(raw) {
|
|
|
23599
22531
|
if (trimmed.length > 500) return { ok: false, message: "Idempotency-Key must be 500 characters or fewer." };
|
|
23600
22532
|
return { ok: true, key: trimmed };
|
|
23601
22533
|
}
|
|
23602
|
-
return { ok: true, key: `directory-${
|
|
22534
|
+
return { ok: true, key: `directory-${randomUUID8()}` };
|
|
23603
22535
|
}
|
|
23604
22536
|
function debitKeyFor(userId, responseKey) {
|
|
23605
|
-
const digest2 =
|
|
22537
|
+
const digest2 = createHash5("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
23606
22538
|
return `directory-workflow:${userId}:${digest2}`;
|
|
23607
22539
|
}
|
|
23608
22540
|
function jobId() {
|
|
23609
|
-
return `dir_${
|
|
22541
|
+
return `dir_${randomUUID8().replace(/-/g, "")}`;
|
|
23610
22542
|
}
|
|
23611
22543
|
function publicStatus(job, result) {
|
|
23612
22544
|
if (job.status === "failed") return "failed";
|
|
@@ -23715,7 +22647,7 @@ async function runSynchronously(c, user, options, plan, responseKey) {
|
|
|
23715
22647
|
const csv = renderDirectoryWorkflowCsv(result);
|
|
23716
22648
|
const artifact = await createDirectoryCsvArtifact({
|
|
23717
22649
|
ownerId: String(user.id),
|
|
23718
|
-
jobId:
|
|
22650
|
+
jobId: createHash5("sha256").update(debitKey2).digest("hex").slice(0, 32),
|
|
23719
22651
|
createdAt: result.extractedAt,
|
|
23720
22652
|
filename: `${options.state}-${options.query}-directory.csv`,
|
|
23721
22653
|
csv,
|
|
@@ -23910,17 +22842,17 @@ directoryApp.get("/jobs/:id", createApiKeyAuth(), async (c) => {
|
|
|
23910
22842
|
});
|
|
23911
22843
|
|
|
23912
22844
|
// src/api/lead-list-input-routes.ts
|
|
23913
|
-
import { createHash as
|
|
22845
|
+
import { createHash as createHash7 } from "crypto";
|
|
23914
22846
|
import { Hono as Hono17 } from "hono";
|
|
23915
22847
|
|
|
23916
22848
|
// src/api/lead-list-input-artifacts.ts
|
|
23917
|
-
import { createHash as
|
|
22849
|
+
import { createHash as createHash6 } from "crypto";
|
|
23918
22850
|
import { rm } from "fs/promises";
|
|
23919
22851
|
import { homedir as homedir3 } from "os";
|
|
23920
22852
|
import { join as join7 } from "path";
|
|
23921
22853
|
|
|
23922
22854
|
// src/api/lead-list-import-repository.ts
|
|
23923
|
-
import { randomUUID as
|
|
22855
|
+
import { randomUUID as randomUUID9 } from "crypto";
|
|
23924
22856
|
var schemaDb3 = null;
|
|
23925
22857
|
var schemaPromise3 = null;
|
|
23926
22858
|
function parseJson3(value, fallback) {
|
|
@@ -23986,10 +22918,10 @@ function bounded(value, field, min, max) {
|
|
|
23986
22918
|
return normalized;
|
|
23987
22919
|
}
|
|
23988
22920
|
function newLeadListUploadId() {
|
|
23989
|
-
return `upl_${
|
|
22921
|
+
return `upl_${randomUUID9().replace(/-/g, "")}`;
|
|
23990
22922
|
}
|
|
23991
22923
|
function newImportedLeadListId() {
|
|
23992
|
-
return `lst_${
|
|
22924
|
+
return `lst_${randomUUID9().replace(/-/g, "")}`;
|
|
23993
22925
|
}
|
|
23994
22926
|
async function ensureLeadListImportRepositorySchema() {
|
|
23995
22927
|
const db = getDb();
|
|
@@ -24237,7 +23169,7 @@ function safeFilenameHint(value) {
|
|
|
24237
23169
|
return cleaned || null;
|
|
24238
23170
|
}
|
|
24239
23171
|
function requestFingerprint2(filenameHint) {
|
|
24240
|
-
return
|
|
23172
|
+
return createHash6("sha256").update(JSON.stringify({ filenameHint })).digest("hex");
|
|
24241
23173
|
}
|
|
24242
23174
|
function publicBaseUrl() {
|
|
24243
23175
|
const configured = process.env.MCP_SCRAPER_PUBLIC_BASE_URL?.trim() || process.env.PUBLIC_BASE_URL?.trim() || (process.env.VERCEL_PROJECT_PRODUCTION_URL ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}` : "") || "https://mcpscraper.dev";
|
|
@@ -24393,7 +23325,7 @@ async function inspectLeadListUpload(uploadId, ownerId2) {
|
|
|
24393
23325
|
maxBytes: LEAD_LIST_UPLOAD_MAX_BYTES
|
|
24394
23326
|
});
|
|
24395
23327
|
if (!buffer) return null;
|
|
24396
|
-
const sha2564 =
|
|
23328
|
+
const sha2564 = createHash6("sha256").update(buffer).digest("hex");
|
|
24397
23329
|
const completed = await completeLeadListUpload({
|
|
24398
23330
|
id: record.id,
|
|
24399
23331
|
ownerId: record.ownerId,
|
|
@@ -24876,7 +23808,7 @@ function canonicalize(value) {
|
|
|
24876
23808
|
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalize(object[key])}`).join(",")}}`;
|
|
24877
23809
|
}
|
|
24878
23810
|
function fingerprint(value) {
|
|
24879
|
-
return
|
|
23811
|
+
return createHash7("sha256").update(canonicalize(value)).digest("hex");
|
|
24880
23812
|
}
|
|
24881
23813
|
async function safeJson(c) {
|
|
24882
23814
|
try {
|
|
@@ -25131,17 +24063,17 @@ leadListInputApp.post("/imports", async (c) => {
|
|
|
25131
24063
|
});
|
|
25132
24064
|
|
|
25133
24065
|
// src/api/lead-list-enrichment-routes.ts
|
|
25134
|
-
import { createHash as
|
|
24066
|
+
import { createHash as createHash8, randomUUID as randomUUID11 } from "crypto";
|
|
25135
24067
|
import { Hono as Hono18 } from "hono";
|
|
25136
24068
|
import { z as z25 } from "zod";
|
|
25137
24069
|
|
|
25138
24070
|
// src/api/lead-list-enrichment-dispatch.ts
|
|
25139
|
-
import { randomUUID as
|
|
24071
|
+
import { randomUUID as randomUUID10 } from "crypto";
|
|
25140
24072
|
function retryDelaySeconds2(attempts) {
|
|
25141
24073
|
return Math.min(3600, Math.max(15, 15 * 2 ** Math.min(8, Math.max(0, attempts - 1))));
|
|
25142
24074
|
}
|
|
25143
24075
|
async function dispatchPendingLeadListEnrichments(limit = 25) {
|
|
25144
|
-
const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${
|
|
24076
|
+
const rows = await claimLeadListEnrichmentOutbox({ workerId: `lead-list-dispatch-${process.pid}-${randomUUID10().slice(0, 8)}`, limit });
|
|
25145
24077
|
const result = { claimed: rows.length, dispatched: 0, failed: 0 };
|
|
25146
24078
|
for (const row of rows) {
|
|
25147
24079
|
if (!row.claimToken) continue;
|
|
@@ -25189,15 +24121,15 @@ function idempotencyKey2(raw) {
|
|
|
25189
24121
|
return { ok: true, value };
|
|
25190
24122
|
}
|
|
25191
24123
|
function newJobId() {
|
|
25192
|
-
return `lle_${
|
|
24124
|
+
return `lle_${randomUUID11().replace(/-/g, "")}`;
|
|
25193
24125
|
}
|
|
25194
24126
|
function debitKeyFor2(userId, key) {
|
|
25195
|
-
const digest2 =
|
|
24127
|
+
const digest2 = createHash8("sha256").update(String(userId)).update("\0").update(key).digest("hex");
|
|
25196
24128
|
return `lead-list-enrichment:${userId}:${digest2}`;
|
|
25197
24129
|
}
|
|
25198
24130
|
function inlineSourceDigest(headers, rows) {
|
|
25199
24131
|
const values = rows.map((row) => headers.map((header) => row[header] ?? null));
|
|
25200
|
-
return
|
|
24132
|
+
return createHash8("sha256").update(stableCanonicalJson({ headers, values })).digest("hex");
|
|
25201
24133
|
}
|
|
25202
24134
|
function issueMessage(error) {
|
|
25203
24135
|
const issue = error.issues[0];
|
|
@@ -26277,43 +25209,230 @@ locationDataApp.post("/admin/sync-census", async (c) => {
|
|
|
26277
25209
|
}
|
|
26278
25210
|
});
|
|
26279
25211
|
|
|
26280
|
-
// src/api/paa-harvest-
|
|
26281
|
-
|
|
26282
|
-
|
|
26283
|
-
|
|
26284
|
-
|
|
26285
|
-
|
|
25212
|
+
// src/api/paa-harvest-direct.ts
|
|
25213
|
+
import { waitUntil } from "@vercel/functions";
|
|
25214
|
+
|
|
25215
|
+
// src/api/paa-harvest-settlement.ts
|
|
25216
|
+
function capturedQuestionCount(result) {
|
|
25217
|
+
if (!result || typeof result !== "object") return 0;
|
|
25218
|
+
const value = result;
|
|
25219
|
+
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
25220
|
+
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
25221
|
+
}
|
|
25222
|
+
function finalCost(result, heldMc) {
|
|
25223
|
+
const questions = capturedQuestionCount(result);
|
|
25224
|
+
if (questions <= 0) return 0;
|
|
25225
|
+
return Math.min(heldMc, MC_COSTS.paa_base + questions * MC_COSTS.paa);
|
|
25226
|
+
}
|
|
25227
|
+
async function settlePaaHarvestJob(jobOrId) {
|
|
25228
|
+
const job = typeof jobOrId === "string" ? await getJob(jobOrId) : jobOrId;
|
|
25229
|
+
if (!job) return false;
|
|
25230
|
+
const options = job.options;
|
|
25231
|
+
const debitKey2 = options.billingDebitKey;
|
|
25232
|
+
const heldMc = Number(options.billingHoldMc ?? 0);
|
|
25233
|
+
if (!debitKey2 || !Number.isSafeInteger(heldMc) || heldMc <= 0) return false;
|
|
25234
|
+
const settlement = await settleDebitMcIdempotent(
|
|
25235
|
+
job.user_id,
|
|
25236
|
+
debitKey2,
|
|
25237
|
+
finalCost(job.result, heldMc),
|
|
25238
|
+
LedgerOperation.PAA_REFUND,
|
|
25239
|
+
"durable PAA harvest settlement",
|
|
25240
|
+
"paa_harvest"
|
|
25241
|
+
);
|
|
25242
|
+
await markJobBillingSettled(job.id, settlement.final_amount_mc);
|
|
25243
|
+
return true;
|
|
25244
|
+
}
|
|
25245
|
+
|
|
25246
|
+
// src/api/paa-harvest-runner.ts
|
|
25247
|
+
var CLEANUP_RESERVE_MS = 4e4;
|
|
25248
|
+
var DIRECT_PAA_MAX_ATTEMPTS = 3;
|
|
25249
|
+
function durableRecordsFromResult(result) {
|
|
25250
|
+
const flat = Array.isArray(result.flat) ? result.flat : [];
|
|
25251
|
+
return flat.flatMap((value) => {
|
|
25252
|
+
if (!value || typeof value !== "object") return [];
|
|
25253
|
+
const row = value;
|
|
25254
|
+
const question = typeof row.question === "string" ? row.question : "";
|
|
25255
|
+
if (!question) return [];
|
|
25256
|
+
const sourceCite = typeof row.source_cite === "string" ? row.source_cite : "";
|
|
25257
|
+
const sourceTitle = typeof row.source_title === "string" ? row.source_title : "";
|
|
25258
|
+
const sourceSite = typeof row.source_site === "string" ? row.source_site : "";
|
|
25259
|
+
return [{
|
|
25260
|
+
question,
|
|
25261
|
+
answer: typeof row.answer === "string" ? row.answer : "",
|
|
25262
|
+
sourceTitle,
|
|
25263
|
+
sourceSite,
|
|
25264
|
+
sourceCite,
|
|
25265
|
+
sources: sourceCite ? [{ title: sourceTitle, site: sourceSite, url: sourceCite }] : [],
|
|
25266
|
+
depth: typeof row.depth === "number" ? row.depth : 1,
|
|
25267
|
+
parentQuestion: typeof row.parent_question === "string" && row.parent_question ? row.parent_question : null
|
|
25268
|
+
}];
|
|
26286
25269
|
});
|
|
26287
25270
|
}
|
|
26288
|
-
|
|
26289
|
-
|
|
26290
|
-
|
|
26291
|
-
|
|
26292
|
-
|
|
26293
|
-
|
|
26294
|
-
|
|
26295
|
-
|
|
26296
|
-
|
|
26297
|
-
|
|
25271
|
+
function questionCount(result) {
|
|
25272
|
+
if (!result || typeof result !== "object") return 0;
|
|
25273
|
+
const value = result;
|
|
25274
|
+
if (typeof value.totalQuestions === "number") return value.totalQuestions;
|
|
25275
|
+
return Array.isArray(value.progress?.records) ? value.progress.records.length : 0;
|
|
25276
|
+
}
|
|
25277
|
+
async function runPaaHarvestJob(jobId2) {
|
|
25278
|
+
const invocationStartedAt = Date.now();
|
|
25279
|
+
const job = await claimDurablePaaJob(jobId2);
|
|
25280
|
+
if (!job) return { jobId: jobId2, status: "missing_or_not_paa" };
|
|
25281
|
+
if (job.status === "done" || job.status === "failed" || job.status === "cancelled") {
|
|
25282
|
+
await settlePaaHarvestJob(job.id);
|
|
25283
|
+
return { jobId: job.id, status: job.status };
|
|
25284
|
+
}
|
|
25285
|
+
const options = job.options;
|
|
25286
|
+
const maxQuestions = Math.max(1, Math.min(100, Number(options.maxQuestions ?? 30)));
|
|
25287
|
+
const timeoutBudget = harvestTimeoutBudget(maxQuestions);
|
|
25288
|
+
const workBudgetMs = Math.max(1, timeoutBudget.serverMs - CLEANUP_RESERVE_MS);
|
|
25289
|
+
const checkpointState = { value: null };
|
|
25290
|
+
try {
|
|
25291
|
+
if (!brightDataSerpEnabled()) throw new Error("SERP browser service is not configured");
|
|
25292
|
+
const navigating = {
|
|
25293
|
+
phase: "navigating",
|
|
25294
|
+
attempt: 1,
|
|
25295
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
25296
|
+
records: [],
|
|
25297
|
+
clickedQuestions: [],
|
|
25298
|
+
finalUrl: null,
|
|
25299
|
+
captchaObserved: false,
|
|
25300
|
+
solverCleared: false,
|
|
25301
|
+
error: null
|
|
25302
|
+
};
|
|
25303
|
+
checkpointState.value = navigating;
|
|
25304
|
+
if (!await checkpointJob(job.id, {
|
|
25305
|
+
progress: navigating,
|
|
25306
|
+
durable: {
|
|
25307
|
+
invocationBudgetMs: timeoutBudget.serverMs,
|
|
25308
|
+
workBudgetMs,
|
|
25309
|
+
cleanupReserveMs: CLEANUP_RESERVE_MS
|
|
25310
|
+
}
|
|
25311
|
+
})) {
|
|
25312
|
+
throw new Error("job_left_running_state_during_capture");
|
|
25313
|
+
}
|
|
25314
|
+
const persistAttempt = createHarvestAttemptRecorder(job.id, job.user_id);
|
|
25315
|
+
let lastAttempt = 1;
|
|
25316
|
+
const onAttemptEvent = async (event2) => {
|
|
25317
|
+
lastAttempt = event2.attemptNumber;
|
|
25318
|
+
await persistAttempt(event2);
|
|
25319
|
+
};
|
|
25320
|
+
const harvestCtx = currentCostContext();
|
|
25321
|
+
const harvested = await runWithCostContext(
|
|
25322
|
+
{ ...harvestCtx, op: "paa", userId: job.user_id },
|
|
25323
|
+
() => harvest({
|
|
25324
|
+
query: options.query ?? job.query,
|
|
25325
|
+
location: options.location ?? void 0,
|
|
25326
|
+
gl: options.gl ?? "us",
|
|
25327
|
+
hl: options.hl ?? "en",
|
|
25328
|
+
maxQuestions,
|
|
25329
|
+
maxAttempts: DIRECT_PAA_MAX_ATTEMPTS,
|
|
25330
|
+
proxyMode: "none",
|
|
25331
|
+
forceManagedSerp: true,
|
|
25332
|
+
headless: true,
|
|
25333
|
+
format: "json",
|
|
25334
|
+
outputDir: "/tmp/paa-output-direct",
|
|
25335
|
+
includeAllSerpFeatures: options.includeAllSerpFeatures ?? false,
|
|
25336
|
+
includeLocalPack: options.includeLocalPack ?? false,
|
|
25337
|
+
includeForums: options.includeForums ?? false,
|
|
25338
|
+
includeVideos: options.includeVideos ?? false,
|
|
25339
|
+
includeAiOverview: options.includeAiOverview ?? false,
|
|
25340
|
+
includeWhatPeopleSaying: options.includeWhatPeopleSaying ?? false,
|
|
25341
|
+
softDeadlineMs: invocationStartedAt + workBudgetMs,
|
|
25342
|
+
onAttemptEvent
|
|
25343
|
+
})
|
|
25344
|
+
);
|
|
25345
|
+
const harvestResult = harvested;
|
|
25346
|
+
const records = durableRecordsFromResult(harvestResult);
|
|
25347
|
+
const captured = {
|
|
25348
|
+
...navigating,
|
|
25349
|
+
phase: "captured",
|
|
25350
|
+
attempt: lastAttempt,
|
|
25351
|
+
observedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
25352
|
+
records,
|
|
25353
|
+
clickedQuestions: records.map((record) => record.question)
|
|
25354
|
+
};
|
|
25355
|
+
checkpointState.value = captured;
|
|
25356
|
+
if (!await checkpointJob(job.id, {
|
|
25357
|
+
progress: captured,
|
|
25358
|
+
durable: {
|
|
25359
|
+
invocationBudgetMs: timeoutBudget.serverMs,
|
|
25360
|
+
workBudgetMs,
|
|
25361
|
+
cleanupReserveMs: CLEANUP_RESERVE_MS
|
|
25362
|
+
}
|
|
25363
|
+
})) {
|
|
25364
|
+
throw new Error("job_left_running_state_during_capture");
|
|
25365
|
+
}
|
|
25366
|
+
const result = {
|
|
25367
|
+
...harvestResult,
|
|
25368
|
+
capture: {
|
|
25369
|
+
...harvestResult.capture,
|
|
25370
|
+
execution: "brightdata_browser_api_controller_fulfilled",
|
|
25371
|
+
provider: "bright_data_browser_api",
|
|
25372
|
+
sameQueryClickGraph: true,
|
|
25373
|
+
rawDomArtifact: null,
|
|
25374
|
+
rawDomArtifactStatus: "unavailable"
|
|
25375
|
+
}
|
|
25376
|
+
};
|
|
25377
|
+
await completeJob(job.id, result);
|
|
25378
|
+
await settlePaaHarvestJob(job.id);
|
|
25379
|
+
if (job.callback_url) {
|
|
25380
|
+
await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
|
|
26298
25381
|
}
|
|
25382
|
+
return {
|
|
25383
|
+
jobId: job.id,
|
|
25384
|
+
status: "done",
|
|
25385
|
+
totalQuestions: questionCount(result),
|
|
25386
|
+
partial: records.length < maxQuestions || records.some((record) => !record.answer || !record.sourceCite),
|
|
25387
|
+
artifactStored: false
|
|
25388
|
+
};
|
|
25389
|
+
} catch (error) {
|
|
25390
|
+
const problem = classifyHarvestProblem(error);
|
|
25391
|
+
const latestCheckpoint = checkpointState.value;
|
|
25392
|
+
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }));
|
|
25393
|
+
await settlePaaHarvestJob(job.id).then(async (settled) => {
|
|
25394
|
+
if (!settled) return;
|
|
25395
|
+
const settledJob = await getJob(job.id);
|
|
25396
|
+
const billedMc = Number(settledJob?.options.billedMc ?? 0);
|
|
25397
|
+
await failJob(job.id, serializeHarvestProblem(problem), harvestProblemEnvelope(problem, { chargeStatus: billedMc > 0 ? "charged" : "refunded" }));
|
|
25398
|
+
}).catch((settlementError) => {
|
|
25399
|
+
console.error("[paa-harvest] settlement pending:", settlementError instanceof Error ? settlementError.message : String(settlementError));
|
|
25400
|
+
});
|
|
25401
|
+
if (job.callback_url) {
|
|
25402
|
+
await deliverWebhook(job.callback_url, {
|
|
25403
|
+
job_id: job.id,
|
|
25404
|
+
status: "failed",
|
|
25405
|
+
...harvestProblemResponse(problem),
|
|
25406
|
+
progress: latestCheckpoint
|
|
25407
|
+
});
|
|
25408
|
+
}
|
|
25409
|
+
throw error;
|
|
26299
25410
|
}
|
|
26300
|
-
|
|
25411
|
+
}
|
|
25412
|
+
|
|
25413
|
+
// src/api/paa-harvest-direct.ts
|
|
25414
|
+
function scheduleDirectPaaHarvest(jobId2) {
|
|
25415
|
+
const work = runPaaHarvestJob(jobId2).catch((error) => {
|
|
25416
|
+
console.error("[paa-harvest/direct] run failed:", error instanceof Error ? error.message : String(error));
|
|
25417
|
+
});
|
|
25418
|
+
waitUntil(work);
|
|
26301
25419
|
}
|
|
26302
25420
|
|
|
26303
25421
|
// src/api/paa-harvest-reconciliation.ts
|
|
26304
25422
|
async function reconcilePaaHarvestSettlements(limit = 25) {
|
|
26305
|
-
const stale = await
|
|
25423
|
+
const stale = await listStaleRunningDurablePaaJobs(limit);
|
|
26306
25424
|
let staleFailed = 0;
|
|
26307
25425
|
for (const job of stale) {
|
|
26308
25426
|
try {
|
|
26309
|
-
const privateError = "
|
|
25427
|
+
const privateError = "direct PAA invocation exceeded its platform safety window; preserved progress remains in result";
|
|
26310
25428
|
const problem = classifyHarvestProblem(new Error("harvest timeout"));
|
|
26311
|
-
await
|
|
26312
|
-
|
|
25429
|
+
if (await failRunningJob(job.id, privateError, harvestProblemEnvelope(problem, { chargeStatus: "refund_pending" }))) {
|
|
25430
|
+
staleFailed += 1;
|
|
25431
|
+
}
|
|
26313
25432
|
} catch {
|
|
26314
25433
|
}
|
|
26315
25434
|
}
|
|
26316
|
-
const jobs = await
|
|
25435
|
+
const jobs = await listRecentTerminalDurablePaaJobs(limit);
|
|
26317
25436
|
let settled = 0;
|
|
26318
25437
|
let failed = 0;
|
|
26319
25438
|
for (const job of jobs) {
|
|
@@ -27012,17 +26131,17 @@ workflowApp.post("/cron/dispatch", async (c) => {
|
|
|
27012
26131
|
|
|
27013
26132
|
// src/api/serp-intelligence-routes.ts
|
|
27014
26133
|
import { Hono as Hono22 } from "hono";
|
|
27015
|
-
import { createHash as
|
|
26134
|
+
import { createHash as createHash10, randomUUID as randomUUID12 } from "crypto";
|
|
27016
26135
|
|
|
27017
26136
|
// src/serp-intelligence/page-snapshot-extractor.ts
|
|
27018
|
-
import { createHash as
|
|
26137
|
+
import { createHash as createHash9 } from "crypto";
|
|
27019
26138
|
import pLimit5 from "p-limit";
|
|
27020
26139
|
var DEFAULT_TIMEOUT_MS = 15e3;
|
|
27021
26140
|
var DEFAULT_MAX_CONCURRENCY = 2;
|
|
27022
26141
|
var DEFAULT_MAX_CONTENT_CHARS = 25e4;
|
|
27023
26142
|
function sha256(value) {
|
|
27024
26143
|
if (!value) return null;
|
|
27025
|
-
return
|
|
26144
|
+
return createHash9("sha256").update(value).digest("hex");
|
|
27026
26145
|
}
|
|
27027
26146
|
function countWords(markdown) {
|
|
27028
26147
|
const matches = markdown.trim().match(/\b[\p{L}\p{N}][\p{L}\p{N}'-]*\b/gu);
|
|
@@ -27798,9 +26917,9 @@ async function logRequestEventBestEffort(input) {
|
|
|
27798
26917
|
}
|
|
27799
26918
|
}
|
|
27800
26919
|
function captureBillingKeys(userId, suppliedKey, body) {
|
|
27801
|
-
const responseKey = suppliedKey?.trim() ||
|
|
27802
|
-
const requestFingerprint3 =
|
|
27803
|
-
const keyDigest =
|
|
26920
|
+
const responseKey = suppliedKey?.trim() || randomUUID12();
|
|
26921
|
+
const requestFingerprint3 = createHash10("sha256").update(JSON.stringify(body)).digest("hex");
|
|
26922
|
+
const keyDigest = createHash10("sha256").update(String(userId)).update("\0").update(responseKey).digest("hex");
|
|
27804
26923
|
return {
|
|
27805
26924
|
responseKey,
|
|
27806
26925
|
debitKey: `serp-capture:${userId}:${keyDigest}`,
|
|
@@ -28128,7 +27247,7 @@ var MemoryMcpToolExecutor = class {
|
|
|
28128
27247
|
};
|
|
28129
27248
|
|
|
28130
27249
|
// src/api/image-source-artifacts.ts
|
|
28131
|
-
import { randomUUID as
|
|
27250
|
+
import { randomUUID as randomUUID13 } from "crypto";
|
|
28132
27251
|
var IMAGE_SOURCE_ARTIFACT_PREFIX = "image-sources/";
|
|
28133
27252
|
var IMAGE_SOURCE_ARTIFACT_TTL_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
28134
27253
|
var IMAGE_SOURCE_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
@@ -28159,7 +27278,7 @@ async function createImageSourceArtifact(args) {
|
|
|
28159
27278
|
if (args.content.length === 0 || args.content.length > IMAGE_SOURCE_MAX_BYTES) {
|
|
28160
27279
|
throw new Error("image_source_size_invalid");
|
|
28161
27280
|
}
|
|
28162
|
-
const id =
|
|
27281
|
+
const id = randomUUID13().replaceAll("-", "");
|
|
28163
27282
|
return createPrivateArtifact({
|
|
28164
27283
|
policy: policy3(),
|
|
28165
27284
|
ownerId: args.ownerId,
|
|
@@ -28908,7 +28027,7 @@ import { randomUUID as randomUUID17 } from "crypto";
|
|
|
28908
28027
|
import { Hono as Hono24 } from "hono";
|
|
28909
28028
|
|
|
28910
28029
|
// src/api/browser-agent-db.ts
|
|
28911
|
-
import { randomUUID as
|
|
28030
|
+
import { randomUUID as randomUUID14 } from "crypto";
|
|
28912
28031
|
var _ready = false;
|
|
28913
28032
|
var _migrationPromise = null;
|
|
28914
28033
|
async function migrateBrowserAgent() {
|
|
@@ -29039,7 +28158,7 @@ async function runBrowserAgentMigration() {
|
|
|
29039
28158
|
}
|
|
29040
28159
|
async function createExtensionRow(input) {
|
|
29041
28160
|
const db = getDb();
|
|
29042
|
-
const id = `bext_${
|
|
28161
|
+
const id = `bext_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
|
|
29043
28162
|
await db.execute({
|
|
29044
28163
|
sql: `INSERT INTO browser_agent_extensions (id, user_id, name, backend_id, backend_name, source, source_url, size_bytes)
|
|
29045
28164
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
@@ -29074,7 +28193,7 @@ async function deleteExtensionRow(userId, name) {
|
|
|
29074
28193
|
}
|
|
29075
28194
|
async function createAuthConnectionRow(input) {
|
|
29076
28195
|
const db = getDb();
|
|
29077
|
-
const connectionId = `authc_${
|
|
28196
|
+
const connectionId = `authc_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
|
|
29078
28197
|
await db.execute({
|
|
29079
28198
|
sql: `INSERT INTO browser_auth_connections (connection_id, domain, profile, account_email, note, status, browser_agent_session_id)
|
|
29080
28199
|
VALUES (?, ?, ?, ?, ?, 'NEEDS_AUTH', ?)`,
|
|
@@ -29159,7 +28278,7 @@ async function deleteProfileLabel(userId, profile) {
|
|
|
29159
28278
|
}
|
|
29160
28279
|
async function createSessionRow(input) {
|
|
29161
28280
|
const db = getDb();
|
|
29162
|
-
const id = `bas_${
|
|
28281
|
+
const id = `bas_${randomUUID14().replace(/-/g, "").slice(0, 20)}`;
|
|
29163
28282
|
await db.execute({
|
|
29164
28283
|
sql: `INSERT INTO browser_agent_sessions (id, runtime_session_id, live_view_url, cdp_ws_url, status, label, user_id, concurrency_lock_id, last_action_at)
|
|
29165
28284
|
VALUES (?, ?, ?, ?, 'open', ?, ?, ?, datetime('now'))`,
|
|
@@ -29237,7 +28356,7 @@ async function recordAction(input) {
|
|
|
29237
28356
|
sql: `INSERT INTO browser_agent_actions (id, session_id, type, params_json, ok, error)
|
|
29238
28357
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
29239
28358
|
args: [
|
|
29240
|
-
`baa_${
|
|
28359
|
+
`baa_${randomUUID14().replace(/-/g, "").slice(0, 20)}`,
|
|
29241
28360
|
input.sessionId,
|
|
29242
28361
|
input.type,
|
|
29243
28362
|
input.params == null ? null : JSON.stringify(input.params),
|
|
@@ -29279,7 +28398,7 @@ async function listReplayRows(sessionId) {
|
|
|
29279
28398
|
}
|
|
29280
28399
|
|
|
29281
28400
|
// src/services/browser-agent/browser-agent-service.ts
|
|
29282
|
-
import
|
|
28401
|
+
import Kernel4 from "@onkernel/sdk";
|
|
29283
28402
|
import { chromium as playwrightChromium } from "playwright";
|
|
29284
28403
|
|
|
29285
28404
|
// src/services/fanout/cdp-capture.ts
|
|
@@ -29988,7 +29107,7 @@ var DEFAULT_TIMEOUT_SECONDS = 600;
|
|
|
29988
29107
|
function client() {
|
|
29989
29108
|
const apiKey = browserServiceApiKey();
|
|
29990
29109
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
29991
|
-
return new
|
|
29110
|
+
return new Kernel4({ apiKey });
|
|
29992
29111
|
}
|
|
29993
29112
|
function isProfileConflict(err) {
|
|
29994
29113
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -30347,17 +29466,66 @@ async function replayList(runtimeSessionId) {
|
|
|
30347
29466
|
}));
|
|
30348
29467
|
}
|
|
30349
29468
|
|
|
29469
|
+
// src/api/serp-identity-db.ts
|
|
29470
|
+
import { randomUUID as randomUUID15 } from "crypto";
|
|
29471
|
+
async function createSerpIdentityRow(input) {
|
|
29472
|
+
const id = `serpi_${randomUUID15().replace(/-/g, "").slice(0, 20)}`;
|
|
29473
|
+
await getDb().execute({
|
|
29474
|
+
sql: `INSERT INTO serp_identities
|
|
29475
|
+
(id, user_id, name, kernel_profile_name, kernel_proxy_id, proxy_type, country, status)
|
|
29476
|
+
VALUES (?, ?, ?, ?, ?, 'isp', ?, 'ready')`,
|
|
29477
|
+
args: [id, input.userId, input.name, input.kernelProfileName, input.kernelProxyId, input.country]
|
|
29478
|
+
});
|
|
29479
|
+
const row = await getSerpIdentityRow(input.userId, input.name);
|
|
29480
|
+
if (!row) throw new Error("SERP identity insert failed");
|
|
29481
|
+
return row;
|
|
29482
|
+
}
|
|
29483
|
+
async function getSerpIdentityRow(userId, name) {
|
|
29484
|
+
const result = await getDb().execute({
|
|
29485
|
+
sql: `SELECT * FROM serp_identities WHERE user_id = ? AND name = ? LIMIT 1`,
|
|
29486
|
+
args: [userId, name]
|
|
29487
|
+
});
|
|
29488
|
+
return result.rows[0] ?? null;
|
|
29489
|
+
}
|
|
29490
|
+
async function listSerpIdentityRows(userId) {
|
|
29491
|
+
const result = await getDb().execute({
|
|
29492
|
+
sql: `SELECT * FROM serp_identities WHERE user_id = ? ORDER BY created_at DESC`,
|
|
29493
|
+
args: [userId]
|
|
29494
|
+
});
|
|
29495
|
+
return result.rows;
|
|
29496
|
+
}
|
|
29497
|
+
async function touchSerpIdentity(userId, name) {
|
|
29498
|
+
await getDb().execute({
|
|
29499
|
+
sql: `UPDATE serp_identities
|
|
29500
|
+
SET last_used_at = datetime('now'), updated_at = datetime('now')
|
|
29501
|
+
WHERE user_id = ? AND name = ?`,
|
|
29502
|
+
args: [userId, name]
|
|
29503
|
+
});
|
|
29504
|
+
}
|
|
29505
|
+
async function setSerpIdentityStatus(userId, name, status) {
|
|
29506
|
+
await getDb().execute({
|
|
29507
|
+
sql: `UPDATE serp_identities SET status = ?, updated_at = datetime('now') WHERE user_id = ? AND name = ?`,
|
|
29508
|
+
args: [status, userId, name]
|
|
29509
|
+
});
|
|
29510
|
+
}
|
|
29511
|
+
async function deleteSerpIdentityRow(userId, name) {
|
|
29512
|
+
await getDb().execute({
|
|
29513
|
+
sql: `DELETE FROM serp_identities WHERE user_id = ? AND name = ?`,
|
|
29514
|
+
args: [userId, name]
|
|
29515
|
+
});
|
|
29516
|
+
}
|
|
29517
|
+
|
|
30350
29518
|
// src/api/serp-identity-service.ts
|
|
30351
|
-
import { createHash as
|
|
30352
|
-
import
|
|
29519
|
+
import { createHash as createHash11, randomUUID as randomUUID16 } from "crypto";
|
|
29520
|
+
import Kernel5 from "@onkernel/sdk";
|
|
30353
29521
|
var MAX_SERP_IDENTITIES_PER_USER = 3;
|
|
30354
29522
|
function kernelClient() {
|
|
30355
29523
|
const apiKey = browserServiceApiKey();
|
|
30356
29524
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
30357
|
-
return new
|
|
29525
|
+
return new Kernel5({ apiKey });
|
|
30358
29526
|
}
|
|
30359
29527
|
function backendName(userId, name, resource) {
|
|
30360
|
-
const digest2 =
|
|
29528
|
+
const digest2 = createHash11("sha256").update(`${userId}:${name}`).digest("hex").slice(0, 16);
|
|
30361
29529
|
const nonce = randomUUID16().replace(/-/g, "").slice(0, 8);
|
|
30362
29530
|
return `mcp-serp-${resource}-${digest2}-${nonce}`;
|
|
30363
29531
|
}
|
|
@@ -31697,7 +30865,7 @@ async function resolveUser3(customerId, emailFallback) {
|
|
|
31697
30865
|
// src/api/oauth-routes.ts
|
|
31698
30866
|
import { Hono as Hono26 } from "hono";
|
|
31699
30867
|
import { getCookie, setCookie } from "hono/cookie";
|
|
31700
|
-
import { createHash as
|
|
30868
|
+
import { createHash as createHash12, randomBytes as randomBytes3, randomUUID as randomUUID18 } from "crypto";
|
|
31701
30869
|
import { importPKCS8, exportJWK, SignJWT } from "jose";
|
|
31702
30870
|
var ISSUER = "https://mcpscraper.dev";
|
|
31703
30871
|
var RESOURCE = () => process.env.MCP_MEMORY_RESOURCE_URL ?? "https://memory.mcpscraper.dev/mcp";
|
|
@@ -31755,7 +30923,7 @@ async function getKeys() {
|
|
|
31755
30923
|
const privateKey = await importPKCS8(pem, "RS256", { extractable: true });
|
|
31756
30924
|
const full = await exportJWK(privateKey);
|
|
31757
30925
|
const publicJwk = { kty: full.kty, n: full.n, e: full.e };
|
|
31758
|
-
const kid =
|
|
30926
|
+
const kid = createHash12("sha256").update(JSON.stringify({ e: publicJwk.e, kty: publicJwk.kty, n: publicJwk.n })).digest("base64url").slice(0, 16);
|
|
31759
30927
|
publicJwk.kid = kid;
|
|
31760
30928
|
publicJwk.alg = "RS256";
|
|
31761
30929
|
publicJwk.use = "sig";
|
|
@@ -32077,7 +31245,7 @@ oauthApp.post("/oauth/authorize", async (c) => {
|
|
|
32077
31245
|
});
|
|
32078
31246
|
function pkceMatches(verifier, challenge) {
|
|
32079
31247
|
if (!verifier) return false;
|
|
32080
|
-
const computed =
|
|
31248
|
+
const computed = createHash12("sha256").update(verifier).digest("base64url");
|
|
32081
31249
|
return computed === challenge;
|
|
32082
31250
|
}
|
|
32083
31251
|
async function mintAccessToken(identity, scope, plan, audience) {
|
|
@@ -33530,7 +32698,7 @@ import { Hono as Hono30 } from "hono";
|
|
|
33530
32698
|
import { Resend } from "resend";
|
|
33531
32699
|
|
|
33532
32700
|
// src/api/resend-support-thread.ts
|
|
33533
|
-
import { createHash as
|
|
32701
|
+
import { createHash as createHash13 } from "crypto";
|
|
33534
32702
|
var SUPPORT_TAGS = [
|
|
33535
32703
|
{
|
|
33536
32704
|
tag: "support",
|
|
@@ -33570,7 +32738,7 @@ function quoteUntrusted(value) {
|
|
|
33570
32738
|
return clean2.split("\n").map((line) => `> ${line}`).join("\n");
|
|
33571
32739
|
}
|
|
33572
32740
|
function shortHash(value, length = 24) {
|
|
33573
|
-
return
|
|
32741
|
+
return createHash13("sha256").update(value).digest("hex").slice(0, length);
|
|
33574
32742
|
}
|
|
33575
32743
|
function safePathPart(value) {
|
|
33576
32744
|
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48) || "contact";
|
|
@@ -35069,7 +34237,7 @@ import { Hono as Hono32 } from "hono";
|
|
|
35069
34237
|
import { z as z32 } from "zod";
|
|
35070
34238
|
|
|
35071
34239
|
// src/editorial-reading-room/render.ts
|
|
35072
|
-
import { createHash as
|
|
34240
|
+
import { createHash as createHash14 } from "crypto";
|
|
35073
34241
|
import { readFileSync as readFileSync3 } from "fs";
|
|
35074
34242
|
import { dirname as dirname3, join as join8, resolve } from "path";
|
|
35075
34243
|
import { Marked, Renderer } from "marked";
|
|
@@ -35230,7 +34398,7 @@ function renderEditorialReadingRoom(input, now = /* @__PURE__ */ new Date()) {
|
|
|
35230
34398
|
articleCount: articles.length,
|
|
35231
34399
|
wordCount: totalWordCount,
|
|
35232
34400
|
bytes,
|
|
35233
|
-
sha256:
|
|
34401
|
+
sha256: createHash14("sha256").update(html).digest("hex"),
|
|
35234
34402
|
warnings
|
|
35235
34403
|
};
|
|
35236
34404
|
}
|
|
@@ -35313,7 +34481,7 @@ ${provenance}`;
|
|
|
35313
34481
|
...rendered,
|
|
35314
34482
|
html,
|
|
35315
34483
|
bytes: Buffer.byteLength(html),
|
|
35316
|
-
sha256:
|
|
34484
|
+
sha256: createHash14("sha256").update(html).digest("hex")
|
|
35317
34485
|
};
|
|
35318
34486
|
}
|
|
35319
34487
|
|
|
@@ -35436,10 +34604,10 @@ import { Hono as Hono33 } from "hono";
|
|
|
35436
34604
|
import { z as z33 } from "zod";
|
|
35437
34605
|
|
|
35438
34606
|
// src/api/commons-repository.ts
|
|
35439
|
-
import { createHash as
|
|
34607
|
+
import { createHash as createHash17, randomUUID as randomUUID20 } from "crypto";
|
|
35440
34608
|
|
|
35441
34609
|
// src/api/commons-image-store.ts
|
|
35442
|
-
import { createHash as
|
|
34610
|
+
import { createHash as createHash15 } from "crypto";
|
|
35443
34611
|
import { lookup as dnsLookup } from "dns/promises";
|
|
35444
34612
|
import { isIP } from "net";
|
|
35445
34613
|
var COMMONS_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
@@ -35578,7 +34746,7 @@ async function hostCommonsImage(input) {
|
|
|
35578
34746
|
415
|
|
35579
34747
|
);
|
|
35580
34748
|
}
|
|
35581
|
-
const digest2 =
|
|
34749
|
+
const digest2 = createHash15("sha256").update(bytes).digest("hex");
|
|
35582
34750
|
const existing = await getDb().execute({
|
|
35583
34751
|
sql: "SELECT id, url, content_type, bytes, source_url FROM commons_images WHERE digest = ? LIMIT 1",
|
|
35584
34752
|
args: [digest2]
|
|
@@ -35693,7 +34861,7 @@ async function schemaObjectsPresent(names) {
|
|
|
35693
34861
|
}
|
|
35694
34862
|
|
|
35695
34863
|
// src/api/commons-linksets.ts
|
|
35696
|
-
import { createHash as
|
|
34864
|
+
import { createHash as createHash16 } from "crypto";
|
|
35697
34865
|
var COMMONS_LINKSET_MEDIA_TYPE = "application/linkset+json";
|
|
35698
34866
|
var COMMONS_RELATIONSHIP_PROFILE = "https://mcpscraper.dev/commons/profiles/relationships/v1";
|
|
35699
34867
|
var REGISTERED_RELATIONS = /* @__PURE__ */ new Set([
|
|
@@ -35800,7 +34968,7 @@ function buildCommonsLinkset(entity, claims) {
|
|
|
35800
34968
|
context[claim.predicate] = targets;
|
|
35801
34969
|
}
|
|
35802
34970
|
const document2 = { linkset: [context] };
|
|
35803
|
-
const etag = `"${
|
|
34971
|
+
const etag = `"${createHash16("sha256").update(JSON.stringify(document2)).digest("hex")}"`;
|
|
35804
34972
|
return { document: document2, etag, publicUrl, linksetUrl, profile: COMMONS_RELATIONSHIP_PROFILE };
|
|
35805
34973
|
}
|
|
35806
34974
|
function commonsLinksetDiscoveryHeader(idOrSlug) {
|
|
@@ -37582,7 +36750,7 @@ function commonsIndexDocuments(entity, now) {
|
|
|
37582
36750
|
return documents;
|
|
37583
36751
|
}
|
|
37584
36752
|
function commonsIndexDocumentId(entityId, documentType, documentKey) {
|
|
37585
|
-
return `commons-index-${
|
|
36753
|
+
return `commons-index-${createHash17("sha256").update(`${entityId}
|
|
37586
36754
|
${documentType}
|
|
37587
36755
|
${documentKey}`).digest("hex").slice(0, 32)}`;
|
|
37588
36756
|
}
|
|
@@ -38160,7 +37328,7 @@ var CommonsRepositoryError = class extends Error {
|
|
|
38160
37328
|
};
|
|
38161
37329
|
|
|
38162
37330
|
// src/api/commons-publication-repository.ts
|
|
38163
|
-
import { createHash as
|
|
37331
|
+
import { createHash as createHash18, randomUUID as randomUUID21 } from "crypto";
|
|
38164
37332
|
|
|
38165
37333
|
// src/editorial-reading-room/article-page.ts
|
|
38166
37334
|
function escapeHtml6(value) {
|
|
@@ -38449,7 +37617,7 @@ async function publishCommonsEditorial(input, user) {
|
|
|
38449
37617
|
JSON.stringify(editionInput.articles),
|
|
38450
37618
|
html,
|
|
38451
37619
|
rendered.filename,
|
|
38452
|
-
|
|
37620
|
+
createHash18("sha256").update(html).digest("hex"),
|
|
38453
37621
|
rendered.articleCount,
|
|
38454
37622
|
rendered.wordCount,
|
|
38455
37623
|
Buffer.byteLength(html),
|
|
@@ -38531,7 +37699,7 @@ async function updateCommonsEditorialArticle(input, user) {
|
|
|
38531
37699
|
JSON.stringify(nextArticles),
|
|
38532
37700
|
html,
|
|
38533
37701
|
rendered.filename,
|
|
38534
|
-
|
|
37702
|
+
createHash18("sha256").update(html).digest("hex"),
|
|
38535
37703
|
rendered.articleCount,
|
|
38536
37704
|
rendered.wordCount,
|
|
38537
37705
|
Buffer.byteLength(html),
|
|
@@ -39266,12 +38434,12 @@ function xmlEscape(value) {
|
|
|
39266
38434
|
import { Hono as Hono34 } from "hono";
|
|
39267
38435
|
import { createMiddleware as createMiddleware4 } from "hono/factory";
|
|
39268
38436
|
import { z as z35 } from "zod";
|
|
39269
|
-
import { createHash as
|
|
38437
|
+
import { createHash as createHash20 } from "crypto";
|
|
39270
38438
|
import Papa4 from "papaparse";
|
|
39271
38439
|
|
|
39272
38440
|
// src/api/analytics-entitlement.ts
|
|
39273
38441
|
import {
|
|
39274
|
-
createHash as
|
|
38442
|
+
createHash as createHash19,
|
|
39275
38443
|
createHmac as createHmac3,
|
|
39276
38444
|
randomBytes as randomBytes4,
|
|
39277
38445
|
timingSafeEqual as timingSafeEqual2
|
|
@@ -39486,7 +38654,7 @@ function dashboardCallbackUrl() {
|
|
|
39486
38654
|
async function createThorbitConnectUrl(userId) {
|
|
39487
38655
|
if (!bridgeConfigured()) throw new Error("Thorbit X-Ray account bridge is not configured");
|
|
39488
38656
|
const state = randomBytes4(32).toString("base64url");
|
|
39489
|
-
const stateHash =
|
|
38657
|
+
const stateHash = createHash19("sha256").update(state).digest("hex");
|
|
39490
38658
|
await getAnalyticsPool().query(
|
|
39491
38659
|
`INSERT INTO analytics_thorbit_connect_states(state_hash,user_id,expires_at)
|
|
39492
38660
|
VALUES ($1,$2,now()+interval '10 minutes')
|
|
@@ -39515,7 +38683,7 @@ function verifyThorbitAssertion(token6) {
|
|
|
39515
38683
|
return parsed.data.entitlement;
|
|
39516
38684
|
}
|
|
39517
38685
|
async function consumeThorbitConnectCallback(input) {
|
|
39518
|
-
const stateHash =
|
|
38686
|
+
const stateHash = createHash19("sha256").update(input.state).digest("hex");
|
|
39519
38687
|
const client2 = await getAnalyticsPool().connect();
|
|
39520
38688
|
try {
|
|
39521
38689
|
await client2.query("BEGIN");
|
|
@@ -39991,7 +39159,7 @@ analyticsApp.post("/forms/:publicId/submissions", async (c) => {
|
|
|
39991
39159
|
const lastName = typeof data.last_name === "string" ? data.last_name.trim() : "";
|
|
39992
39160
|
const fullName = [firstName, lastName].filter(Boolean).join(" ") || (typeof data.name === "string" ? data.name.trim() : "") || email || "Website lead";
|
|
39993
39161
|
const identitySeed = email || `${fullName}:${Date.now()}`;
|
|
39994
|
-
const suffix2 =
|
|
39162
|
+
const suffix2 = createHash20("sha256").update(identitySeed).digest("hex").slice(0, 12);
|
|
39995
39163
|
const path5 = `Leads/person-${suffix2}`;
|
|
39996
39164
|
const { key: memoryKey, error: memoryKeyError } = await getOrCreateUserMemoryKey(user);
|
|
39997
39165
|
const existing = memoryKey ? await memoryCall("getTool", { vault: "People", path: path5 }, memoryKey) : { ok: false, error: memoryKeyError || "memory_credential_unavailable" };
|
|
@@ -40090,9 +39258,9 @@ Submitted the published form.
|
|
|
40090
39258
|
occurredAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
40091
39259
|
});
|
|
40092
39260
|
const match = {
|
|
40093
|
-
...email ? { emailSha256:
|
|
39261
|
+
...email ? { emailSha256: createHash20("sha256").update(email).digest("hex") } : {},
|
|
40094
39262
|
...phone ? {
|
|
40095
|
-
phoneSha256:
|
|
39263
|
+
phoneSha256: createHash20("sha256").update(phone.replace(/\D/g, "")).digest("hex")
|
|
40096
39264
|
} : {}
|
|
40097
39265
|
};
|
|
40098
39266
|
const activationQueued = await queueAnalyticsActivation({
|
|
@@ -40520,7 +39688,7 @@ analyticsApp.post("/sites/:siteId/crm-imports", auth3, async (c) => {
|
|
|
40520
39688
|
rejectedCount += 1;
|
|
40521
39689
|
continue;
|
|
40522
39690
|
}
|
|
40523
|
-
const digest2 =
|
|
39691
|
+
const digest2 = createHash20("sha256").update(
|
|
40524
39692
|
`${parsed.data.sourceSystem}:${externalId || email || phone || `${fullName}:${index}`}`
|
|
40525
39693
|
).digest("hex").slice(0, 16);
|
|
40526
39694
|
const path5 = `Leads/person-${digest2}`;
|
|
@@ -40971,13 +40139,13 @@ import { Hono as Hono35 } from "hono";
|
|
|
40971
40139
|
import { z as z36 } from "zod";
|
|
40972
40140
|
|
|
40973
40141
|
// src/api/scheduled-artifact-owner.ts
|
|
40974
|
-
import { createHash as
|
|
40142
|
+
import { createHash as createHash21 } from "crypto";
|
|
40975
40143
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
40976
|
-
return
|
|
40144
|
+
return createHash21("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
40977
40145
|
}
|
|
40978
40146
|
|
|
40979
40147
|
// src/api/scheduled-run-view-links.ts
|
|
40980
|
-
import { createHash as
|
|
40148
|
+
import { createHash as createHash22, randomBytes as randomBytes5, randomUUID as randomUUID22 } from "crypto";
|
|
40981
40149
|
var schemaReady2 = false;
|
|
40982
40150
|
async function ensureScheduledRunViewLinksSchema() {
|
|
40983
40151
|
if (schemaReady2) return;
|
|
@@ -41031,7 +40199,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
41031
40199
|
schemaReady2 = true;
|
|
41032
40200
|
}
|
|
41033
40201
|
function tokenHash2(token6) {
|
|
41034
|
-
return
|
|
40202
|
+
return createHash22("sha256").update(token6).digest("hex");
|
|
41035
40203
|
}
|
|
41036
40204
|
function mapRow(row) {
|
|
41037
40205
|
return {
|
|
@@ -41118,7 +40286,7 @@ async function revokeScheduledRunViewLink(ownerId2, runId, shareId, now = /* @__
|
|
|
41118
40286
|
}
|
|
41119
40287
|
|
|
41120
40288
|
// src/personal-authority/render.ts
|
|
41121
|
-
import { createHash as
|
|
40289
|
+
import { createHash as createHash23 } from "crypto";
|
|
41122
40290
|
function escapeHtml8(value) {
|
|
41123
40291
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
41124
40292
|
"&": "&",
|
|
@@ -41178,13 +40346,13 @@ ${section(flags.finalCta && Boolean(finalCta), `<section class="section cta" id=
|
|
|
41178
40346
|
html,
|
|
41179
40347
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
41180
40348
|
bytes,
|
|
41181
|
-
sha256:
|
|
40349
|
+
sha256: createHash23("sha256").update(html).digest("hex"),
|
|
41182
40350
|
generatedAt: generatedAtIso
|
|
41183
40351
|
};
|
|
41184
40352
|
}
|
|
41185
40353
|
|
|
41186
40354
|
// src/personal-authority/render-v2.ts
|
|
41187
|
-
import { createHash as
|
|
40355
|
+
import { createHash as createHash24 } from "crypto";
|
|
41188
40356
|
var FONT_STACKS = Object.freeze({
|
|
41189
40357
|
"editorial-serif": "Iowan Old Style, Baskerville, Times New Roman, serif",
|
|
41190
40358
|
"modern-sans": "Avenir Next, Avenir, Helvetica Neue, Arial, sans-serif",
|
|
@@ -41288,13 +40456,13 @@ ${section2(flags.finalCta && Boolean(finalCta), `<section class="section cta" id
|
|
|
41288
40456
|
html,
|
|
41289
40457
|
filename: `${input.name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "personal-authority"}.html`,
|
|
41290
40458
|
bytes,
|
|
41291
|
-
sha256:
|
|
40459
|
+
sha256: createHash24("sha256").update(html).digest("hex"),
|
|
41292
40460
|
generatedAt: generatedAtIso
|
|
41293
40461
|
};
|
|
41294
40462
|
}
|
|
41295
40463
|
|
|
41296
40464
|
// src/newsroom-publisher/render.ts
|
|
41297
|
-
import { createHash as
|
|
40465
|
+
import { createHash as createHash25 } from "crypto";
|
|
41298
40466
|
function escapeHtml10(value) {
|
|
41299
40467
|
return value.replace(/[&<>"']/g, (character) => ({
|
|
41300
40468
|
"&": "&",
|
|
@@ -41400,13 +40568,13 @@ body{overflow-x:hidden}.lead-story__body,.lead-support__item,.story-card__body{m
|
|
|
41400
40568
|
@media(max-width:620px){.desk-section__grid--panorama,.desk-section__grid--mosaic,.desk-section__grid--tiles{grid-template-columns:1fr}.story-card--panorama{display:block}.story-card--panorama h3,.story-card--mosaic-lead h3{font-size:34px}.story-card--mosaic-lead{grid-column:auto;grid-row:auto}.desk-section__grid--tiles>.story-card:nth-child(3n+2){margin-top:0}}
|
|
41401
40569
|
</style></head><body data-theme="${escapeHtml10(config.theme)}" id="top" data-scheduled-renderer="newsroom_publisher_v1">${ticker}${masthead}${navigation}<main>${leadGrid}${latest}${sections}${newsletter}${pressRoom}${trustCenter}</main>${trustFooter}</body></html>`;
|
|
41402
40570
|
const bytes = Buffer.byteLength(html);
|
|
41403
|
-
const sha2564 =
|
|
40571
|
+
const sha2564 = createHash25("sha256").update(html).digest("hex");
|
|
41404
40572
|
const filename2 = `${brand.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "newsroom"}-news-site.html`;
|
|
41405
40573
|
return { html, filename: filename2, bytes, sha256: sha2564, generatedAt: generatedAtIso };
|
|
41406
40574
|
}
|
|
41407
40575
|
|
|
41408
40576
|
// src/blog-article/render.ts
|
|
41409
|
-
import { createHash as
|
|
40577
|
+
import { createHash as createHash26 } from "crypto";
|
|
41410
40578
|
function escapeHtml11(value) {
|
|
41411
40579
|
return value.replace(
|
|
41412
40580
|
/[&<>"']/g,
|
|
@@ -41494,7 +40662,7 @@ function renderBlogArticleV1(input, config, generatedAt) {
|
|
|
41494
40662
|
:root{--ink:#10213d;--ink2:#1c355a;--blue:#2563eb;--blue2:#1749b6;--mint:#45d5aa;--paper:#f7f5f0;--line:#dce4ef;--card:#fff;--body:#475569;--serif:Iowan Old Style,Palatino Linotype,Palatino,Georgia,serif;--sans:Inter,Avenir Next,ui-sans-serif,system-ui,sans-serif}body[data-theme="slate"]{--ink:#1e293b;--ink2:#334155;--blue:#475569;--blue2:#1e293b;--mint:#94a3b8;--paper:#f8fafc}body[data-theme="forest"]{--ink:#12352e;--ink2:#23584c;--blue:#147d64;--blue2:#0c5b48;--mint:#79d8bd;--paper:#f4f8f3}*{box-sizing:border-box}html{scroll-behavior:smooth;background:var(--paper)}body{margin:0;color:#252a33;background:var(--paper);font-family:var(--sans);line-height:1.6}a{color:inherit}button{font:inherit}button:focus-visible,a:focus-visible{outline:3px solid var(--mint);outline-offset:3px}.progress{position:fixed;z-index:50;top:0;left:0;height:4px;width:0;background:var(--mint)}.hero{position:relative;overflow:hidden;color:#fff;background:var(--ink)}.hero:after{position:absolute;right:-110px;top:-150px;width:420px;height:420px;border:1px solid #ffffff1a;border-radius:50%;content:""}.hero-inner{position:relative;z-index:1;max-width:1240px;margin:auto;padding:58px 42px 126px}.breadcrumb{display:flex;gap:8px;margin-bottom:34px;color:#dbeafecc;font-size:14px}.eyebrow{display:block;color:var(--blue);font-size:12px;font-weight:800;letter-spacing:.16em;text-transform:uppercase}.hero .eyebrow{color:var(--mint)}h1,h2,h3,p{margin-top:0}.hero h1{max-width:1000px;margin:14px 0 24px;font:700 clamp(44px,6vw,78px)/.98 var(--serif);letter-spacing:-.035em}.hero .dek{max-width:780px;color:#dbeafe;font-size:20px}.contributors{margin-top:30px;color:#eff6ff;font-size:14px}.contributors a{font-weight:800}.meta{display:flex;flex-wrap:wrap;gap:12px;margin-top:6px;color:#dbeafecc}.layout{position:relative;z-index:2;display:grid;grid-template-columns:minmax(0,760px) 290px;gap:64px;max-width:1240px;margin:-78px auto 0;padding:0 42px 80px}.main{min-width:0}.takeaways{padding:38px 44px;border-radius:24px;background:#fff;box-shadow:0 24px 70px #10213d24}.takeaways h2{margin:5px 0 20px;font:700 32px/1.1 var(--serif);color:var(--ink)}.takeaways ul{display:grid;gap:16px;margin:0;padding:0;list-style:none}.takeaways li{position:relative;padding-left:26px;color:var(--body);font-size:17px}.takeaways li:before{position:absolute;left:2px;top:.7em;width:8px;height:8px;border-radius:50%;background:var(--blue);content:""}.intro{padding:40px 0 4px}.intro p,.article-section>p{margin-bottom:22px;color:var(--body);font:400 18px/1.75 var(--sans)}.article-section{scroll-margin-top:28px;padding-top:45px}.article-section h2,.faq h2{margin:6px 0 20px;color:var(--ink);font:700 38px/1.08 var(--serif);letter-spacing:-.02em}.article-section ul{display:grid;gap:10px;margin:22px 0;padding-left:24px;color:var(--body);font-size:17px}.article-section li::marker{color:var(--blue)}.callout{margin:34px 0;padding:24px 28px;border-left:4px solid var(--blue);border-radius:0 16px 16px 0;background:#fff}.callout span{color:var(--blue);font-size:11px;font-weight:800;letter-spacing:.13em;text-transform:uppercase}.callout h3{margin:5px 0 8px;color:var(--ink);font:700 24px/1.15 var(--serif)}.callout p{margin:0;color:var(--body)}.rail{padding-top:0}.sidebar-media{overflow:hidden;margin:0 0 18px;border:1px solid #dbeafe;border-radius:16px;background:#fff}.sidebar-media img,.sidebar-media svg{display:block;width:100%;aspect-ratio:16/9;object-fit:cover}.sidebar-media figcaption{padding:10px 13px;color:#64748b;font-size:11px}.disclosure{margin-bottom:28px;border-top:1px solid #dbeafe;border-bottom:1px solid #dbeafe}.disclosure button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:14px;padding:15px 0;border:0;color:#64748b;background:transparent;text-align:left;font-size:12px;cursor:pointer}.disclosure button b{color:var(--blue);font-size:18px;transition:transform .2s}.disclosure button[aria-expanded="true"] b{transform:rotate(180deg)}.disclosure p{padding:0 0 16px;margin:0;color:#64748b;font-size:12px}.toc{position:sticky;top:30px;padding:22px;border:1px solid var(--line);border-radius:16px;background:#fff;box-shadow:0 2px 5px #10213d12}.toc header{display:flex;align-items:center;justify-content:space-between}.toc header strong{color:var(--ink);font:700 21px/1 var(--serif)}.toc header button{width:34px;height:34px;border:0;border-radius:50%;color:var(--blue);background:transparent;cursor:pointer}.toc nav{display:grid;margin-top:14px}.toc nav a{display:flex;align-items:flex-start;gap:11px;padding:8px 0;color:#64748b;text-decoration:none;font-size:13px;line-height:1.35}.toc nav i{width:8px;height:8px;margin-top:5px;border:2px solid #bfdbfe;border-radius:50%}.toc nav a[aria-current="location"]{color:var(--ink);font-weight:800}.toc nav a[aria-current="location"] i{border-color:var(--blue);background:var(--blue)}.back-top{width:100%;margin-top:18px;padding:16px 0 0;border:0;border-top:1px solid var(--line);color:var(--blue);background:transparent;text-align:right;font-size:13px;font-weight:800;cursor:pointer}.faq{scroll-margin-top:28px;padding-top:64px}.faq>div{border-top:1px solid var(--line);border-bottom:1px solid var(--line)}.faq article+article{border-top:1px solid var(--line)}.faq h3{margin:0}.faq h3 button{display:flex;width:100%;align-items:center;justify-content:space-between;gap:18px;padding:19px 0;border:0;color:var(--ink);background:transparent;text-align:left;font-weight:800;cursor:pointer}.faq h3 b{color:var(--blue);font-size:22px}.faq article>p{padding:0 0 20px;margin:0;color:var(--body);font-size:16px}.author-card{margin-top:64px;padding:32px 36px;border-radius:18px;background:#fff}.article-actions{display:flex;flex-wrap:wrap;justify-content:space-between;gap:14px}.article-actions button{display:inline-flex;min-height:46px;align-items:center;justify-content:center;gap:8px;padding:8px 18px;border:2px solid var(--blue);border-radius:999px;color:var(--ink);background:#fff;font-weight:800;cursor:pointer}.article-actions button:hover{color:#fff;background:var(--blue)}.action-status{min-height:20px;margin:8px 0 0;color:var(--blue);text-align:right;font-size:13px;font-weight:700}.author-grid{display:grid;grid-template-columns:auto 1fr auto;gap:22px;align-items:center;margin-top:24px}.author-portrait{width:88px;height:88px;border-radius:50%;object-fit:cover}.author-initials{display:grid;place-items:center;color:var(--ink);background:#fecdd3;font-size:22px;font-weight:900}.author-grid span{color:#64748b;font-size:13px}.author-grid h2{margin:2px 0;color:var(--ink);font:700 27px/1 var(--serif)}.author-grid h2 a{text-decoration:none}.author-grid p{margin:8px 0;color:#64748b;font-size:14px}.author-grid nav{display:flex;flex-wrap:wrap;gap:14px}.author-grid nav a{color:#64748b;font-size:12px}.author-more{display:inline-flex;min-height:48px;align-items:center;justify-content:center;padding:10px 24px;border:2px solid var(--blue);color:var(--blue);text-align:center;text-decoration:none;font-size:13px;font-weight:800}.author-bio{margin:24px 0 0!important;padding-top:20px;border-top:1px solid var(--line);color:var(--body);font-size:16px}dialog{width:min(720px,calc(100% - 30px));max-height:calc(100dvh - 30px);padding:0;border:0;border-radius:18px;color:var(--ink);background:#fff;box-shadow:0 24px 70px #10213d38}dialog::backdrop{background:#0f172a7a;backdrop-filter:blur(2px)}dialog>div{position:relative;padding:30px}dialog h2{margin:0 50px 2px 0;font:700 30px/1.1 var(--serif)}dialog>div>p{color:#64748b}.dialog-close{position:absolute;top:13px;right:13px;width:44px;height:44px;border:0;border-radius:50%;color:var(--blue);background:transparent;font-size:29px;cursor:pointer}dialog section{border-top:1px solid var(--line)}dialog section button{display:block;width:100%;padding:17px 5px;border:0;border-bottom:1px solid var(--line);color:var(--body);background:#fff;text-align:left;line-height:1.65;cursor:pointer}.citation-status{min-height:22px;margin:14px 0 0!important;color:var(--blue)!important;font-size:13px;font-weight:800}.generated{max-width:1156px;margin:-52px auto 50px;padding:0 42px;color:#94a3b8;font-size:11px}@media(max-width:900px){.layout{display:block}.rail{display:none}.hero-inner{padding-bottom:116px}}@media(max-width:620px){.hero-inner{padding:36px 20px 104px}.hero h1{font-size:46px}.hero .dek{font-size:17px}.layout{margin-top:-70px;padding:0 16px 54px}.takeaways{padding:28px 24px}.article-section h2,.faq h2{font-size:32px}.author-card{padding:24px}.article-actions{display:grid;justify-content:start}.author-grid{grid-template-columns:1fr;gap:12px}.author-more{width:100%}dialog>div{padding:24px}.generated{padding:0 20px}}
|
|
41495
40663
|
</style></head><body data-theme="${escapeHtml11(config.theme)}" data-scheduled-renderer="blog_article_v1"><div class="progress" aria-hidden="true"></div><header class="hero"><div class="hero-inner"><nav class="breadcrumb" aria-label="Breadcrumb"><a href="${safeHref4(input.canonicalUrl)}">${escapeHtml11(input.category)}</a><span>/</span><span>Article</span></nav><span class="eyebrow">${escapeHtml11(input.eyebrow)}</span><h1>${escapeHtml11(input.title)}</h1>${input.dek ? `<p class="dek">${escapeHtml11(input.dek)}</p>` : ""}<div class="contributors">${contributorText}<div class="meta"><time datetime="${escapeHtml11(articleDate)}">Updated ${formatDate2(articleDate)}</time><span>\u2022</span><span>${input.readTimeMinutes} min read</span>${input.reviewer ? "<span>\u2022</span><strong>Expert reviewed</strong>" : ""}</div></div></div></header><main class="layout"><article class="main"><section class="takeaways"><span class="eyebrow">The short answer</span><h2>Key takeaways</h2><ul>${input.keyTakeaways.map((item) => `<li>${escapeHtml11(item)}</li>`).join("")}</ul></section><section class="intro">${input.introduction.map((paragraph) => `<p>${escapeHtml11(paragraph)}</p>`).join("")}</section>${sections}${faq}${author}</article><div class="rail">${media}${disclosure}${toc}</div></main>${config.showGeneratedAt ? `<p class="generated">Rendered ${formatDate2(generatedAtIso)} by MCP Scraper.</p>` : ""}${citationDialog}<script>(()=>{const citations=${jsonForScript(citationValues)};const canonical=${jsonForScript(input.canonicalUrl)};const title=${jsonForScript(input.title)};const progress=document.querySelector('.progress');const tocLinks=[...document.querySelectorAll('.toc nav a')];const targets=tocLinks.map(link=>document.querySelector(link.hash)).filter(Boolean);const update=()=>{const max=document.documentElement.scrollHeight-innerHeight;if(progress)progress.style.width=(max>0?scrollY/max*100:0)+'%';let active=targets[0]?.id;for(const target of targets){if(target.getBoundingClientRect().top<=innerHeight*.55)active=target.id}tocLinks.forEach(link=>link.toggleAttribute('aria-current',link.hash==='#'+active));tocLinks.forEach(link=>{if(link.hash==='#'+active)link.setAttribute('aria-current','location');else link.removeAttribute('aria-current')})};addEventListener('scroll',update,{passive:true});addEventListener('resize',update);update();document.querySelectorAll('.faq h3 button').forEach(button=>button.addEventListener('click',()=>{const panel=document.getElementById(button.getAttribute('aria-controls'));const open=button.getAttribute('aria-expanded')==='true';button.setAttribute('aria-expanded',String(!open));panel.hidden=open;button.querySelector('b').textContent=open?'+':'\u2212'}));const disclosure=document.querySelector('.disclosure button');disclosure?.addEventListener('click',()=>{const panel=document.getElementById(disclosure.getAttribute('aria-controls'));const open=disclosure.getAttribute('aria-expanded')==='true';disclosure.setAttribute('aria-expanded',String(!open));panel.hidden=open});const tocToggle=document.querySelector('.toc header button');tocToggle?.addEventListener('click',()=>{const nav=document.getElementById('toc-links');const open=tocToggle.getAttribute('aria-expanded')==='true';tocToggle.setAttribute('aria-expanded',String(!open));nav.hidden=open;tocToggle.textContent=open?'\u2304':'\u2303'});document.querySelector('.back-top')?.addEventListener('click',()=>scrollTo({top:0,behavior:'smooth'}));const copy=async text=>{try{await navigator.clipboard.writeText(text)}catch{const area=document.createElement('textarea');area.value=text;area.style.position='fixed';area.style.opacity='0';document.body.append(area);area.select();document.execCommand('copy');area.remove()}};const dialog=document.getElementById('citation-dialog');document.querySelector('[data-cite]')?.addEventListener('click',()=>dialog.showModal());document.querySelector('.dialog-close')?.addEventListener('click',()=>dialog.close());dialog?.addEventListener('click',event=>{if(event.target===dialog)dialog.close()});document.querySelectorAll('[data-citation]').forEach(button=>button.addEventListener('click',async()=>{const style=button.dataset.citation;await copy(citations[style]);document.querySelector('.citation-status').textContent=style+' citation copied to clipboard.'}));document.querySelector('[data-share]')?.addEventListener('click',async()=>{const status=document.querySelector('.action-status');if(navigator.share){try{await navigator.share({title,url:canonical});status.textContent='Sharing options opened.';return}catch(error){if(error.name==='AbortError')return}}await copy(canonical);status.textContent='Article link copied to clipboard.'})})()</script></body></html>`;
|
|
41496
40664
|
const bytes = Buffer.byteLength(html);
|
|
41497
|
-
const sha2564 =
|
|
40665
|
+
const sha2564 = createHash26("sha256").update(html).digest("hex");
|
|
41498
40666
|
return {
|
|
41499
40667
|
html,
|
|
41500
40668
|
filename: "blog-article.html",
|
|
@@ -41797,7 +40965,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
|
|
|
41797
40965
|
import { Hono as Hono36 } from "hono";
|
|
41798
40966
|
|
|
41799
40967
|
// src/scheduled-artifacts/scheduled-run-artifact-store.ts
|
|
41800
|
-
import { createHash as
|
|
40968
|
+
import { createHash as createHash27 } from "crypto";
|
|
41801
40969
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
41802
40970
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
41803
40971
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -41813,7 +40981,7 @@ function policy5() {
|
|
|
41813
40981
|
};
|
|
41814
40982
|
}
|
|
41815
40983
|
function runStorageSegment(runId) {
|
|
41816
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${
|
|
40984
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash27("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
41817
40985
|
}
|
|
41818
40986
|
async function createScheduledRunArtifact(args) {
|
|
41819
40987
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -41920,7 +41088,7 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
|
|
|
41920
41088
|
import { Hono as Hono37 } from "hono";
|
|
41921
41089
|
|
|
41922
41090
|
// src/api/scheduler-integration-auth.ts
|
|
41923
|
-
import { createHash as
|
|
41091
|
+
import { createHash as createHash28, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
41924
41092
|
|
|
41925
41093
|
// src/api/service-connections.ts
|
|
41926
41094
|
import { randomUUID as randomUUID23 } from "crypto";
|
|
@@ -42218,7 +41386,7 @@ function signingSecret() {
|
|
|
42218
41386
|
return secret;
|
|
42219
41387
|
}
|
|
42220
41388
|
function schedulerIntegrationSignature(args) {
|
|
42221
|
-
const bodyHash =
|
|
41389
|
+
const bodyHash = createHash28("sha256").update(args.body).digest("hex");
|
|
42222
41390
|
return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
42223
41391
|
${args.path}
|
|
42224
41392
|
${args.timestamp}
|
|
@@ -42410,7 +41578,7 @@ function shouldRunSiteExtractInBackground(input) {
|
|
|
42410
41578
|
}
|
|
42411
41579
|
|
|
42412
41580
|
// src/api/page-media-artifacts.ts
|
|
42413
|
-
import { createHash as
|
|
41581
|
+
import { createHash as createHash29, randomBytes as randomBytes6 } from "crypto";
|
|
42414
41582
|
import { homedir as homedir4 } from "os";
|
|
42415
41583
|
import { basename, extname, join as join9 } from "path";
|
|
42416
41584
|
import pLimit6 from "p-limit";
|
|
@@ -42557,7 +41725,7 @@ async function fetchMedia(rawUrl, expectedType) {
|
|
|
42557
41725
|
finalUrl: checked.parsed.href,
|
|
42558
41726
|
width: dimensions?.width ?? null,
|
|
42559
41727
|
height: dimensions?.height ?? null,
|
|
42560
|
-
sha256:
|
|
41728
|
+
sha256: createHash29("sha256").update(bytes).digest("hex")
|
|
42561
41729
|
};
|
|
42562
41730
|
}
|
|
42563
41731
|
throw new Error("media_redirect_rejected");
|
|
@@ -42763,13 +41931,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
42763
41931
|
}
|
|
42764
41932
|
|
|
42765
41933
|
// src/api/page-diff.ts
|
|
42766
|
-
import { createHash as
|
|
41934
|
+
import { createHash as createHash30 } from "crypto";
|
|
42767
41935
|
import { diffLines } from "diff";
|
|
42768
41936
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
42769
41937
|
var MAX_DIFF_HUNKS = 200;
|
|
42770
41938
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
42771
41939
|
function sha256Hex(value) {
|
|
42772
|
-
return
|
|
41940
|
+
return createHash30("sha256").update(value).digest("hex");
|
|
42773
41941
|
}
|
|
42774
41942
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
42775
41943
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -42883,10 +42051,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
42883
42051
|
}
|
|
42884
42052
|
|
|
42885
42053
|
// src/api/scrape-image-sink.ts
|
|
42886
|
-
import { createHash as
|
|
42054
|
+
import { createHash as createHash31 } from "crypto";
|
|
42887
42055
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
42888
42056
|
function idempotencyKey3(userId, vault, input) {
|
|
42889
|
-
return `scrape-image-${
|
|
42057
|
+
return `scrape-image-${createHash31("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
42890
42058
|
}
|
|
42891
42059
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
42892
42060
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -43358,7 +42526,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
43358
42526
|
}
|
|
43359
42527
|
|
|
43360
42528
|
// src/api/connection-memory-import.ts
|
|
43361
|
-
import { createHash as
|
|
42529
|
+
import { createHash as createHash32 } from "crypto";
|
|
43362
42530
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
43363
42531
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
43364
42532
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -43388,7 +42556,7 @@ function canonicalJson(value) {
|
|
|
43388
42556
|
return JSON.stringify(value);
|
|
43389
42557
|
}
|
|
43390
42558
|
function sha2562(value) {
|
|
43391
|
-
return
|
|
42559
|
+
return createHash32("sha256").update(value).digest("hex");
|
|
43392
42560
|
}
|
|
43393
42561
|
function sensitiveKey(key) {
|
|
43394
42562
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -43657,7 +42825,7 @@ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
|
43657
42825
|
}
|
|
43658
42826
|
|
|
43659
42827
|
// src/api/site-export-reader.ts
|
|
43660
|
-
import { createHash as
|
|
42828
|
+
import { createHash as createHash33 } from "crypto";
|
|
43661
42829
|
var SiteExportFormatUnavailableError = class extends Error {
|
|
43662
42830
|
constructor(format) {
|
|
43663
42831
|
super(`The ${format} representation was not retained for this export page.`);
|
|
@@ -43667,7 +42835,7 @@ var SiteExportFormatUnavailableError = class extends Error {
|
|
|
43667
42835
|
format;
|
|
43668
42836
|
};
|
|
43669
42837
|
function sha2563(value) {
|
|
43670
|
-
return
|
|
42838
|
+
return createHash33("sha256").update(value).digest("hex");
|
|
43671
42839
|
}
|
|
43672
42840
|
function publicPageRecord(page) {
|
|
43673
42841
|
const { bodyMarkdown: _body, contentRef: _contentRef, discoveryLinks: _discovery, ...metadata } = page;
|
|
@@ -44519,7 +43687,7 @@ async function dbUsage(identity, plan) {
|
|
|
44519
43687
|
}
|
|
44520
43688
|
|
|
44521
43689
|
// src/api/nango-control.ts
|
|
44522
|
-
import { createHash as
|
|
43690
|
+
import { createHash as createHash35, randomUUID as randomUUID27 } from "crypto";
|
|
44523
43691
|
|
|
44524
43692
|
// src/api/slack-archive-analysis.ts
|
|
44525
43693
|
function isRecord2(value) {
|
|
@@ -44893,7 +44061,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
44893
44061
|
}
|
|
44894
44062
|
|
|
44895
44063
|
// src/api/main-nango-transport.ts
|
|
44896
|
-
import { createHash as
|
|
44064
|
+
import { createHash as createHash34, randomUUID as randomUUID26 } from "crypto";
|
|
44897
44065
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
44898
44066
|
|
|
44899
44067
|
// src/api/connected-tool-policy.ts
|
|
@@ -45286,7 +44454,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
45286
44454
|
});
|
|
45287
44455
|
const readTools = [...policies.values()].filter((policy7) => policy7.classification === "read").map((policy7) => policy7.name);
|
|
45288
44456
|
const actionTools = [...policies.values()].filter((policy7) => policy7.classification === "action").map((policy7) => policy7.name);
|
|
45289
|
-
const revision =
|
|
44457
|
+
const revision = createHash34("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
45290
44458
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
45291
44459
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
45292
44460
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -45314,7 +44482,7 @@ async function callNangoToolDirect(args) {
|
|
|
45314
44482
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
45315
44483
|
});
|
|
45316
44484
|
const requestId = args.requestId?.trim() || randomUUID26();
|
|
45317
|
-
const idempotencyKey4 = `main-nango:${
|
|
44485
|
+
const idempotencyKey4 = `main-nango:${createHash34("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
45318
44486
|
const startedAt = /* @__PURE__ */ new Date();
|
|
45319
44487
|
const started = performance.now();
|
|
45320
44488
|
let result;
|
|
@@ -45339,7 +44507,7 @@ async function callNangoToolDirect(args) {
|
|
|
45339
44507
|
toolName: args.tool,
|
|
45340
44508
|
operationKind: args.operationKind ?? args.classification,
|
|
45341
44509
|
outcome: providerError ? "error" : "partial",
|
|
45342
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
44510
|
+
requestId: requestId.length <= 200 ? requestId : createHash34("sha256").update(requestId).digest("hex"),
|
|
45343
44511
|
startedAt: startedAt.toISOString(),
|
|
45344
44512
|
completedAt: completedAt.toISOString()
|
|
45345
44513
|
}
|
|
@@ -45381,7 +44549,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
45381
44549
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
45382
44550
|
protocolVersion: null,
|
|
45383
44551
|
schemaSource: "live_tools_list",
|
|
45384
|
-
schemaHash:
|
|
44552
|
+
schemaHash: createHash34("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
45385
44553
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
45386
44554
|
};
|
|
45387
44555
|
}
|
|
@@ -46105,8 +45273,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
46105
45273
|
return data.connection.actionsEnabled === true;
|
|
46106
45274
|
}
|
|
46107
45275
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey4) {
|
|
46108
|
-
const requestId = `main-connected-action:${
|
|
46109
|
-
const requestDigest =
|
|
45276
|
+
const requestId = `main-connected-action:${createHash35("sha256").update(identity).update("\0").update(idempotencyKey4?.trim() || randomUUID27()).digest("hex")}`;
|
|
45277
|
+
const requestDigest = createHash35("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
46110
45278
|
if (mainOwnsIntegrations()) {
|
|
46111
45279
|
const selectedTool = tool?.trim();
|
|
46112
45280
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -46257,7 +45425,7 @@ function canonicalJson2(value) {
|
|
|
46257
45425
|
return JSON.stringify(value);
|
|
46258
45426
|
}
|
|
46259
45427
|
function projectedToolSchemaHash(tool) {
|
|
46260
|
-
return
|
|
45428
|
+
return createHash35("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
46261
45429
|
}
|
|
46262
45430
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
46263
45431
|
if (mainOwnsIntegrations()) {
|
|
@@ -46536,7 +45704,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
46536
45704
|
}
|
|
46537
45705
|
|
|
46538
45706
|
// src/api/resend-control.ts
|
|
46539
|
-
import { createHash as
|
|
45707
|
+
import { createHash as createHash36 } from "crypto";
|
|
46540
45708
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
46541
45709
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
46542
45710
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -46795,7 +45963,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
46795
45963
|
return isRecord5(data) ? data.result ?? data : data;
|
|
46796
45964
|
}
|
|
46797
45965
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey4) {
|
|
46798
|
-
const requestId = `main-resend-action:${
|
|
45966
|
+
const requestId = `main-resend-action:${createHash36("sha256").update(identity).update("\0").update(idempotencyKey4.trim()).digest("hex")}`;
|
|
46799
45967
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
46800
45968
|
method: "POST",
|
|
46801
45969
|
headers: { "x-request-id": requestId },
|
|
@@ -48651,8 +47819,6 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
48651
47819
|
if (checked.error) return c.json({ error: checked.error }, 400);
|
|
48652
47820
|
body.callback_url = checked.parsed?.href;
|
|
48653
47821
|
}
|
|
48654
|
-
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
48655
|
-
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
48656
47822
|
const options = {
|
|
48657
47823
|
query: body.query.trim(),
|
|
48658
47824
|
location: body.location,
|
|
@@ -48674,6 +47840,26 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
48674
47840
|
includeAiOverview: body.includeAiOverview ?? false,
|
|
48675
47841
|
includeWhatPeopleSaying: body.includeWhatPeopleSaying ?? false
|
|
48676
47842
|
};
|
|
47843
|
+
const suppliedIdempotencyKey = c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
|
|
47844
|
+
let derivedJobId = null;
|
|
47845
|
+
let requestFingerprint3 = null;
|
|
47846
|
+
if (!options.serpOnly && suppliedIdempotencyKey != null) {
|
|
47847
|
+
const trimmedKey = suppliedIdempotencyKey.trim();
|
|
47848
|
+
if (!trimmedKey || trimmedKey.length > 200) {
|
|
47849
|
+
return c.json({ error: "Idempotency-Key must contain 1-200 characters.", errorCode: "invalid_idempotency_key", retryable: false }, 400);
|
|
47850
|
+
}
|
|
47851
|
+
derivedJobId = `paa-${sha256Hex(`${user.id}\0${trimmedKey}`).slice(0, 24)}`;
|
|
47852
|
+
requestFingerprint3 = sha256Hex(JSON.stringify({ options, callbackUrl: body.callback_url ?? null }));
|
|
47853
|
+
const existing = await getJob(derivedJobId, user.id);
|
|
47854
|
+
if (existing) {
|
|
47855
|
+
if (existing.options.requestFingerprint !== requestFingerprint3) {
|
|
47856
|
+
return c.json({ error: "Idempotency-Key was already used for a different asynchronous harvest_paa call.", errorCode: "idempotency_conflict", retryable: false }, 409);
|
|
47857
|
+
}
|
|
47858
|
+
return c.json({ job_id: existing.id, status: existing.status, replayed: true }, ["done", "failed", "cancelled"].includes(existing.status) ? 200 : 202);
|
|
47859
|
+
}
|
|
47860
|
+
}
|
|
47861
|
+
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
47862
|
+
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
48677
47863
|
const harvestCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount(options.maxQuestions);
|
|
48678
47864
|
let jobId2;
|
|
48679
47865
|
if (options.serpOnly) {
|
|
@@ -48686,7 +47872,7 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
48686
47872
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
48687
47873
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
48688
47874
|
} else {
|
|
48689
|
-
jobId2 = randomUUID28();
|
|
47875
|
+
jobId2 = derivedJobId ?? randomUUID28();
|
|
48690
47876
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
48691
47877
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
48692
47878
|
const hold = await debitMcIdempotent(
|
|
@@ -48700,11 +47886,16 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
48700
47886
|
try {
|
|
48701
47887
|
await createJobWithId(jobId2, user.id, options.query, {
|
|
48702
47888
|
...options,
|
|
48703
|
-
executionOwner: "
|
|
47889
|
+
executionOwner: "direct",
|
|
48704
47890
|
billingHoldMc: harvestCost,
|
|
48705
|
-
billingDebitKey
|
|
47891
|
+
billingDebitKey,
|
|
47892
|
+
...requestFingerprint3 ? { requestFingerprint: requestFingerprint3 } : {}
|
|
48706
47893
|
}, body.callback_url);
|
|
48707
47894
|
} catch (error) {
|
|
47895
|
+
const existing = derivedJobId ? await getJob(derivedJobId, user.id).catch(() => null) : null;
|
|
47896
|
+
if (existing && existing.options.requestFingerprint === requestFingerprint3) {
|
|
47897
|
+
return c.json({ job_id: existing.id, status: existing.status, replayed: true }, 202);
|
|
47898
|
+
}
|
|
48708
47899
|
await settleDebitMcIdempotent(
|
|
48709
47900
|
user.id,
|
|
48710
47901
|
billingDebitKey,
|
|
@@ -48715,11 +47906,9 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
48715
47906
|
).catch(() => void 0);
|
|
48716
47907
|
throw error;
|
|
48717
47908
|
}
|
|
48718
|
-
|
|
48719
|
-
console.error("[harvest] durable PAA dispatch pending reconciliation:", error instanceof Error ? error.message : String(error));
|
|
48720
|
-
});
|
|
47909
|
+
scheduleDirectPaaHarvest(jobId2);
|
|
48721
47910
|
}
|
|
48722
|
-
if (process.env.CRON_SECRET) {
|
|
47911
|
+
if (options.serpOnly && process.env.CRON_SECRET) {
|
|
48723
47912
|
const url = new URL(c.req.url);
|
|
48724
47913
|
void fetch(`${url.origin}/cron/tick`, {
|
|
48725
47914
|
headers: { Authorization: `Bearer ${process.env.CRON_SECRET}` }
|
|
@@ -50237,11 +49426,11 @@ app.get("/cron/tick", async (c) => {
|
|
|
50237
49426
|
return c.json({ error: "Unauthorized" }, 401);
|
|
50238
49427
|
}
|
|
50239
49428
|
const startedAt = Date.now();
|
|
50240
|
-
const { drainQueue } = await import("./worker-
|
|
49429
|
+
const { drainQueue } = await import("./worker-UTBUWEXQ.js");
|
|
50241
49430
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
50242
49431
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
50243
49432
|
const unfinished = [];
|
|
50244
|
-
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, leadListDispatch, leadListReconciliation, leadListArtifactCleanup, leadListInputCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation,
|
|
49433
|
+
const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, leadListDispatch, leadListReconciliation, leadListArtifactCleanup, leadListInputCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestReconciliation, commonsEmbeddingDrain, localSourcebookRedispatch, analyticsRollup, analyticsFormDelivery, workflowDispatchResult, scheduledRunReconciliation, retentionSweep] = await Promise.all([
|
|
50245
49434
|
settleWithinTickBudget("drainQueue", unfinished, drainQueue(budget), []),
|
|
50246
49435
|
settleWithinTickBudget("monthlyRefreshSweep", unfinished, runMonthlyRefreshSweep(), { status: "skipped_deadline" }),
|
|
50247
49436
|
settleWithinTickBudget("reapBrowserSessions", unfinished, reapIdleBrowserSessions(120), { reaped: 0 }),
|
|
@@ -50260,7 +49449,6 @@ app.get("/cron/tick", async (c) => {
|
|
|
50260
49449
|
settleWithinTickBudget("siteExtractArtifactCleanup", unfinished, cleanupExpiredSiteExtractArtifacts(), { deleted: 0, store: "none" }),
|
|
50261
49450
|
settleWithinTickBudget("siteExtractRedispatch", unfinished, redispatchFundedSiteExtractJobs(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
50262
49451
|
settleWithinTickBudget("siteExtractReconcile", unfinished, reconcileSiteExtractSettlements(10), { checked: 0, settled: 0, failed: 0 }),
|
|
50263
|
-
settleWithinTickBudget("paaRedispatch", unfinished, redispatchPendingPaaHarvests(10), { checked: 0, dispatched: 0, failed: 0 }),
|
|
50264
49452
|
settleWithinTickBudget("paaReconcile", unfinished, reconcilePaaHarvestSettlements(10), { checked: 0, staleFailed: 0, settled: 0, failed: 0 }),
|
|
50265
49453
|
settleWithinTickBudget("commonsEmbeddingDrain", unfinished, embedQueuedCommonsDocuments(50), { claimed: 0, embedded: 0, failed: 0, remaining: 0 }),
|
|
50266
49454
|
settleWithinTickBudget("localSourcebookRedispatch", unfinished, redispatchQueuedLocalSourcebookAcquisitions(5), { checked: 0, dispatched: 0, failed: 0 }),
|
|
@@ -50295,7 +49483,6 @@ app.get("/cron/tick", async (c) => {
|
|
|
50295
49483
|
siteExtractArtifactCleanup,
|
|
50296
49484
|
siteExtractRedispatch,
|
|
50297
49485
|
siteExtractReconciliation,
|
|
50298
|
-
paaHarvestRedispatch,
|
|
50299
49486
|
paaHarvestReconciliation,
|
|
50300
49487
|
commonsEmbeddingDrain,
|
|
50301
49488
|
localSourcebookRedispatch,
|
|
@@ -50317,8 +49504,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
50317
49504
|
finishExtractJob: finishExtractJob2,
|
|
50318
49505
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
50319
49506
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
50320
|
-
} = await import("./site-extract-repository-
|
|
50321
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
49507
|
+
} = await import("./site-extract-repository-NTM4MDQN.js");
|
|
49508
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-EA5URLXN.js");
|
|
50322
49509
|
const existing = await getExtractJob2(jobId2);
|
|
50323
49510
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
50324
49511
|
if (existing.status !== "failed") {
|
|
@@ -50345,7 +49532,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
50345
49532
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
50346
49533
|
let settlement = "already_settled_or_refunded";
|
|
50347
49534
|
if (job.billedMc == null && job.userId != null) {
|
|
50348
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
49535
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-NTM4MDQN.js");
|
|
50349
49536
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
50350
49537
|
const successful = await countSuccessfulPages2(jobId2);
|
|
50351
49538
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -50362,7 +49549,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
50362
49549
|
});
|
|
50363
49550
|
app.on(["GET", "POST", "PUT"], "/api/inngest", serveInngest({
|
|
50364
49551
|
client: inngest,
|
|
50365
|
-
functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, leadListEnrichmentFn,
|
|
49552
|
+
functions: [siteAuditFn, siteExtractFn, directoryWorkflowFn, leadListEnrichmentFn, localSourcebookFn]
|
|
50366
49553
|
}));
|
|
50367
49554
|
app.route("/admin/credits", adminCreditsApp);
|
|
50368
49555
|
app.route("/api/internal/site-architecture-auditor", siteAuditApp);
|
|
@@ -50532,4 +49719,3 @@ app.get("/blog/:slug/", (c) => {
|
|
|
50532
49719
|
export {
|
|
50533
49720
|
app
|
|
50534
49721
|
};
|
|
50535
|
-
//# sourceMappingURL=server-MTKWBSAR.js.map
|