mcp-scraper 0.65.1 → 0.66.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -1
- package/README.md +11 -6
- package/dist/{analytics-repository-IKE53S3E.js → analytics-repository-RUXBFSFX.js} +1 -2
- package/dist/bin/api-server.cjs +1844 -71
- package/dist/bin/api-server.js +4 -5
- package/dist/bin/mcp-scraper-cli.cjs +1 -2
- package/dist/bin/mcp-scraper-cli.js +6 -7
- package/dist/bin/mcp-scraper-install.cjs +7 -3
- package/dist/bin/mcp-scraper-install.js +2 -3
- package/dist/bin/mcp-stdio-server.cjs +5150 -3467
- package/dist/bin/mcp-stdio-server.js +11 -11
- package/dist/bin/paa-harvest.cjs +8 -3
- package/dist/bin/paa-harvest.js +4 -5
- package/dist/{chunk-5PIO7QBG.js → chunk-3PIXKLGZ.js} +3 -4
- package/dist/{chunk-VIWT35S6.js → chunk-4OT7BAIE.js} +1 -2
- package/dist/{chunk-ORKXE7VX.js → chunk-4P3PHFPQ.js} +11 -6
- package/dist/{chunk-AMLFKPLL.js → chunk-75VW7CWE.js} +4 -7
- package/dist/{chunk-FG6LJGJ7.js → chunk-77LQV73B.js} +0 -1
- package/dist/{chunk-DKIG2TEE.js → chunk-AQKJOSIB.js} +2 -3
- package/dist/{chunk-2XUXVKT4.js → chunk-AYJHI2FK.js} +0 -1
- package/dist/{chunk-5BEPOFKG.js → chunk-CNLLTD5B.js} +0 -1
- package/dist/{chunk-EKJ65E3Z.js → chunk-DNM65UCK.js} +0 -1
- package/dist/{chunk-PDAKCYHZ.js → chunk-E4LMY5OJ.js} +6258 -3747
- package/dist/{chunk-MQGBCWZD.js → chunk-GGZEC22A.js} +64 -1
- package/dist/{chunk-4VG3NUUY.js → chunk-GTGSD4YS.js} +2 -2
- package/dist/{chunk-MQQS7ONH.js → chunk-KBTUPRTF.js} +8 -296
- package/dist/{chunk-XPEN6OG5.js → chunk-NS754CRS.js} +15 -3
- package/dist/{chunk-UN4LZVCB.js → chunk-OZJMVCDK.js} +1 -2
- package/dist/{chunk-342OUCSG.js → chunk-PGJQDMC2.js} +1 -2
- package/dist/{chunk-G3P3ZDB4.js → chunk-RJ7JVYKU.js} +0 -1
- package/dist/{chunk-N54JS5WP.js → chunk-TNJIPV74.js} +1 -2
- package/dist/{chunk-4LINSJNI.js → chunk-TNLGHNLX.js} +1 -2
- package/dist/chunk-VRJ7CO6L.js +6 -0
- package/dist/chunk-VXLU74YZ.js +293 -0
- package/dist/{chunk-DCWXVAQT.js → chunk-WID3MFRS.js} +6 -2
- package/dist/{db-BRI465XQ.js → db-PO7N7ZP3.js} +1 -2
- package/dist/{extract-bundle-UWKJT4MU.js → extract-bundle-SIBFUA6I.js} +8 -10
- package/dist/index.cjs +8 -3
- package/dist/index.js +4 -5
- package/dist/{lead-list-enrichment-repository-ODWGRVCQ.js → lead-list-enrichment-repository-OTAUQONJ.js} +2 -3
- package/dist/{location-data-repository-67MMEVZQ.js → location-data-repository-RM2KJ3H5.js} +3 -4
- package/dist/{server-RSHSZBUZ.js → server-ELEJNRRE.js} +875 -1629
- package/dist/{site-extract-repository-J3RH3MOS.js → site-extract-repository-ASYEJO7Z.js} +4 -5
- package/dist/{worker-YKRIK5LS.js → worker-MAVYYWM7.js} +8 -9
- package/package.json +3 -1
- package/dist/analytics-repository-IKE53S3E.js.map +0 -1
- package/dist/bin/api-server.cjs.map +0 -1
- package/dist/bin/api-server.js.map +0 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +0 -1
- package/dist/bin/mcp-scraper-cli.js.map +0 -1
- package/dist/bin/mcp-scraper-install.cjs.map +0 -1
- package/dist/bin/mcp-scraper-install.js.map +0 -1
- package/dist/bin/mcp-stdio-server.cjs.map +0 -1
- package/dist/bin/mcp-stdio-server.js.map +0 -1
- package/dist/bin/paa-harvest.cjs.map +0 -1
- package/dist/bin/paa-harvest.js.map +0 -1
- package/dist/chunk-2XUXVKT4.js.map +0 -1
- package/dist/chunk-342OUCSG.js.map +0 -1
- package/dist/chunk-4LINSJNI.js.map +0 -1
- package/dist/chunk-4VG3NUUY.js.map +0 -1
- package/dist/chunk-5BEPOFKG.js.map +0 -1
- package/dist/chunk-5PIO7QBG.js.map +0 -1
- package/dist/chunk-AMLFKPLL.js.map +0 -1
- package/dist/chunk-CGKWNXJR.js +0 -7
- package/dist/chunk-CGKWNXJR.js.map +0 -1
- package/dist/chunk-DCWXVAQT.js.map +0 -1
- package/dist/chunk-DKIG2TEE.js.map +0 -1
- package/dist/chunk-EKJ65E3Z.js.map +0 -1
- package/dist/chunk-FG6LJGJ7.js.map +0 -1
- package/dist/chunk-G3P3ZDB4.js.map +0 -1
- package/dist/chunk-MQGBCWZD.js.map +0 -1
- package/dist/chunk-MQQS7ONH.js.map +0 -1
- package/dist/chunk-N54JS5WP.js.map +0 -1
- package/dist/chunk-ORKXE7VX.js.map +0 -1
- package/dist/chunk-PDAKCYHZ.js.map +0 -1
- package/dist/chunk-UN4LZVCB.js.map +0 -1
- package/dist/chunk-UVFB24RZ.js +0 -67
- package/dist/chunk-UVFB24RZ.js.map +0 -1
- package/dist/chunk-VIWT35S6.js.map +0 -1
- package/dist/chunk-XPEN6OG5.js.map +0 -1
- package/dist/db-BRI465XQ.js.map +0 -1
- package/dist/extract-bundle-UWKJT4MU.js.map +0 -1
- package/dist/index.cjs.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lead-list-enrichment-repository-ODWGRVCQ.js.map +0 -1
- package/dist/location-data-repository-67MMEVZQ.js.map +0 -1
- package/dist/server-RSHSZBUZ.js.map +0 -1
- package/dist/site-extract-repository-J3RH3MOS.js.map +0 -1
- package/dist/worker-YKRIK5LS.js.map +0 -1
|
@@ -10,13 +10,13 @@ import {
|
|
|
10
10
|
queuedCommonsDocumentCount,
|
|
11
11
|
semanticCommonsEntityScores,
|
|
12
12
|
typeFromMime
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-75VW7CWE.js";
|
|
14
14
|
import {
|
|
15
15
|
REQUIRED_LOCATION_STATE_CODES,
|
|
16
16
|
importHostedCensusPlacesCsv,
|
|
17
17
|
importHostedZipGroupsCsv,
|
|
18
18
|
listActiveHostedLocationDatasets
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-AQKJOSIB.js";
|
|
20
20
|
import {
|
|
21
21
|
cancelPendingLeadListEnrichmentRows,
|
|
22
22
|
claimLeadListEnrichmentOutbox,
|
|
@@ -43,7 +43,7 @@ import {
|
|
|
43
43
|
saveLeadListEnrichmentRowResult,
|
|
44
44
|
setInitializingLeadListEnrichmentConcurrencyLock,
|
|
45
45
|
stableCanonicalJson
|
|
46
|
-
} from "./chunk-
|
|
46
|
+
} from "./chunk-TNJIPV74.js";
|
|
47
47
|
import {
|
|
48
48
|
ANALYTICS_CONTENT_SORTS,
|
|
49
49
|
AnalyticsRepositoryError,
|
|
@@ -99,7 +99,7 @@ import {
|
|
|
99
99
|
updateAnalyticsPixel,
|
|
100
100
|
upsertAnalyticsAdSpend,
|
|
101
101
|
verifySession
|
|
102
|
-
} from "./chunk-
|
|
102
|
+
} from "./chunk-CNLLTD5B.js";
|
|
103
103
|
import {
|
|
104
104
|
classifyHarvestProblem,
|
|
105
105
|
createHarvestAttemptRecorder,
|
|
@@ -108,8 +108,8 @@ import {
|
|
|
108
108
|
harvestProblemResponse,
|
|
109
109
|
publicErrorBoundary,
|
|
110
110
|
serializeHarvestProblem
|
|
111
|
-
} from "./chunk-
|
|
112
|
-
import "./chunk-
|
|
111
|
+
} from "./chunk-NS754CRS.js";
|
|
112
|
+
import "./chunk-OZJMVCDK.js";
|
|
113
113
|
import {
|
|
114
114
|
AIModeSelectors,
|
|
115
115
|
AIOverviewSelectors,
|
|
@@ -137,7 +137,7 @@ import {
|
|
|
137
137
|
resolveKernelProxyId,
|
|
138
138
|
serpArrivalUrl,
|
|
139
139
|
withRuntimeUserAgent
|
|
140
|
-
} from "./chunk-
|
|
140
|
+
} from "./chunk-4P3PHFPQ.js";
|
|
141
141
|
import {
|
|
142
142
|
ApiErrorResponseSchema,
|
|
143
143
|
ArtifactTemplatePresetKeySchema,
|
|
@@ -159,6 +159,9 @@ import {
|
|
|
159
159
|
GetScheduledRunResponseSchema,
|
|
160
160
|
HttpMcpToolExecutor,
|
|
161
161
|
InternalScheduledArtifactRenderRequestSchema,
|
|
162
|
+
LeadColumnMapSchema,
|
|
163
|
+
LeadListEnrichmentOptionsSchema,
|
|
164
|
+
LeadListRowsSchema,
|
|
162
165
|
ListScheduledRunsQuerySchema,
|
|
163
166
|
LocalSourcebookSchemaTypeSchema,
|
|
164
167
|
NEWSROOM_PUBLISHER_TEMPLATE_PRESET,
|
|
@@ -187,12 +190,15 @@ import {
|
|
|
187
190
|
createConnectedDataArtifact,
|
|
188
191
|
createDirectoryCsvArtifact,
|
|
189
192
|
defaultLocalSourcebookSchemaType,
|
|
193
|
+
enrichLeadRow,
|
|
190
194
|
exportFanout,
|
|
191
195
|
handleMcpTasksHttpRequest,
|
|
192
196
|
harvestTimeoutBudget,
|
|
193
197
|
hashOwnerId,
|
|
194
198
|
isLoopbackBaseUrl,
|
|
199
|
+
leadListOutputRows,
|
|
195
200
|
localSourcebookSchemaContract,
|
|
201
|
+
normalizeLeadRows,
|
|
196
202
|
outputBaseDir,
|
|
197
203
|
registerBrowserAgentMcpTools,
|
|
198
204
|
registerMemoryMcpTools,
|
|
@@ -203,18 +209,18 @@ import {
|
|
|
203
209
|
resolveDeploymentProfile,
|
|
204
210
|
resolveLocalSourcebookSchemaType,
|
|
205
211
|
transcribeMediaUrl
|
|
206
|
-
} from "./chunk-
|
|
212
|
+
} from "./chunk-E4LMY5OJ.js";
|
|
207
213
|
import {
|
|
208
214
|
auditImageUrls,
|
|
209
215
|
auditImages,
|
|
210
216
|
buildLinkReport,
|
|
211
217
|
computeIssues
|
|
212
|
-
} from "./chunk-
|
|
218
|
+
} from "./chunk-PGJQDMC2.js";
|
|
213
219
|
import {
|
|
214
220
|
sanitizeAttempts,
|
|
215
221
|
sanitizeHarvestResult,
|
|
216
222
|
sanitizeOutboundDiagnostics
|
|
217
|
-
} from "./chunk-
|
|
223
|
+
} from "./chunk-4OT7BAIE.js";
|
|
218
224
|
import {
|
|
219
225
|
browserServiceApiKey,
|
|
220
226
|
browserServiceProfileName,
|
|
@@ -224,31 +230,22 @@ import {
|
|
|
224
230
|
recordKernelSession,
|
|
225
231
|
recordVendorUsage,
|
|
226
232
|
runWithCostContext
|
|
227
|
-
} from "./chunk-
|
|
233
|
+
} from "./chunk-TNLGHNLX.js";
|
|
228
234
|
import {
|
|
229
|
-
buildSerpEmailQuery,
|
|
230
|
-
classifyEmailScope,
|
|
231
|
-
contactAttemptUrls,
|
|
232
|
-
extractContactEvidence,
|
|
233
|
-
extractSerpEmailEvidence,
|
|
234
235
|
listWorkflowDefinitions,
|
|
235
236
|
runWorkflow,
|
|
236
237
|
runWorkflowStep,
|
|
237
|
-
selectPrimaryEmail,
|
|
238
238
|
slugify,
|
|
239
239
|
workflowDefinition,
|
|
240
240
|
workflowOutputBaseDir,
|
|
241
241
|
workflowStepCount,
|
|
242
242
|
workflowSupportsSteps
|
|
243
|
-
} from "./chunk-
|
|
244
|
-
import {
|
|
245
|
-
loadHtmlDocument
|
|
246
|
-
} from "./chunk-MQGBCWZD.js";
|
|
243
|
+
} from "./chunk-KBTUPRTF.js";
|
|
247
244
|
import {
|
|
248
245
|
csvRecords,
|
|
249
246
|
parseCsv,
|
|
250
247
|
rowsToCsv
|
|
251
|
-
} from "./chunk-
|
|
248
|
+
} from "./chunk-RJ7JVYKU.js";
|
|
252
249
|
import {
|
|
253
250
|
DEFAULT_MAPS_PROXY_MODE,
|
|
254
251
|
DEFAULT_PROXY_MODE,
|
|
@@ -258,14 +255,18 @@ import {
|
|
|
258
255
|
RawMapsHoursRowSchema,
|
|
259
256
|
RawMapsOverviewSchema,
|
|
260
257
|
RawMapsReviewStatsSchema
|
|
261
|
-
} from "./chunk-
|
|
258
|
+
} from "./chunk-77LQV73B.js";
|
|
259
|
+
import {
|
|
260
|
+
extractContactEvidence
|
|
261
|
+
} from "./chunk-VXLU74YZ.js";
|
|
262
262
|
import {
|
|
263
263
|
isPrivateIpAddress,
|
|
264
|
+
loadHtmlDocument,
|
|
264
265
|
validatePublicHttpUrl
|
|
265
|
-
} from "./chunk-
|
|
266
|
+
} from "./chunk-GGZEC22A.js";
|
|
266
267
|
import {
|
|
267
268
|
PACKAGE_VERSION
|
|
268
|
-
} from "./chunk-
|
|
269
|
+
} from "./chunk-VRJ7CO6L.js";
|
|
269
270
|
import {
|
|
270
271
|
SITE_EXTRACT_ARTIFACT_PREFIX,
|
|
271
272
|
abandonExtractSettlement,
|
|
@@ -296,14 +297,14 @@ import {
|
|
|
296
297
|
setExtractJobTotal,
|
|
297
298
|
settleExtractJob,
|
|
298
299
|
terminalExtractJobStatus
|
|
299
|
-
} from "./chunk-
|
|
300
|
+
} from "./chunk-3PIXKLGZ.js";
|
|
300
301
|
import {
|
|
301
302
|
createPrivateArtifact,
|
|
302
303
|
privateArtifactOwnerId,
|
|
303
304
|
readPrivateArtifactBuffer,
|
|
304
305
|
readPrivateArtifactWindow,
|
|
305
306
|
renewPrivateArtifactDownload
|
|
306
|
-
} from "./chunk-
|
|
307
|
+
} from "./chunk-DNM65UCK.js";
|
|
307
308
|
import {
|
|
308
309
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
309
310
|
CONCURRENCY_PACK_INVOICE_DESCRIPTION,
|
|
@@ -356,7 +357,7 @@ import {
|
|
|
356
357
|
tierChangeLeavesPrivateOffer,
|
|
357
358
|
tierCreditsPerInterval,
|
|
358
359
|
vendorCostUsdToMc
|
|
359
|
-
} from "./chunk-
|
|
360
|
+
} from "./chunk-AYJHI2FK.js";
|
|
360
361
|
import {
|
|
361
362
|
CaptchaError,
|
|
362
363
|
MapsSearchIncompletePageError,
|
|
@@ -462,7 +463,7 @@ import {
|
|
|
462
463
|
startHarvestAttempt,
|
|
463
464
|
upsertPageSnapshot,
|
|
464
465
|
verifyPassword
|
|
465
|
-
} from "./chunk-
|
|
466
|
+
} from "./chunk-GTGSD4YS.js";
|
|
466
467
|
|
|
467
468
|
// src/blog/registry.ts
|
|
468
469
|
var posts = [
|
|
@@ -9553,7 +9554,7 @@ var siteExtractFn = inngest.createFunction(
|
|
|
9553
9554
|
});
|
|
9554
9555
|
}) : null;
|
|
9555
9556
|
const finalized = await step.run("finalize", async () => {
|
|
9556
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
9557
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-SIBFUA6I.js");
|
|
9557
9558
|
const current = await getExtractJob(jobId2);
|
|
9558
9559
|
if (!current) throw new Error("extract job disappeared before finalization");
|
|
9559
9560
|
const stored = await assembleExtractArtifacts(current, { branding, imageAudit });
|
|
@@ -10513,7 +10514,7 @@ async function resolveUsZipsPath(requestedPath) {
|
|
|
10513
10514
|
async function loadZipGroups(stateAbbr, requestedPath, warnings, sourceMode) {
|
|
10514
10515
|
const mode = resolvedLocationDataMode({ zipSource: sourceMode });
|
|
10515
10516
|
if (mode === "hosted") {
|
|
10516
|
-
const { getHostedZipGroups } = await import("./location-data-repository-
|
|
10517
|
+
const { getHostedZipGroups } = await import("./location-data-repository-RM2KJ3H5.js");
|
|
10517
10518
|
const hosted = await getHostedZipGroups(stateAbbr);
|
|
10518
10519
|
if (!hosted.dataset) {
|
|
10519
10520
|
throw new DirectoryLocationError(
|
|
@@ -10576,7 +10577,7 @@ async function resolveDirectoryMarkets(options, config = {}) {
|
|
|
10576
10577
|
const sourceUrl = censusStateUrl(state.fips);
|
|
10577
10578
|
const warnings = [];
|
|
10578
10579
|
if (resolvedLocationDataMode(config) === "hosted") {
|
|
10579
|
-
const { queryHostedLocationMarkets } = await import("./location-data-repository-
|
|
10580
|
+
const { queryHostedLocationMarkets } = await import("./location-data-repository-RM2KJ3H5.js");
|
|
10580
10581
|
const hosted = await queryHostedLocationMarkets({
|
|
10581
10582
|
state: state.abbr,
|
|
10582
10583
|
minPopulation: options.minPopulation,
|
|
@@ -11698,6 +11699,19 @@ async function acquireConcurrencyGate(user, operation, options = {}) {
|
|
|
11698
11699
|
if (Number(res.rowsAffected ?? 0) > 0) {
|
|
11699
11700
|
return { ok: true, lockId, active, limit, operation };
|
|
11700
11701
|
}
|
|
11702
|
+
await logRequestEvent({
|
|
11703
|
+
userId: user.id,
|
|
11704
|
+
source: "concurrency_gate",
|
|
11705
|
+
status: "failed",
|
|
11706
|
+
query: operation,
|
|
11707
|
+
result: { active, limit },
|
|
11708
|
+
error: "concurrency_limit_exceeded"
|
|
11709
|
+
}).catch((error) => {
|
|
11710
|
+
console.error("[concurrency-gate] rejection telemetry failed", {
|
|
11711
|
+
operation,
|
|
11712
|
+
error: error instanceof Error ? error.message : String(error)
|
|
11713
|
+
});
|
|
11714
|
+
});
|
|
11701
11715
|
return { ok: false, active, limit, operation, retryAfterSeconds: DEFAULT_RETRY_AFTER_SECONDS };
|
|
11702
11716
|
}
|
|
11703
11717
|
async function releaseConcurrencyGate(lockId) {
|
|
@@ -11893,803 +11907,6 @@ var directoryWorkflowFn = inngest.createFunction(
|
|
|
11893
11907
|
}
|
|
11894
11908
|
);
|
|
11895
11909
|
|
|
11896
|
-
// src/leads/lead-list-enrichment.ts
|
|
11897
|
-
import { z as z11 } from "zod";
|
|
11898
|
-
|
|
11899
|
-
// src/leads/serp-people-evidence.ts
|
|
11900
|
-
var MAX_CONTEXT_CHARS = 400;
|
|
11901
|
-
var MAX_CANDIDATES = 5;
|
|
11902
|
-
var ROLE = [
|
|
11903
|
-
"president\\s*[/&-]\\s*owner",
|
|
11904
|
-
"owner\\s*[/&-]\\s*president",
|
|
11905
|
-
"chief executive officer",
|
|
11906
|
-
"operations manager",
|
|
11907
|
-
"general manager",
|
|
11908
|
-
"managing member",
|
|
11909
|
-
"managing partner",
|
|
11910
|
-
"business owner",
|
|
11911
|
-
"co[- ]founder",
|
|
11912
|
-
"team member",
|
|
11913
|
-
"president",
|
|
11914
|
-
"principal",
|
|
11915
|
-
"director",
|
|
11916
|
-
"manager",
|
|
11917
|
-
"partner",
|
|
11918
|
-
"founder",
|
|
11919
|
-
"owner",
|
|
11920
|
-
"ceo"
|
|
11921
|
-
].join("|");
|
|
11922
|
-
var NAME = String.raw`(?:Mr\.?|Ms\.?|Mrs\.?|Dr\.?)?\s*([\p{Lu}][\p{L}'’.-]+(?:\s+[\p{Lu}][\p{L}'’.-]+){1,3})`;
|
|
11923
|
-
var NAME_THEN_ROLE_SOURCE = `${NAME}\\s*(?:,|[-\u2013\u2014]|\\bis\\b|\\bas\\b|\\bserves as\\b)?\\s*(${ROLE})\\b`;
|
|
11924
|
-
var ROLE_THEN_NAME_SOURCE = `\\b(${ROLE})\\b\\s*(?:[,:\u2013\u2014-]|\\bis\\b|\\bof\\b)\\s*${NAME}`;
|
|
11925
|
-
var NAME_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
11926
|
-
"about",
|
|
11927
|
-
"additional",
|
|
11928
|
-
"business",
|
|
11929
|
-
"company",
|
|
11930
|
-
"contact",
|
|
11931
|
-
"customer",
|
|
11932
|
-
"details",
|
|
11933
|
-
"information",
|
|
11934
|
-
"management",
|
|
11935
|
-
"member",
|
|
11936
|
-
"owner",
|
|
11937
|
-
"president",
|
|
11938
|
-
"profile",
|
|
11939
|
-
"roofing",
|
|
11940
|
-
"team",
|
|
11941
|
-
"website"
|
|
11942
|
-
]);
|
|
11943
|
-
var LEGAL_SUFFIXES = /* @__PURE__ */ new Set(["co", "company", "corp", "corporation", "inc", "incorporated", "llc", "ltd", "limited", "pllc"]);
|
|
11944
|
-
function compact(value, max = MAX_CONTEXT_CHARS) {
|
|
11945
|
-
return value.replace(/\s+/g, " ").trim().slice(0, max);
|
|
11946
|
-
}
|
|
11947
|
-
function words(value) {
|
|
11948
|
-
return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().split(/\s+/).filter(Boolean);
|
|
11949
|
-
}
|
|
11950
|
-
function businessWords(value) {
|
|
11951
|
-
const tokens2 = words(value);
|
|
11952
|
-
while (tokens2.length && LEGAL_SUFFIXES.has(tokens2[tokens2.length - 1])) tokens2.pop();
|
|
11953
|
-
return tokens2;
|
|
11954
|
-
}
|
|
11955
|
-
function cleanName(value) {
|
|
11956
|
-
return value.replace(/^(?:Mr\.?|Ms\.?|Mrs\.?|Dr\.?)\s+/i, "").replace(/\s+/g, " ").trim();
|
|
11957
|
-
}
|
|
11958
|
-
function plausibleName(value, businessName) {
|
|
11959
|
-
const tokens2 = words(value);
|
|
11960
|
-
if (tokens2.length < 2 || tokens2.length > 4 || tokens2.some((token6) => NAME_STOP_WORDS.has(token6))) return false;
|
|
11961
|
-
const business = new Set(businessWords(businessName));
|
|
11962
|
-
return tokens2.some((token6) => !business.has(token6));
|
|
11963
|
-
}
|
|
11964
|
-
function relationshipFor(role) {
|
|
11965
|
-
const normalized = role.toLowerCase().replace(/\s+/g, " ");
|
|
11966
|
-
if (/founder/.test(normalized)) return "founder";
|
|
11967
|
-
if (/owner|managing member|partner/.test(normalized)) return "owner";
|
|
11968
|
-
if (/manager/.test(normalized)) return "manager";
|
|
11969
|
-
if (/president|chief executive officer|\bceo\b|principal|director/.test(normalized)) return "executive";
|
|
11970
|
-
return "team_member";
|
|
11971
|
-
}
|
|
11972
|
-
function normalizedRole(role) {
|
|
11973
|
-
return compact(role, 80).replace(/\bceo\b/i, "CEO").replace(/\b\w/g, (letter) => letter.toUpperCase());
|
|
11974
|
-
}
|
|
11975
|
-
function resultContext(result) {
|
|
11976
|
-
return compact(`${result.title} | ${result.snippet ?? ""}`);
|
|
11977
|
-
}
|
|
11978
|
-
function orderedResults(results) {
|
|
11979
|
-
return results.map((result, index) => ({ result, index })).sort((left, right) => {
|
|
11980
|
-
const a = Number.isFinite(left.result.position) ? left.result.position : Number.MAX_SAFE_INTEGER;
|
|
11981
|
-
const b = Number.isFinite(right.result.position) ? right.result.position : Number.MAX_SAFE_INTEGER;
|
|
11982
|
-
return a - b || left.index - right.index;
|
|
11983
|
-
}).map((item) => item.result);
|
|
11984
|
-
}
|
|
11985
|
-
function quoted(value) {
|
|
11986
|
-
return `"${value.replace(/["\\]/g, " ").replace(/\s+/g, " ").trim()}"`;
|
|
11987
|
-
}
|
|
11988
|
-
function buildAssociatedPeopleQuery(input) {
|
|
11989
|
-
const roles = input.mode === "owners" ? 'owner founder "managing member" president' : "owner founder president manager leadership team";
|
|
11990
|
-
return [quoted(input.businessName), roles, compact(input.location ?? "", 120)].filter(Boolean).join(" ");
|
|
11991
|
-
}
|
|
11992
|
-
function buildLinkedInPersonQuery(input) {
|
|
11993
|
-
return `site:linkedin.com/in ${quoted(input.personName)} ${quoted(input.businessName)}`;
|
|
11994
|
-
}
|
|
11995
|
-
function extractAssociatedPeopleEvidence(input) {
|
|
11996
|
-
const accepted = /* @__PURE__ */ new Map();
|
|
11997
|
-
for (const result of orderedResults(input.results)) {
|
|
11998
|
-
const context = resultContext(result);
|
|
11999
|
-
if (!containsBusiness(context, input.businessName)) continue;
|
|
12000
|
-
const matches = [];
|
|
12001
|
-
for (const match of context.matchAll(new RegExp(NAME_THEN_ROLE_SOURCE, "giu"))) matches.push({ name: match[1] ?? "", role: match[2] ?? "" });
|
|
12002
|
-
for (const match of context.matchAll(new RegExp(ROLE_THEN_NAME_SOURCE, "giu"))) matches.push({ name: match[2] ?? "", role: match[1] ?? "" });
|
|
12003
|
-
for (const match of matches) {
|
|
12004
|
-
const name = cleanName(match.name);
|
|
12005
|
-
const role = normalizedRole(match.role);
|
|
12006
|
-
if (!plausibleName(name, input.businessName)) continue;
|
|
12007
|
-
const relationship = relationshipFor(role);
|
|
12008
|
-
if (input.mode === "owners" && relationship !== "owner" && relationship !== "founder") continue;
|
|
12009
|
-
const key = words(name).join(" ");
|
|
12010
|
-
if (!key || accepted.has(key)) continue;
|
|
12011
|
-
accepted.set(key, {
|
|
12012
|
-
name,
|
|
12013
|
-
role,
|
|
12014
|
-
relationship,
|
|
12015
|
-
linkedinUrl: null,
|
|
12016
|
-
confidence: 0.88,
|
|
12017
|
-
reasons: ["name_role_in_search_result", "exact_business_query_context"],
|
|
12018
|
-
sources: [{
|
|
12019
|
-
sourceType: "serp_snippet",
|
|
12020
|
-
purpose: "role",
|
|
12021
|
-
sourceUrl: result.url,
|
|
12022
|
-
sourceTitle: result.title,
|
|
12023
|
-
sourcePosition: result.position,
|
|
12024
|
-
context
|
|
12025
|
-
}]
|
|
12026
|
-
});
|
|
12027
|
-
if (accepted.size >= MAX_CANDIDATES) return [...accepted.values()];
|
|
12028
|
-
}
|
|
12029
|
-
}
|
|
12030
|
-
return [...accepted.values()];
|
|
12031
|
-
}
|
|
12032
|
-
function publicLinkedInProfileUrl(value) {
|
|
12033
|
-
try {
|
|
12034
|
-
const parsed = new URL(value);
|
|
12035
|
-
const host2 = parsed.hostname.toLowerCase().replace(/^www\./, "");
|
|
12036
|
-
if (parsed.protocol !== "https:" || host2 !== "linkedin.com" || !/^\/in\/[^/]+\/?$/i.test(parsed.pathname)) return null;
|
|
12037
|
-
parsed.search = "";
|
|
12038
|
-
parsed.hash = "";
|
|
12039
|
-
return parsed.href;
|
|
12040
|
-
} catch {
|
|
12041
|
-
return null;
|
|
12042
|
-
}
|
|
12043
|
-
}
|
|
12044
|
-
function containsPerson(text2, personName) {
|
|
12045
|
-
const haystack = new Set(words(text2));
|
|
12046
|
-
const name = words(personName);
|
|
12047
|
-
return name.length >= 2 && name.every((token6) => haystack.has(token6));
|
|
12048
|
-
}
|
|
12049
|
-
function containsBusiness(text2, businessName) {
|
|
12050
|
-
const haystack = new Set(words(text2));
|
|
12051
|
-
const business = businessWords(businessName);
|
|
12052
|
-
if (business.length < 2) return false;
|
|
12053
|
-
const hits = business.filter((token6) => haystack.has(token6)).length;
|
|
12054
|
-
return hits >= 2 && hits / business.length >= 0.6;
|
|
12055
|
-
}
|
|
12056
|
-
function attachLinkedInProfile(input) {
|
|
12057
|
-
for (const result of orderedResults(input.results)) {
|
|
12058
|
-
const linkedinUrl = publicLinkedInProfileUrl(result.url);
|
|
12059
|
-
if (!linkedinUrl) continue;
|
|
12060
|
-
const context = resultContext(result);
|
|
12061
|
-
if (!containsPerson(context, input.person.name) || !containsBusiness(context, input.businessName)) continue;
|
|
12062
|
-
const locationTokens = words((input.location ?? "").split(",")[0] ?? "").filter((token6) => token6.length > 2);
|
|
12063
|
-
const locationMatched = locationTokens.some((token6) => words(context).includes(token6));
|
|
12064
|
-
return {
|
|
12065
|
-
...input.person,
|
|
12066
|
-
linkedinUrl,
|
|
12067
|
-
confidence: locationMatched ? 0.97 : 0.94,
|
|
12068
|
-
reasons: [.../* @__PURE__ */ new Set([...input.person.reasons, "linkedin_name_match", "linkedin_company_match", ...locationMatched ? ["linkedin_location_match"] : []])],
|
|
12069
|
-
sources: [...input.person.sources, {
|
|
12070
|
-
sourceType: "serp_snippet",
|
|
12071
|
-
purpose: "linkedin",
|
|
12072
|
-
sourceUrl: result.url,
|
|
12073
|
-
sourceTitle: result.title,
|
|
12074
|
-
sourcePosition: result.position,
|
|
12075
|
-
context
|
|
12076
|
-
}]
|
|
12077
|
-
};
|
|
12078
|
-
}
|
|
12079
|
-
return input.person;
|
|
12080
|
-
}
|
|
12081
|
-
|
|
12082
|
-
// src/leads/lead-list-enrichment.ts
|
|
12083
|
-
var LeadScalarSchema = z11.union([z11.string(), z11.number().finite(), z11.boolean(), z11.null()]);
|
|
12084
|
-
var RESERVED_OUTPUT_PREFIX = "enriched_";
|
|
12085
|
-
var MAX_ROWS = 250;
|
|
12086
|
-
var MAX_COLUMNS = 100;
|
|
12087
|
-
var MAX_CELL_BYTES = 32 * 1024;
|
|
12088
|
-
var MAX_INLINE_BYTES = 500 * 1024;
|
|
12089
|
-
var MAX_NORMALIZED_ARTIFACT_BYTES = 5 * 1024 * 1024;
|
|
12090
|
-
var LeadListRowsSchema = z11.array(z11.record(z11.string(), LeadScalarSchema)).min(1).max(MAX_ROWS).superRefine((rows, ctx) => {
|
|
12091
|
-
rows.forEach((row, rowIndex) => {
|
|
12092
|
-
Object.keys(row).forEach((header) => {
|
|
12093
|
-
if (header.toLowerCase().startsWith(RESERVED_OUTPUT_PREFIX)) {
|
|
12094
|
-
ctx.addIssue({ code: "custom", message: `Input header ${header} uses the reserved enriched_ prefix`, path: [rowIndex, header] });
|
|
12095
|
-
}
|
|
12096
|
-
});
|
|
12097
|
-
});
|
|
12098
|
-
});
|
|
12099
|
-
var LeadColumnMapSchema = z11.object({
|
|
12100
|
-
name: z11.string().min(1),
|
|
12101
|
-
location: z11.string().min(1).optional(),
|
|
12102
|
-
city: z11.string().min(1).optional(),
|
|
12103
|
-
region: z11.string().min(1).optional(),
|
|
12104
|
-
organization: z11.string().min(1).optional(),
|
|
12105
|
-
websiteUrl: z11.string().min(1).optional(),
|
|
12106
|
-
profileUrl: z11.string().min(1).optional(),
|
|
12107
|
-
phone: z11.string().min(1).optional(),
|
|
12108
|
-
email: z11.string().min(1).optional(),
|
|
12109
|
-
entityType: z11.string().min(1).optional()
|
|
12110
|
-
}).strict();
|
|
12111
|
-
var LeadListEnrichmentOptionsSchema = z11.object({
|
|
12112
|
-
resolveMissingUrls: z11.boolean().default(true),
|
|
12113
|
-
defaultEntityType: z11.enum(["business", "person"]).default("business"),
|
|
12114
|
-
gl: z11.string().regex(/^[a-z]{2}$/i).transform((value) => value.toLowerCase()).default("us"),
|
|
12115
|
-
hl: z11.string().regex(/^[a-z]{2}(?:-[a-z]{2})?$/i).transform((value) => value.toLowerCase()).default("en"),
|
|
12116
|
-
maxPagesPerLead: z11.number().int().min(1).max(4).default(3),
|
|
12117
|
-
concurrency: z11.number().int().min(1).max(5).default(3),
|
|
12118
|
-
emailSearchFallback: z11.enum(["off", "serp_snippets"]).default("off"),
|
|
12119
|
-
peopleDiscovery: z11.enum(["off", "owners", "leadership"]).default("off"),
|
|
12120
|
-
maxPeoplePerLead: z11.number().int().min(1).max(3).default(2),
|
|
12121
|
-
outputFormats: z11.array(z11.enum(["csv", "xlsx"])).min(1).max(2).transform((values) => [...new Set(values)]).default(["csv", "xlsx"])
|
|
12122
|
-
}).strict();
|
|
12123
|
-
var DEFAULT_HEADER_ALIASES = {
|
|
12124
|
-
name: ["name", "business_name", "business name", "company", "company_name", "full_name", "full name"],
|
|
12125
|
-
location: ["location", "market", "address"],
|
|
12126
|
-
city: ["city", "town"],
|
|
12127
|
-
region: ["region", "state", "province"],
|
|
12128
|
-
organization: ["organization", "organisation", "brokerage", "agency", "company"],
|
|
12129
|
-
websiteUrl: ["websiteUrl", "website_url", "website url", "website", "url"],
|
|
12130
|
-
profileUrl: ["profileUrl", "profile_url", "profile url", "profile"],
|
|
12131
|
-
phone: ["phone", "phone_number", "phone number", "telephone"],
|
|
12132
|
-
email: ["email", "email_address", "email address"],
|
|
12133
|
-
entityType: ["entityType", "entity_type", "entity type", "type"]
|
|
12134
|
-
};
|
|
12135
|
-
function utf8Bytes(value) {
|
|
12136
|
-
return Buffer.byteLength(String(value ?? ""), "utf8");
|
|
12137
|
-
}
|
|
12138
|
-
function scalarString(value) {
|
|
12139
|
-
if (value == null) return null;
|
|
12140
|
-
const text2 = String(value).trim();
|
|
12141
|
-
return text2 || null;
|
|
12142
|
-
}
|
|
12143
|
-
function privateIpv4(hostname) {
|
|
12144
|
-
const parts = hostname.split(".").map(Number);
|
|
12145
|
-
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false;
|
|
12146
|
-
return parts[0] === 10 || parts[0] === 127 || parts[0] === 169 && parts[1] === 254 || parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31 || parts[0] === 192 && parts[1] === 168 || parts[0] === 0;
|
|
12147
|
-
}
|
|
12148
|
-
function publicHttpUrl(value) {
|
|
12149
|
-
if (!value) return null;
|
|
12150
|
-
try {
|
|
12151
|
-
const parsed = new URL(value);
|
|
12152
|
-
const host2 = parsed.hostname.toLowerCase();
|
|
12153
|
-
if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
|
|
12154
|
-
if (host2 === "localhost" || host2.endsWith(".localhost") || host2 === "::1" || host2.endsWith(".local") || privateIpv4(host2)) return null;
|
|
12155
|
-
return parsed.href;
|
|
12156
|
-
} catch {
|
|
12157
|
-
return null;
|
|
12158
|
-
}
|
|
12159
|
-
}
|
|
12160
|
-
function inferColumnMap(headers, explicit) {
|
|
12161
|
-
const lower = new Map(headers.map((header) => [header.toLowerCase(), header]));
|
|
12162
|
-
const result = { ...explicit };
|
|
12163
|
-
for (const [field, aliases] of Object.entries(DEFAULT_HEADER_ALIASES)) {
|
|
12164
|
-
if (result[field]) continue;
|
|
12165
|
-
const match = aliases.map((alias) => lower.get(alias.toLowerCase())).find(Boolean);
|
|
12166
|
-
if (match) result[field] = match;
|
|
12167
|
-
}
|
|
12168
|
-
if (!result.name) throw new Error(`invalid_column_map: name must identify one exact source header; available headers: ${headers.join(", ")}`);
|
|
12169
|
-
return LeadColumnMapSchema.parse(result);
|
|
12170
|
-
}
|
|
12171
|
-
function assertMappedHeaders(headers, map) {
|
|
12172
|
-
for (const [field, header] of Object.entries(map)) {
|
|
12173
|
-
if (field === "entityType" && (header === "person" || header === "business") && !headers.includes(header)) continue;
|
|
12174
|
-
if (!headers.includes(header)) throw new Error(`invalid_column_map: ${field} references missing header ${header}; available headers: ${headers.join(", ")}`);
|
|
12175
|
-
}
|
|
12176
|
-
}
|
|
12177
|
-
function mappedValue(row, header) {
|
|
12178
|
-
return header ? scalarString(row[header]) : null;
|
|
12179
|
-
}
|
|
12180
|
-
function combineLocation(location2, city, region) {
|
|
12181
|
-
if (location2) return location2;
|
|
12182
|
-
return [city, region].filter(Boolean).join(", ") || null;
|
|
12183
|
-
}
|
|
12184
|
-
function normalizeLeadRows(input) {
|
|
12185
|
-
const rows = LeadListRowsSchema.parse(input.rows);
|
|
12186
|
-
const options = LeadListEnrichmentOptionsSchema.parse(input.options ?? {});
|
|
12187
|
-
const headers = Object.keys(rows[0]);
|
|
12188
|
-
if (!headers.length || headers.length > MAX_COLUMNS) throw new Error(`lead_list_bounds_exceeded: input must contain 1-${MAX_COLUMNS} columns`);
|
|
12189
|
-
if (headers.some((header) => !header.trim())) throw new Error("Input headers must not be blank");
|
|
12190
|
-
if (new Set(headers).size !== headers.length) throw new Error("Input headers must be unique");
|
|
12191
|
-
for (const [rowIndex, row] of rows.entries()) {
|
|
12192
|
-
const rowHeaders = Object.keys(row);
|
|
12193
|
-
if (rowHeaders.length !== headers.length || rowHeaders.some((header, index) => header !== headers[index])) {
|
|
12194
|
-
throw new Error(`All rows must have the same columns in the same order; row ${rowIndex + 1} differs`);
|
|
12195
|
-
}
|
|
12196
|
-
for (const [header, value] of Object.entries(row)) {
|
|
12197
|
-
if (utf8Bytes(value) > MAX_CELL_BYTES) throw new Error(`lead_list_bounds_exceeded: cell in column ${header} exceeds 32 KiB`);
|
|
12198
|
-
}
|
|
12199
|
-
}
|
|
12200
|
-
const serializedBytes = Buffer.byteLength(JSON.stringify(rows), "utf8");
|
|
12201
|
-
const maxSerializedBytes = input.maxSerializedBytes ?? MAX_INLINE_BYTES;
|
|
12202
|
-
if (!Number.isInteger(maxSerializedBytes) || maxSerializedBytes < 1 || maxSerializedBytes > MAX_NORMALIZED_ARTIFACT_BYTES) {
|
|
12203
|
-
throw new Error("maxSerializedBytes must be an integer no greater than 5 MiB");
|
|
12204
|
-
}
|
|
12205
|
-
if (serializedBytes > maxSerializedBytes) {
|
|
12206
|
-
const label = maxSerializedBytes === MAX_INLINE_BYTES ? "inline rows exceed 500 KiB" : "normalized rows exceed the configured byte limit";
|
|
12207
|
-
throw new Error(`lead_list_bounds_exceeded: ${label}`);
|
|
12208
|
-
}
|
|
12209
|
-
const columnMap = inferColumnMap(headers, input.columnMap);
|
|
12210
|
-
assertMappedHeaders(headers, columnMap);
|
|
12211
|
-
const normalizedRows = rows.map((source, ordinal) => {
|
|
12212
|
-
const name = mappedValue(source, columnMap.name);
|
|
12213
|
-
if (!name) throw new Error(`Row ${ordinal + 1} requires a non-empty name`);
|
|
12214
|
-
const websiteRaw = mappedValue(source, columnMap.websiteUrl);
|
|
12215
|
-
const profileRaw = mappedValue(source, columnMap.profileUrl);
|
|
12216
|
-
const websiteUrl = websiteRaw ? publicHttpUrl(websiteRaw) : null;
|
|
12217
|
-
const profileUrl = profileRaw ? publicHttpUrl(profileRaw) : null;
|
|
12218
|
-
if (websiteRaw && !websiteUrl || profileRaw && !profileUrl) throw new Error(`Row ${ordinal + 1} URL must be a public HTTP(S) URL`);
|
|
12219
|
-
const location2 = mappedValue(source, columnMap.location);
|
|
12220
|
-
const city = mappedValue(source, columnMap.city);
|
|
12221
|
-
const region = mappedValue(source, columnMap.region);
|
|
12222
|
-
if (!websiteUrl && !profileUrl && (!options.resolveMissingUrls || !combineLocation(location2, city, region))) {
|
|
12223
|
-
throw new Error(`Row ${ordinal + 1} requires a public website/profile URL or a location with resolveMissingUrls enabled`);
|
|
12224
|
-
}
|
|
12225
|
-
const mappedEntityType = columnMap.entityType === "person" || columnMap.entityType === "business" ? columnMap.entityType : mappedValue(source, columnMap.entityType);
|
|
12226
|
-
if (mappedEntityType && mappedEntityType !== "person" && mappedEntityType !== "business") {
|
|
12227
|
-
throw new Error(`Row ${ordinal + 1} entityType must be business or person`);
|
|
12228
|
-
}
|
|
12229
|
-
const entityType = mappedEntityType === "person" || mappedEntityType === "business" ? mappedEntityType : options.defaultEntityType;
|
|
12230
|
-
return {
|
|
12231
|
-
ordinal,
|
|
12232
|
-
source: Object.fromEntries(headers.map((header) => [header, source[header]])),
|
|
12233
|
-
mapped: {
|
|
12234
|
-
name,
|
|
12235
|
-
location: location2,
|
|
12236
|
-
city,
|
|
12237
|
-
region,
|
|
12238
|
-
organization: mappedValue(source, columnMap.organization),
|
|
12239
|
-
websiteUrl,
|
|
12240
|
-
profileUrl,
|
|
12241
|
-
phone: mappedValue(source, columnMap.phone),
|
|
12242
|
-
email: mappedValue(source, columnMap.email)?.toLowerCase() ?? null,
|
|
12243
|
-
entityType
|
|
12244
|
-
}
|
|
12245
|
-
};
|
|
12246
|
-
});
|
|
12247
|
-
return { headers, rows: normalizedRows, serializedBytes, columnMap, options };
|
|
12248
|
-
}
|
|
12249
|
-
function buildResolutionQuery(row) {
|
|
12250
|
-
return {
|
|
12251
|
-
query: [row.name, row.organization].filter(Boolean).join(" "),
|
|
12252
|
-
location: combineLocation(row.location, row.city, row.region)
|
|
12253
|
-
};
|
|
12254
|
-
}
|
|
12255
|
-
var LEGAL_SUFFIXES2 = /* @__PURE__ */ new Set(["co", "company", "corp", "corporation", "inc", "incorporated", "llc", "llp", "lp", "ltd", "limited", "pllc", "pc"]);
|
|
12256
|
-
function normalizedName(value) {
|
|
12257
|
-
const words2 = value.toLowerCase().replace(/&/g, " and ").replace(/[^a-z0-9]+/g, " ").trim().split(/\s+/).filter(Boolean);
|
|
12258
|
-
while (words2.length && LEGAL_SUFFIXES2.has(words2[words2.length - 1])) words2.pop();
|
|
12259
|
-
return words2;
|
|
12260
|
-
}
|
|
12261
|
-
function nameSimilarity(a, b) {
|
|
12262
|
-
const left = normalizedName(a);
|
|
12263
|
-
const right = normalizedName(b);
|
|
12264
|
-
if (!left.length || !right.length) return 0;
|
|
12265
|
-
if (left.join(" ") === right.join(" ")) return 1;
|
|
12266
|
-
const leftSet = new Set(left);
|
|
12267
|
-
const overlap = right.filter((word) => leftSet.has(word)).length;
|
|
12268
|
-
const dice = 2 * overlap / (left.length + right.length);
|
|
12269
|
-
const containment = overlap / Math.min(left.length, right.length);
|
|
12270
|
-
return Math.max(dice, containment >= 1 ? 0.9 : containment * 0.85);
|
|
12271
|
-
}
|
|
12272
|
-
function normalizedPhone(value) {
|
|
12273
|
-
const digits = (value ?? "").replace(/\D/g, "");
|
|
12274
|
-
return digits.length > 10 ? digits.slice(-10) : digits;
|
|
12275
|
-
}
|
|
12276
|
-
function domainOf(value) {
|
|
12277
|
-
try {
|
|
12278
|
-
return new URL(value ?? "").hostname.toLowerCase().replace(/^www\./, "") || null;
|
|
12279
|
-
} catch {
|
|
12280
|
-
return null;
|
|
12281
|
-
}
|
|
12282
|
-
}
|
|
12283
|
-
function includesWords(haystack, needle) {
|
|
12284
|
-
const wanted = normalizedName(needle ?? "").filter((word) => word.length > 2);
|
|
12285
|
-
const source = normalizedName(haystack ?? "");
|
|
12286
|
-
return Boolean(wanted.length && wanted.every((word) => source.includes(word)));
|
|
12287
|
-
}
|
|
12288
|
-
function scoreCandidate(row, candidate) {
|
|
12289
|
-
const similarity2 = nameSimilarity(row.name, candidate.name);
|
|
12290
|
-
if (similarity2 < 0.72) return null;
|
|
12291
|
-
const reasons = [similarity2 === 1 ? "exact_name_match" : "near_name_match"];
|
|
12292
|
-
const corroborators = [];
|
|
12293
|
-
const rowPhone = normalizedPhone(row.phone);
|
|
12294
|
-
const candidatePhone = normalizedPhone(candidate.phone);
|
|
12295
|
-
if (rowPhone.length >= 7 && candidatePhone.endsWith(rowPhone.slice(-7))) corroborators.push("phone_match");
|
|
12296
|
-
const rowLocation = combineLocation(row.location, row.city, row.region);
|
|
12297
|
-
if (rowLocation && includesWords(candidate.address, rowLocation)) corroborators.push("location_match");
|
|
12298
|
-
if (row.city && includesWords(candidate.address, row.city)) corroborators.push("location_match");
|
|
12299
|
-
if (row.organization && (includesWords(candidate.organization, row.organization) || includesWords(candidate.name, row.organization))) corroborators.push("organization_match");
|
|
12300
|
-
const suppliedDomain = domainOf(row.websiteUrl ?? row.profileUrl);
|
|
12301
|
-
if (suppliedDomain && suppliedDomain === domainOf(candidate.websiteUrl)) corroborators.push("domain_match");
|
|
12302
|
-
const uniqueCorroborators = [...new Set(corroborators)];
|
|
12303
|
-
if (!uniqueCorroborators.length) return null;
|
|
12304
|
-
reasons.push(...uniqueCorroborators);
|
|
12305
|
-
const confidence = Math.min(1, Number((similarity2 * 0.7 + 0.2 + Math.min(0.1, (uniqueCorroborators.length - 1) * 0.05)).toFixed(3)));
|
|
12306
|
-
return { candidate, confidence, reasons, corroborators: uniqueCorroborators };
|
|
12307
|
-
}
|
|
12308
|
-
function rankIdentityCandidates(row, candidates) {
|
|
12309
|
-
const providedUrl = row.profileUrl ?? row.websiteUrl;
|
|
12310
|
-
if (providedUrl) {
|
|
12311
|
-
return { status: "provided_url", confidence: 1, reasons: ["caller_provided_url"], selected: null, candidates: [] };
|
|
12312
|
-
}
|
|
12313
|
-
if (!row.name) return { status: "invalid", confidence: 0, reasons: ["missing_name"], selected: null, candidates: [] };
|
|
12314
|
-
const ranked = candidates.map((candidate) => scoreCandidate(row, candidate)).filter((item) => Boolean(item)).sort((a, b) => b.confidence - a.confidence || a.candidate.name.localeCompare(b.candidate.name));
|
|
12315
|
-
if (!ranked.length) return { status: "not_found", confidence: 0, reasons: ["no_corroborated_candidate"], selected: null, candidates: [] };
|
|
12316
|
-
const plausible = ranked.filter((item) => item.confidence >= ranked[0].confidence - 0.08);
|
|
12317
|
-
if (plausible.length > 1) {
|
|
12318
|
-
return { status: "ambiguous", confidence: ranked[0].confidence, reasons: ["multiple_plausible_candidates"], selected: null, candidates: plausible.slice(0, 3) };
|
|
12319
|
-
}
|
|
12320
|
-
return { status: "matched", confidence: ranked[0].confidence, reasons: ranked[0].reasons, selected: ranked[0].candidate, candidates: ranked.slice(0, 3) };
|
|
12321
|
-
}
|
|
12322
|
-
function serpResults(response) {
|
|
12323
|
-
return Array.isArray(response) ? response : response.organicResults;
|
|
12324
|
-
}
|
|
12325
|
-
async function discoverAssociatedPeople(input) {
|
|
12326
|
-
if (input.options.peopleDiscovery === "off") return [];
|
|
12327
|
-
if (input.mapped.entityType !== "business") {
|
|
12328
|
-
input.warnings.push("people_discovery_skipped_for_person");
|
|
12329
|
-
return [];
|
|
12330
|
-
}
|
|
12331
|
-
if (!input.adapters.searchPeopleSerp) {
|
|
12332
|
-
input.warnings.push("people_search_failed");
|
|
12333
|
-
return [];
|
|
12334
|
-
}
|
|
12335
|
-
const searchPeopleSerp = input.adapters.searchPeopleSerp;
|
|
12336
|
-
const mode = input.options.peopleDiscovery;
|
|
12337
|
-
const location2 = combineLocation(input.mapped.location, input.mapped.city, input.mapped.region);
|
|
12338
|
-
let candidates;
|
|
12339
|
-
try {
|
|
12340
|
-
const response = await searchPeopleSerp({
|
|
12341
|
-
query: buildAssociatedPeopleQuery({ businessName: input.mapped.name, location: location2, mode }),
|
|
12342
|
-
gl: input.options.gl,
|
|
12343
|
-
hl: input.options.hl
|
|
12344
|
-
});
|
|
12345
|
-
input.usage.serpSearches += 1;
|
|
12346
|
-
candidates = extractAssociatedPeopleEvidence({
|
|
12347
|
-
businessName: input.mapped.name,
|
|
12348
|
-
mode,
|
|
12349
|
-
results: serpResults(response)
|
|
12350
|
-
}).slice(0, input.options.maxPeoplePerLead);
|
|
12351
|
-
} catch {
|
|
12352
|
-
input.warnings.push("people_search_failed");
|
|
12353
|
-
return [];
|
|
12354
|
-
}
|
|
12355
|
-
if (!candidates.length) {
|
|
12356
|
-
input.warnings.push("people_not_found");
|
|
12357
|
-
return [];
|
|
12358
|
-
}
|
|
12359
|
-
const people = [];
|
|
12360
|
-
for (const person of candidates) {
|
|
12361
|
-
try {
|
|
12362
|
-
const response = await searchPeopleSerp({
|
|
12363
|
-
query: buildLinkedInPersonQuery({ personName: person.name, businessName: input.mapped.name }),
|
|
12364
|
-
gl: input.options.gl,
|
|
12365
|
-
hl: input.options.hl
|
|
12366
|
-
});
|
|
12367
|
-
input.usage.serpSearches += 1;
|
|
12368
|
-
people.push(attachLinkedInProfile({
|
|
12369
|
-
person,
|
|
12370
|
-
businessName: input.mapped.name,
|
|
12371
|
-
location: location2,
|
|
12372
|
-
results: serpResults(response)
|
|
12373
|
-
}));
|
|
12374
|
-
} catch {
|
|
12375
|
-
input.warnings.push("people_linkedin_search_failed");
|
|
12376
|
-
people.push(person);
|
|
12377
|
-
}
|
|
12378
|
-
}
|
|
12379
|
-
return people;
|
|
12380
|
-
}
|
|
12381
|
-
function emptyContacts() {
|
|
12382
|
-
return { website: null, phone: null, primaryEmail: null, emails: [], socialProfiles: [], contactFormUrl: null, sourceUrls: [] };
|
|
12383
|
-
}
|
|
12384
|
-
function mergeEvidence(target, source) {
|
|
12385
|
-
const emailKeys = new Set(target.emails.map((item) => item.value));
|
|
12386
|
-
for (const item of source.emails) {
|
|
12387
|
-
if (target.emails.length >= 20) break;
|
|
12388
|
-
if (!emailKeys.has(item.value)) {
|
|
12389
|
-
target.emails.push(item);
|
|
12390
|
-
emailKeys.add(item.value);
|
|
12391
|
-
}
|
|
12392
|
-
}
|
|
12393
|
-
const socialKeys = new Set(target.socialProfiles.map((item) => item.url));
|
|
12394
|
-
for (const item of source.socialProfiles) {
|
|
12395
|
-
if (target.socialProfiles.length >= 20) break;
|
|
12396
|
-
if (!socialKeys.has(item.url)) {
|
|
12397
|
-
target.socialProfiles.push(item);
|
|
12398
|
-
socialKeys.add(item.url);
|
|
12399
|
-
}
|
|
12400
|
-
}
|
|
12401
|
-
const formKeys = new Set(target.contactForms.map((item) => item.url));
|
|
12402
|
-
for (const item of source.contactForms) {
|
|
12403
|
-
if (target.contactForms.length >= 10) break;
|
|
12404
|
-
if (!formKeys.has(item.url)) {
|
|
12405
|
-
target.contactForms.push(item);
|
|
12406
|
-
formKeys.add(item.url);
|
|
12407
|
-
}
|
|
12408
|
-
}
|
|
12409
|
-
const pageKeys = new Set(target.candidatePageUrls.map((item) => item.url));
|
|
12410
|
-
for (const item of source.candidatePageUrls) {
|
|
12411
|
-
if (target.candidatePageUrls.length >= 20) break;
|
|
12412
|
-
if (!pageKeys.has(item.url)) {
|
|
12413
|
-
target.candidatePageUrls.push(item);
|
|
12414
|
-
pageKeys.add(item.url);
|
|
12415
|
-
}
|
|
12416
|
-
}
|
|
12417
|
-
}
|
|
12418
|
-
function kpoEvidence(page, sourceUrl) {
|
|
12419
|
-
const empty = { emails: [], socialProfiles: [], contactForms: [], candidatePageUrls: [] };
|
|
12420
|
-
const email = page.kpo?.email?.trim().toLowerCase();
|
|
12421
|
-
if (email) empty.emails.push({ value: email, sourceUrl, context: `Structured entity data for ${page.kpo?.entityName ?? "the page entity"}`, sourceType: "structured_data" });
|
|
12422
|
-
for (const value of [...page.kpo?.sameAs ?? [], ...page.kpo?.socialLinks ?? []]) {
|
|
12423
|
-
try {
|
|
12424
|
-
const url = new URL(value);
|
|
12425
|
-
if (!["http:", "https:"].includes(url.protocol)) continue;
|
|
12426
|
-
const network = url.hostname.replace(/^www\./, "").split(".")[0];
|
|
12427
|
-
if (!empty.socialProfiles.some((item) => item.url === url.href)) empty.socialProfiles.push({ url: url.href, network, sourceUrl, context: "Structured entity social profile" });
|
|
12428
|
-
} catch {
|
|
12429
|
-
}
|
|
12430
|
-
}
|
|
12431
|
-
return empty;
|
|
12432
|
-
}
|
|
12433
|
-
function normalizeCandidate(input) {
|
|
12434
|
-
return {
|
|
12435
|
-
name: input.name,
|
|
12436
|
-
websiteUrl: input.websiteUrl ?? null,
|
|
12437
|
-
address: input.address ?? null,
|
|
12438
|
-
phone: input.phone ?? null,
|
|
12439
|
-
placeUrl: input.placeUrl ?? null,
|
|
12440
|
-
cid: input.cid == null ? null : String(input.cid),
|
|
12441
|
-
organization: input.organization ?? null,
|
|
12442
|
-
category: input.category ?? null
|
|
12443
|
-
};
|
|
12444
|
-
}
|
|
12445
|
-
function isNormalizedLeadRow(value) {
|
|
12446
|
-
return typeof value === "object" && value !== null && "source" in value && typeof value.source === "object" && value.source !== null && "mapped" in value && typeof value.mapped === "object" && value.mapped !== null;
|
|
12447
|
-
}
|
|
12448
|
-
async function enrichLeadRow(input) {
|
|
12449
|
-
let normalized;
|
|
12450
|
-
let options;
|
|
12451
|
-
try {
|
|
12452
|
-
if (isNormalizedLeadRow(input.row)) {
|
|
12453
|
-
normalized = input.row;
|
|
12454
|
-
options = LeadListEnrichmentOptionsSchema.parse(input.options ?? {});
|
|
12455
|
-
} else {
|
|
12456
|
-
const list = normalizeLeadRows({ rows: [input.row], columnMap: input.columnMap, options: input.options });
|
|
12457
|
-
normalized = { ...list.rows[0], ordinal: input.ordinal };
|
|
12458
|
-
options = list.options;
|
|
12459
|
-
}
|
|
12460
|
-
} catch {
|
|
12461
|
-
return {
|
|
12462
|
-
ordinal: input.ordinal,
|
|
12463
|
-
source: isNormalizedLeadRow(input.row) ? input.row.source : input.row,
|
|
12464
|
-
rowStatus: "failed",
|
|
12465
|
-
identity: { status: "invalid", confidence: 0, reasons: ["invalid_input"], selected: null, candidates: [] },
|
|
12466
|
-
contacts: emptyContacts(),
|
|
12467
|
-
people: [],
|
|
12468
|
-
usage: { mapsAttempts: 0, pageAttempts: 0, pageSuccesses: 0, serpSearches: 0 },
|
|
12469
|
-
warnings: [],
|
|
12470
|
-
error: { code: "invalid_row", message: "The row could not be normalized with the supplied column mapping." }
|
|
12471
|
-
};
|
|
12472
|
-
}
|
|
12473
|
-
const usage = { mapsAttempts: 0, pageAttempts: 0, pageSuccesses: 0, serpSearches: 0 };
|
|
12474
|
-
let identity;
|
|
12475
|
-
if (normalized.mapped.profileUrl || normalized.mapped.websiteUrl) {
|
|
12476
|
-
identity = rankIdentityCandidates(normalized.mapped, []);
|
|
12477
|
-
} else {
|
|
12478
|
-
const resolutionQuery = buildResolutionQuery(normalized.mapped);
|
|
12479
|
-
try {
|
|
12480
|
-
usage.mapsAttempts = 1;
|
|
12481
|
-
const response = await input.adapters.mapsSearch({ query: resolutionQuery.query, location: resolutionQuery.location, maxResults: 5, gl: options.gl, hl: options.hl });
|
|
12482
|
-
const candidates = (Array.isArray(response) ? response : response.results).map(normalizeCandidate);
|
|
12483
|
-
identity = rankIdentityCandidates(normalized.mapped, candidates);
|
|
12484
|
-
} catch {
|
|
12485
|
-
return {
|
|
12486
|
-
ordinal: normalized.ordinal,
|
|
12487
|
-
source: normalized.source,
|
|
12488
|
-
rowStatus: "failed",
|
|
12489
|
-
identity: { status: "not_found", confidence: 0, reasons: ["maps_provider_failure"], selected: null, candidates: [] },
|
|
12490
|
-
contacts: emptyContacts(),
|
|
12491
|
-
people: [],
|
|
12492
|
-
usage,
|
|
12493
|
-
warnings: [],
|
|
12494
|
-
error: { code: "resolution_failed", message: "The business lookup could not be completed for this row." }
|
|
12495
|
-
};
|
|
12496
|
-
}
|
|
12497
|
-
}
|
|
12498
|
-
if (identity.status === "ambiguous" || identity.status === "not_found" || identity.status === "invalid") {
|
|
12499
|
-
return { ordinal: normalized.ordinal, source: normalized.source, rowStatus: identity.status === "invalid" ? "failed" : "complete", identity, contacts: emptyContacts(), people: [], usage, warnings: [], error: identity.status === "invalid" ? { code: "invalid_row", message: "The row could not be resolved." } : null };
|
|
12500
|
-
}
|
|
12501
|
-
const seedUrl = normalized.mapped.profileUrl ?? normalized.mapped.websiteUrl ?? identity.selected?.websiteUrl ?? null;
|
|
12502
|
-
const website = seedUrl ? { value: seedUrl, sourceUrl: identity.status === "provided_url" ? "provided" : identity.selected?.placeUrl ?? "maps_search" } : null;
|
|
12503
|
-
const phone = normalized.mapped.phone ? { value: normalized.mapped.phone, sourceUrl: "provided" } : identity.selected?.phone ? { value: identity.selected.phone, sourceUrl: identity.selected.placeUrl ?? "maps_search" } : null;
|
|
12504
|
-
const warnings = [];
|
|
12505
|
-
if (!seedUrl) {
|
|
12506
|
-
const people2 = await discoverAssociatedPeople({ mapped: normalized.mapped, options, adapters: input.adapters, usage, warnings });
|
|
12507
|
-
const publicWarnings2 = [
|
|
12508
|
-
...warnings.includes("people_search_failed") ? ["The optional associated-people search could not be completed."] : [],
|
|
12509
|
-
...warnings.includes("people_linkedin_search_failed") ? ["One or more optional LinkedIn candidate searches could not be completed."] : [],
|
|
12510
|
-
...warnings.includes("people_not_found") ? ["No role-bearing associated people were found in the bounded search."] : [],
|
|
12511
|
-
...warnings.includes("people_discovery_skipped_for_person") ? ["Associated-people discovery applies to business rows and was skipped for this person row."] : []
|
|
12512
|
-
];
|
|
12513
|
-
return {
|
|
12514
|
-
ordinal: normalized.ordinal,
|
|
12515
|
-
source: normalized.source,
|
|
12516
|
-
rowStatus: "complete",
|
|
12517
|
-
identity,
|
|
12518
|
-
contacts: { ...emptyContacts(), website, phone, sourceUrls: [website?.sourceUrl, phone?.sourceUrl].filter((value) => Boolean(value)) },
|
|
12519
|
-
people: people2,
|
|
12520
|
-
usage,
|
|
12521
|
-
warnings: ["The matched listing did not expose a public website.", ...publicWarnings2],
|
|
12522
|
-
error: null
|
|
12523
|
-
};
|
|
12524
|
-
}
|
|
12525
|
-
const allEvidence = { emails: [], socialProfiles: [], contactForms: [], candidatePageUrls: [] };
|
|
12526
|
-
if (normalized.mapped.email) {
|
|
12527
|
-
allEvidence.emails.push({
|
|
12528
|
-
value: normalized.mapped.email,
|
|
12529
|
-
sourceUrl: "provided",
|
|
12530
|
-
context: `Provided for ${normalized.mapped.name}`,
|
|
12531
|
-
sourceType: "provided"
|
|
12532
|
-
});
|
|
12533
|
-
}
|
|
12534
|
-
const attempted = /* @__PURE__ */ new Set();
|
|
12535
|
-
let plan = [seedUrl];
|
|
12536
|
-
let pagePhone = null;
|
|
12537
|
-
while (plan.length && attempted.size < options.maxPagesPerLead) {
|
|
12538
|
-
const url = plan.find((item) => !attempted.has(item));
|
|
12539
|
-
if (!url) break;
|
|
12540
|
-
attempted.add(url);
|
|
12541
|
-
usage.pageAttempts += 1;
|
|
12542
|
-
try {
|
|
12543
|
-
const page = await input.adapters.extractPage({ url });
|
|
12544
|
-
usage.pageSuccesses += 1;
|
|
12545
|
-
const pageUrl = page.url ?? url;
|
|
12546
|
-
const evidence = page.contactEvidence ?? extractContactEvidence(page.bodyHtml ?? page.bodyMarkdown ?? "", pageUrl);
|
|
12547
|
-
mergeEvidence(evidence, kpoEvidence(page, pageUrl));
|
|
12548
|
-
mergeEvidence(allEvidence, evidence);
|
|
12549
|
-
if (!pagePhone && page.kpo?.phone) pagePhone = { value: page.kpo.phone, sourceUrl: pageUrl };
|
|
12550
|
-
plan = contactAttemptUrls(seedUrl, allEvidence.candidatePageUrls, options.maxPagesPerLead);
|
|
12551
|
-
} catch {
|
|
12552
|
-
warnings.push("page_failed");
|
|
12553
|
-
plan = contactAttemptUrls(seedUrl, allEvidence.candidatePageUrls, options.maxPagesPerLead);
|
|
12554
|
-
}
|
|
12555
|
-
}
|
|
12556
|
-
if (!allEvidence.emails.length && options.emailSearchFallback === "serp_snippets") {
|
|
12557
|
-
const query = buildSerpEmailQuery(seedUrl);
|
|
12558
|
-
if (query) {
|
|
12559
|
-
try {
|
|
12560
|
-
const response = await input.adapters.searchEmailSerp({ query, gl: options.gl, hl: options.hl });
|
|
12561
|
-
usage.serpSearches += 1;
|
|
12562
|
-
const organicResults = Array.isArray(response) ? response : response.organicResults;
|
|
12563
|
-
mergeEvidence(allEvidence, {
|
|
12564
|
-
emails: extractSerpEmailEvidence({ websiteUrl: seedUrl, results: organicResults }),
|
|
12565
|
-
socialProfiles: [],
|
|
12566
|
-
contactForms: [],
|
|
12567
|
-
candidatePageUrls: []
|
|
12568
|
-
});
|
|
12569
|
-
} catch {
|
|
12570
|
-
warnings.push("serp_failed");
|
|
12571
|
-
}
|
|
12572
|
-
}
|
|
12573
|
-
}
|
|
12574
|
-
const people = await discoverAssociatedPeople({ mapped: normalized.mapped, options, adapters: input.adapters, usage, warnings });
|
|
12575
|
-
const publicWarnings = [
|
|
12576
|
-
...warnings.includes("page_failed") ? ["One or more selected website pages could not be read."] : [],
|
|
12577
|
-
...warnings.includes("serp_failed") ? ["The optional email search could not be completed."] : [],
|
|
12578
|
-
...warnings.includes("people_search_failed") ? ["The optional associated-people search could not be completed."] : [],
|
|
12579
|
-
...warnings.includes("people_linkedin_search_failed") ? ["One or more optional LinkedIn candidate searches could not be completed."] : [],
|
|
12580
|
-
...warnings.includes("people_not_found") ? ["No role-bearing associated people were found in the bounded search."] : [],
|
|
12581
|
-
...warnings.includes("people_discovery_skipped_for_person") ? ["Associated-people discovery applies to business rows and was skipped for this person row."] : []
|
|
12582
|
-
];
|
|
12583
|
-
if (!usage.pageSuccesses && !allEvidence.emails.length && !people.length) {
|
|
12584
|
-
return {
|
|
12585
|
-
ordinal: normalized.ordinal,
|
|
12586
|
-
source: normalized.source,
|
|
12587
|
-
rowStatus: "failed",
|
|
12588
|
-
identity,
|
|
12589
|
-
contacts: { ...emptyContacts(), website, phone },
|
|
12590
|
-
people,
|
|
12591
|
-
usage,
|
|
12592
|
-
warnings: publicWarnings,
|
|
12593
|
-
error: { code: "page_extraction_failed", message: "No selected website page could be read for this row." }
|
|
12594
|
-
};
|
|
12595
|
-
}
|
|
12596
|
-
const entity = { entityType: normalized.mapped.entityType, name: normalized.mapped.name, organization: normalized.mapped.organization };
|
|
12597
|
-
const sourcePriority = ["provided", "first_party_page", "structured_data", "serp_snippet"];
|
|
12598
|
-
const scopePriority = ["person", "team", "office", "unknown"];
|
|
12599
|
-
const emails = allEvidence.emails.map((item) => ({ ...item, ...classifyEmailScope(item, entity) })).sort((left, right) => sourcePriority.indexOf(left.sourceType) - sourcePriority.indexOf(right.sourceType) || scopePriority.indexOf(left.scope) - scopePriority.indexOf(right.scope));
|
|
12600
|
-
const primaryEmail = selectPrimaryEmail(allEvidence.emails, entity);
|
|
12601
|
-
const contactFormUrl = allEvidence.contactForms[0] ? { value: allEvidence.contactForms[0].url, sourceUrl: allEvidence.contactForms[0].sourceUrl } : null;
|
|
12602
|
-
const resolvedPhone = phone ?? pagePhone;
|
|
12603
|
-
const sourceUrls = [...new Set([
|
|
12604
|
-
website?.sourceUrl,
|
|
12605
|
-
resolvedPhone?.sourceUrl,
|
|
12606
|
-
...emails.map((item) => item.sourceUrl),
|
|
12607
|
-
...allEvidence.socialProfiles.map((item) => item.sourceUrl),
|
|
12608
|
-
contactFormUrl?.sourceUrl
|
|
12609
|
-
].filter((value) => Boolean(value)))];
|
|
12610
|
-
return {
|
|
12611
|
-
ordinal: normalized.ordinal,
|
|
12612
|
-
source: normalized.source,
|
|
12613
|
-
rowStatus: "complete",
|
|
12614
|
-
identity,
|
|
12615
|
-
contacts: { website, phone: resolvedPhone, primaryEmail, emails, socialProfiles: allEvidence.socialProfiles.slice(0, 20), contactFormUrl, sourceUrls },
|
|
12616
|
-
people,
|
|
12617
|
-
usage,
|
|
12618
|
-
warnings: publicWarnings,
|
|
12619
|
-
error: null
|
|
12620
|
-
};
|
|
12621
|
-
}
|
|
12622
|
-
var LEAD_LIST_ENRICHED_HEADERS = [
|
|
12623
|
-
"enriched_row_status",
|
|
12624
|
-
"enriched_identity_status",
|
|
12625
|
-
"enriched_match_confidence",
|
|
12626
|
-
"enriched_matched_name",
|
|
12627
|
-
"enriched_organization",
|
|
12628
|
-
"enriched_website",
|
|
12629
|
-
"enriched_phone",
|
|
12630
|
-
"enriched_primary_email",
|
|
12631
|
-
"enriched_primary_email_scope",
|
|
12632
|
-
"enriched_secondary_email",
|
|
12633
|
-
"enriched_all_emails",
|
|
12634
|
-
"enriched_social_profiles",
|
|
12635
|
-
"enriched_contact_form_url",
|
|
12636
|
-
"enriched_primary_person_name",
|
|
12637
|
-
"enriched_primary_person_role",
|
|
12638
|
-
"enriched_primary_person_linkedin",
|
|
12639
|
-
"enriched_all_people",
|
|
12640
|
-
"enriched_maps_url",
|
|
12641
|
-
"enriched_cid",
|
|
12642
|
-
"enriched_source_urls",
|
|
12643
|
-
"enriched_warnings"
|
|
12644
|
-
];
|
|
12645
|
-
function jsonCell(value) {
|
|
12646
|
-
return JSON.stringify(value);
|
|
12647
|
-
}
|
|
12648
|
-
function leadListOutputRows(results, sourceHeaders) {
|
|
12649
|
-
const headers = sourceHeaders ?? Object.keys(results[0]?.source ?? {});
|
|
12650
|
-
for (const header of headers) if (header.toLowerCase().startsWith(RESERVED_OUTPUT_PREFIX)) throw new Error(`Source header ${header} collides with reserved enriched_ output columns`);
|
|
12651
|
-
const ordered = [...results].sort((a, b) => a.ordinal - b.ordinal);
|
|
12652
|
-
return {
|
|
12653
|
-
headers: [...headers, ...LEAD_LIST_ENRICHED_HEADERS],
|
|
12654
|
-
rows: ordered.map((result) => {
|
|
12655
|
-
const secondaryEmail = result.contacts.primaryEmail ? result.contacts.emails.find((item) => item.value !== result.contacts.primaryEmail?.value) ?? null : null;
|
|
12656
|
-
return [
|
|
12657
|
-
...headers.map((header) => result.source[header] ?? null),
|
|
12658
|
-
result.rowStatus,
|
|
12659
|
-
result.identity.status,
|
|
12660
|
-
result.identity.confidence,
|
|
12661
|
-
result.identity.selected?.name ?? "",
|
|
12662
|
-
result.identity.selected?.organization ?? "",
|
|
12663
|
-
result.contacts.website?.value ?? "",
|
|
12664
|
-
result.contacts.phone?.value ?? "",
|
|
12665
|
-
result.contacts.primaryEmail?.value ?? "",
|
|
12666
|
-
result.contacts.primaryEmail?.scope ?? "",
|
|
12667
|
-
secondaryEmail?.value ?? "",
|
|
12668
|
-
jsonCell(result.contacts.emails.map((item) => ({
|
|
12669
|
-
value: item.value,
|
|
12670
|
-
scope: item.scope,
|
|
12671
|
-
sourceType: item.sourceType,
|
|
12672
|
-
sourceUrl: item.sourceUrl,
|
|
12673
|
-
...item.sourceTitle ? { sourceTitle: item.sourceTitle } : {},
|
|
12674
|
-
...item.sourcePosition != null ? { sourcePosition: item.sourcePosition } : {},
|
|
12675
|
-
context: item.context,
|
|
12676
|
-
reasons: item.reasons
|
|
12677
|
-
}))),
|
|
12678
|
-
jsonCell(result.contacts.socialProfiles.map((item) => ({ url: item.url, network: item.network, sourceUrl: item.sourceUrl }))),
|
|
12679
|
-
result.contacts.contactFormUrl?.value ?? "",
|
|
12680
|
-
result.people[0]?.name ?? "",
|
|
12681
|
-
result.people[0]?.role ?? "",
|
|
12682
|
-
result.people[0]?.linkedinUrl ?? "",
|
|
12683
|
-
jsonCell(result.people),
|
|
12684
|
-
result.identity.selected?.placeUrl ?? "",
|
|
12685
|
-
result.identity.selected?.cid ?? "",
|
|
12686
|
-
jsonCell(result.contacts.sourceUrls),
|
|
12687
|
-
jsonCell([...result.warnings, ...result.error ? [result.error.message] : []])
|
|
12688
|
-
];
|
|
12689
|
-
})
|
|
12690
|
-
};
|
|
12691
|
-
}
|
|
12692
|
-
|
|
12693
11910
|
// src/leads/lead-list-workbook.ts
|
|
12694
11911
|
import ExcelJS from "exceljs";
|
|
12695
11912
|
import Papa from "papaparse";
|
|
@@ -12847,7 +12064,7 @@ function summarizePersistedLeadRows(rows, warnings = []) {
|
|
|
12847
12064
|
async function settlePersistedLeadListEnrichment(jobId2) {
|
|
12848
12065
|
const jobs = await listLeadListEnrichmentJobsNeedingSettlement(100);
|
|
12849
12066
|
const job = jobs.find((candidate) => candidate.id === jobId2);
|
|
12850
|
-
const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-
|
|
12067
|
+
const { getLeadListEnrichmentJob: getLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-OTAUQONJ.js");
|
|
12851
12068
|
const current = job ?? await getLeadListEnrichmentJob2(jobId2);
|
|
12852
12069
|
if (!current) return null;
|
|
12853
12070
|
const rows = await listLeadListEnrichmentRows(jobId2);
|
|
@@ -12903,7 +12120,7 @@ async function reconcileLeadListEnrichmentSettlements(limit = 25) {
|
|
|
12903
12120
|
const persisted = job.result ?? summarizePersistedLeadRows(rows, job.warnings);
|
|
12904
12121
|
const status = job.cancelRequestedAt ? "cancelled" : persisted.failedRows > 0 && persisted.completeRows > 0 ? "partial" : persisted.failedRows > 0 ? "cancelled" : persisted.rowsWithContacts > 0 ? "complete" : "empty";
|
|
12905
12122
|
if (persisted.failedRows > 0 && persisted.completeRows === 0 && !job.cancelRequestedAt) {
|
|
12906
|
-
const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-
|
|
12123
|
+
const { failLeadListEnrichmentJob: failLeadListEnrichmentJob2 } = await import("./lead-list-enrichment-repository-OTAUQONJ.js");
|
|
12907
12124
|
await failLeadListEnrichmentJob2({
|
|
12908
12125
|
jobId: job.id,
|
|
12909
12126
|
errorCode: "lead_list_all_rows_failed",
|
|
@@ -12964,12 +12181,13 @@ function createLeadListEnrichmentAdapters(userId) {
|
|
|
12964
12181
|
pages: 1,
|
|
12965
12182
|
maxQuestions: 1,
|
|
12966
12183
|
headless: true,
|
|
12184
|
+
includeAiOverview: input.includeAiOverview === true,
|
|
12967
12185
|
format: "json",
|
|
12968
12186
|
outputDir: "/tmp/paa-output-api",
|
|
12969
12187
|
kernelApiKey,
|
|
12970
12188
|
proxyMode: DEFAULT_PROXY_MODE
|
|
12971
12189
|
}));
|
|
12972
|
-
return result
|
|
12190
|
+
return result;
|
|
12973
12191
|
};
|
|
12974
12192
|
return {
|
|
12975
12193
|
async mapsSearch(input) {
|
|
@@ -13002,8 +12220,13 @@ function createLeadListEnrichmentAdapters(userId) {
|
|
|
13002
12220
|
contactEvidence: page.contactEvidence
|
|
13003
12221
|
};
|
|
13004
12222
|
},
|
|
13005
|
-
searchEmailSerp
|
|
13006
|
-
|
|
12223
|
+
async searchEmailSerp(input) {
|
|
12224
|
+
return (await searchSerp(input)).organicResults;
|
|
12225
|
+
},
|
|
12226
|
+
async searchPeopleSerp(input) {
|
|
12227
|
+
const result = await searchSerp(input);
|
|
12228
|
+
return { organicResults: result.organicResults, aiOverview: result.aiOverview };
|
|
12229
|
+
}
|
|
13007
12230
|
};
|
|
13008
12231
|
}
|
|
13009
12232
|
function cancelledResult(row) {
|
|
@@ -15059,7 +14282,7 @@ var MapsImageCollector = class {
|
|
|
15059
14282
|
};
|
|
15060
14283
|
|
|
15061
14284
|
// src/extractor/MapsExtractor.ts
|
|
15062
|
-
import { z as
|
|
14285
|
+
import { z as z11 } from "zod";
|
|
15063
14286
|
var MapsExtractor = class {
|
|
15064
14287
|
constructor(driver) {
|
|
15065
14288
|
this.driver = driver;
|
|
@@ -15329,7 +14552,7 @@ var MapsExtractor = class {
|
|
|
15329
14552
|
});
|
|
15330
14553
|
return rows;
|
|
15331
14554
|
}, { hoursTable: MapsSelectors.hoursTable, hoursTableAlt: MapsSelectors.hoursTableAlt });
|
|
15332
|
-
const result =
|
|
14555
|
+
const result = z11.array(RawMapsHoursRowSchema).safeParse(raw);
|
|
15333
14556
|
if (!result.success) {
|
|
15334
14557
|
console.warn("[MapsExtractor] hours parse failed", result.error.flatten());
|
|
15335
14558
|
return [];
|
|
@@ -15393,7 +14616,7 @@ var MapsExtractor = class {
|
|
|
15393
14616
|
});
|
|
15394
14617
|
return results;
|
|
15395
14618
|
});
|
|
15396
|
-
const result =
|
|
14619
|
+
const result = z11.array(RawMapsAboutAttributeSchema).safeParse(raw);
|
|
15397
14620
|
if (!result.success) {
|
|
15398
14621
|
console.warn("[MapsExtractor] about parse failed", result.error.flatten());
|
|
15399
14622
|
return [];
|
|
@@ -16903,7 +16126,7 @@ internalMemoryApp.post("/ai-debit", async (c) => {
|
|
|
16903
16126
|
|
|
16904
16127
|
// src/api/internal-billing-routes.ts
|
|
16905
16128
|
import { Hono as Hono3 } from "hono";
|
|
16906
|
-
import { z as
|
|
16129
|
+
import { z as z13 } from "zod";
|
|
16907
16130
|
|
|
16908
16131
|
// src/api/unified-billing.ts
|
|
16909
16132
|
import { createHash as createHash4, createHmac, randomUUID as randomUUID5, timingSafeEqual } from "crypto";
|
|
@@ -17606,7 +16829,7 @@ async function syncScheduledActionCredentials(user) {
|
|
|
17606
16829
|
|
|
17607
16830
|
// src/api/connected-usage-billing.ts
|
|
17608
16831
|
import { createHash as createHash5, randomUUID as randomUUID6 } from "crypto";
|
|
17609
|
-
import { z as
|
|
16832
|
+
import { z as z12 } from "zod";
|
|
17610
16833
|
|
|
17611
16834
|
// src/api/connected-cost-telemetry.ts
|
|
17612
16835
|
async function recordNangoConnectionCost(userId, connectionMonths) {
|
|
@@ -17640,32 +16863,32 @@ async function recordNangoUsageCost(userId, usage) {
|
|
|
17640
16863
|
// src/api/connected-usage-billing.ts
|
|
17641
16864
|
var CONNECTED_USAGE_BILLING_CLASS = "connected_usage";
|
|
17642
16865
|
var CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE = "mastra_scheduler";
|
|
17643
|
-
var ConnectedUsageSafeMetadataSchema =
|
|
17644
|
-
toolName:
|
|
17645
|
-
operationKind:
|
|
17646
|
-
outcome:
|
|
17647
|
-
requestId:
|
|
17648
|
-
occurrenceId:
|
|
17649
|
-
scheduleId:
|
|
17650
|
-
nangoOperationId:
|
|
17651
|
-
startedAt:
|
|
17652
|
-
completedAt:
|
|
16866
|
+
var ConnectedUsageSafeMetadataSchema = z12.object({
|
|
16867
|
+
toolName: z12.string().trim().min(1).max(200).optional(),
|
|
16868
|
+
operationKind: z12.enum(["read", "action", "sync", "export", "proxy", "other"]).optional(),
|
|
16869
|
+
outcome: z12.enum(["success", "error", "partial"]).optional(),
|
|
16870
|
+
requestId: z12.string().trim().min(1).max(200).optional(),
|
|
16871
|
+
occurrenceId: z12.string().trim().min(1).max(200).optional(),
|
|
16872
|
+
scheduleId: z12.string().trim().min(1).max(200).optional(),
|
|
16873
|
+
nangoOperationId: z12.string().trim().min(1).max(200).optional(),
|
|
16874
|
+
startedAt: z12.string().datetime({ offset: true }).optional(),
|
|
16875
|
+
completedAt: z12.string().datetime({ offset: true }).optional()
|
|
17653
16876
|
}).strict();
|
|
17654
|
-
var ConnectedUsageSettlementInputSchema =
|
|
17655
|
-
identity:
|
|
17656
|
-
idempotencyKey:
|
|
17657
|
-
ratePolicyVersion:
|
|
17658
|
-
connectionId:
|
|
17659
|
-
providerConfigKey:
|
|
17660
|
-
functionRuns:
|
|
17661
|
-
proxyRequests:
|
|
17662
|
-
computeMilliseconds:
|
|
17663
|
-
sourceSurface:
|
|
16877
|
+
var ConnectedUsageSettlementInputSchema = z12.object({
|
|
16878
|
+
identity: z12.string().trim().min(1).max(320),
|
|
16879
|
+
idempotencyKey: z12.string().trim().min(1).max(500),
|
|
16880
|
+
ratePolicyVersion: z12.string().trim().min(1).max(100),
|
|
16881
|
+
connectionId: z12.string().trim().min(1).max(500),
|
|
16882
|
+
providerConfigKey: z12.string().trim().min(1).max(200),
|
|
16883
|
+
functionRuns: z12.number().int().safe().nonnegative(),
|
|
16884
|
+
proxyRequests: z12.number().int().safe().nonnegative(),
|
|
16885
|
+
computeMilliseconds: z12.number().int().safe().nonnegative(),
|
|
16886
|
+
sourceSurface: z12.enum(["main_mcp", "mastra_scheduler", "nango_reconciliation"]).default(CONNECTED_USAGE_DEFAULT_SOURCE_SURFACE),
|
|
17664
16887
|
metadata: ConnectedUsageSafeMetadataSchema.optional()
|
|
17665
16888
|
}).strict();
|
|
17666
|
-
var ConnectedUsagePreflightInputSchema =
|
|
17667
|
-
identity:
|
|
17668
|
-
ratePolicyVersion:
|
|
16889
|
+
var ConnectedUsagePreflightInputSchema = z12.object({
|
|
16890
|
+
identity: z12.string().trim().min(1).max(320),
|
|
16891
|
+
ratePolicyVersion: z12.string().trim().min(1).max(100)
|
|
17669
16892
|
}).strict();
|
|
17670
16893
|
var ConnectedUsageBillingError = class extends Error {
|
|
17671
16894
|
constructor(code, message, httpStatus, details = {}) {
|
|
@@ -18044,40 +17267,40 @@ async function listConnectedUsageHistory(userId, limit = 100) {
|
|
|
18044
17267
|
|
|
18045
17268
|
// src/api/internal-billing-routes.ts
|
|
18046
17269
|
var internalBillingApp = new Hono3();
|
|
18047
|
-
var metadataSchema =
|
|
18048
|
-
var schedulingAccessSchema =
|
|
18049
|
-
identity:
|
|
18050
|
-
provisionCredentials:
|
|
18051
|
-
});
|
|
18052
|
-
var authorizeSchema =
|
|
18053
|
-
identity:
|
|
18054
|
-
idempotencyKey:
|
|
18055
|
-
maximumCredits:
|
|
17270
|
+
var metadataSchema = z13.record(z13.string(), z13.unknown()).optional();
|
|
17271
|
+
var schedulingAccessSchema = z13.object({
|
|
17272
|
+
identity: z13.string().min(1).max(320),
|
|
17273
|
+
provisionCredentials: z13.boolean().optional()
|
|
17274
|
+
});
|
|
17275
|
+
var authorizeSchema = z13.object({
|
|
17276
|
+
identity: z13.string().min(1).max(320),
|
|
17277
|
+
idempotencyKey: z13.string().min(1).max(500),
|
|
17278
|
+
maximumCredits: z13.number().finite().positive().optional(),
|
|
18056
17279
|
metadata: metadataSchema
|
|
18057
17280
|
});
|
|
18058
|
-
var settleSchema =
|
|
18059
|
-
identity:
|
|
18060
|
-
authorizationId:
|
|
18061
|
-
authorizationToken:
|
|
18062
|
-
modelCostStatus:
|
|
18063
|
-
rawCostUsd:
|
|
18064
|
-
inputTokens:
|
|
18065
|
-
outputTokens:
|
|
18066
|
-
cachedTokens:
|
|
18067
|
-
modelSteps:
|
|
17281
|
+
var settleSchema = z13.object({
|
|
17282
|
+
identity: z13.string().min(1).max(320),
|
|
17283
|
+
authorizationId: z13.string().uuid(),
|
|
17284
|
+
authorizationToken: z13.string().min(20).max(500),
|
|
17285
|
+
modelCostStatus: z13.enum(["reported", "missing", "not_applicable"]),
|
|
17286
|
+
rawCostUsd: z13.number().finite().nonnegative().nullable().optional(),
|
|
17287
|
+
inputTokens: z13.number().finite().nonnegative().optional(),
|
|
17288
|
+
outputTokens: z13.number().finite().nonnegative().optional(),
|
|
17289
|
+
cachedTokens: z13.number().finite().nonnegative().optional(),
|
|
17290
|
+
modelSteps: z13.number().finite().nonnegative().optional(),
|
|
18068
17291
|
metadata: metadataSchema
|
|
18069
17292
|
});
|
|
18070
|
-
var startSchema =
|
|
18071
|
-
identity:
|
|
18072
|
-
authorizationId:
|
|
18073
|
-
authorizationToken:
|
|
17293
|
+
var startSchema = z13.object({
|
|
17294
|
+
identity: z13.string().min(1).max(320),
|
|
17295
|
+
authorizationId: z13.string().uuid(),
|
|
17296
|
+
authorizationToken: z13.string().min(20).max(500),
|
|
18074
17297
|
metadata: metadataSchema
|
|
18075
17298
|
});
|
|
18076
|
-
var voidSchema =
|
|
18077
|
-
identity:
|
|
18078
|
-
authorizationId:
|
|
18079
|
-
authorizationToken:
|
|
18080
|
-
reason:
|
|
17299
|
+
var voidSchema = z13.object({
|
|
17300
|
+
identity: z13.string().min(1).max(320),
|
|
17301
|
+
authorizationId: z13.string().uuid(),
|
|
17302
|
+
authorizationToken: z13.string().min(20).max(500),
|
|
17303
|
+
reason: z13.string().max(500).optional()
|
|
18081
17304
|
});
|
|
18082
17305
|
function internalRequestAuthorized(header) {
|
|
18083
17306
|
const expected = process.env.INTERNAL_BILLING_SECRET?.trim() || process.env.INTERNAL_MEMORY_DEBIT_SECRET?.trim();
|
|
@@ -18227,7 +17450,7 @@ internalBillingApp.post("/scheduled-run/void", async (c) => {
|
|
|
18227
17450
|
|
|
18228
17451
|
// src/api/inbox-routes.ts
|
|
18229
17452
|
import { Hono as Hono4 } from "hono";
|
|
18230
|
-
import { z as
|
|
17453
|
+
import { z as z14 } from "zod";
|
|
18231
17454
|
|
|
18232
17455
|
// src/api/api-auth.ts
|
|
18233
17456
|
import { createMiddleware as createMiddleware2 } from "hono/factory";
|
|
@@ -18266,10 +17489,10 @@ inboxApp.delete("/:id", async (c) => {
|
|
|
18266
17489
|
if (!deleted) return c.json({ ok: false, error: "Message not found" }, 404);
|
|
18267
17490
|
return c.json({ ok: true });
|
|
18268
17491
|
});
|
|
18269
|
-
var broadcastSchema =
|
|
18270
|
-
subject:
|
|
18271
|
-
body_md:
|
|
18272
|
-
recipients:
|
|
17492
|
+
var broadcastSchema = z14.object({
|
|
17493
|
+
subject: z14.string().min(1).max(500),
|
|
17494
|
+
body_md: z14.string().min(1).max(2e5),
|
|
17495
|
+
recipients: z14.union([z14.literal("all"), z14.array(z14.string().min(1).max(320)).min(1).max(1e3)])
|
|
18273
17496
|
});
|
|
18274
17497
|
function internalRequestAuthorized2(header) {
|
|
18275
17498
|
const expected = process.env.INTERNAL_BILLING_SECRET ?? process.env.INTERNAL_MEMORY_DEBIT_SECRET;
|
|
@@ -18588,17 +17811,17 @@ import path4 from "path";
|
|
|
18588
17811
|
import Papa2 from "papaparse";
|
|
18589
17812
|
|
|
18590
17813
|
// src/youtube/schemas.ts
|
|
18591
|
-
import { z as
|
|
18592
|
-
var YouTubeHarvestOptionsSchema =
|
|
18593
|
-
mode:
|
|
18594
|
-
query:
|
|
18595
|
-
channelHandle:
|
|
18596
|
-
maxVideos:
|
|
18597
|
-
downloadMp3:
|
|
18598
|
-
outputDir:
|
|
18599
|
-
kernelApiKey:
|
|
18600
|
-
kernelProxyId:
|
|
18601
|
-
headless:
|
|
17814
|
+
import { z as z15 } from "zod";
|
|
17815
|
+
var YouTubeHarvestOptionsSchema = z15.object({
|
|
17816
|
+
mode: z15.enum(["search", "channel"]),
|
|
17817
|
+
query: z15.string().optional(),
|
|
17818
|
+
channelHandle: z15.string().optional(),
|
|
17819
|
+
maxVideos: z15.number().int().min(1).max(5e3).default(100),
|
|
17820
|
+
downloadMp3: z15.boolean().default(false),
|
|
17821
|
+
outputDir: z15.string().default("./yt-output"),
|
|
17822
|
+
kernelApiKey: z15.string().optional(),
|
|
17823
|
+
kernelProxyId: z15.string().optional(),
|
|
17824
|
+
headless: z15.boolean().default(false)
|
|
18602
17825
|
}).refine(
|
|
18603
17826
|
(o) => o.mode === "search" ? !!o.query : !!o.channelHandle,
|
|
18604
17827
|
{ message: "search mode requires query; channel mode requires channelHandle" }
|
|
@@ -19360,8 +18583,8 @@ function finalizeCaptions(result) {
|
|
|
19360
18583
|
};
|
|
19361
18584
|
});
|
|
19362
18585
|
const lastEnd = chunks.length ? chunks[chunks.length - 1].timestamp[1] : 0;
|
|
19363
|
-
const
|
|
19364
|
-
const wordEstSec =
|
|
18586
|
+
const words = result.text.split(/\s+/).filter(Boolean).length;
|
|
18587
|
+
const wordEstSec = words > 0 ? Math.ceil(words / 2.5) : 0;
|
|
19365
18588
|
const videoDurationMs = lastEnd > 1 ? Math.round(lastEnd * 1e3) : wordEstSec * 1e3;
|
|
19366
18589
|
return { ...result, chunks, durationMs: videoDurationMs };
|
|
19367
18590
|
}
|
|
@@ -19378,153 +18601,153 @@ async function fetchCaptions(videoId, language = "en") {
|
|
|
19378
18601
|
}
|
|
19379
18602
|
|
|
19380
18603
|
// src/api/server-schemas.ts
|
|
19381
|
-
import { z as
|
|
19382
|
-
var HarvestBodySchema =
|
|
19383
|
-
query:
|
|
19384
|
-
location:
|
|
19385
|
-
depth:
|
|
19386
|
-
maxQuestions:
|
|
19387
|
-
callback_url:
|
|
19388
|
-
gl:
|
|
19389
|
-
hl:
|
|
19390
|
-
device:
|
|
19391
|
-
serpIdentity:
|
|
19392
|
-
proxyMode:
|
|
19393
|
-
proxyZip:
|
|
19394
|
-
debug:
|
|
19395
|
-
serpOnly:
|
|
19396
|
-
pages:
|
|
19397
|
-
recency:
|
|
19398
|
-
includeAllSerpFeatures:
|
|
19399
|
-
includeLocalPack:
|
|
19400
|
-
includeForums:
|
|
19401
|
-
includeVideos:
|
|
19402
|
-
includeAiOverview:
|
|
19403
|
-
includeWhatPeopleSaying:
|
|
18604
|
+
import { z as z16 } from "zod";
|
|
18605
|
+
var HarvestBodySchema = z16.object({
|
|
18606
|
+
query: z16.string().min(1, "query is required"),
|
|
18607
|
+
location: z16.string().optional(),
|
|
18608
|
+
depth: z16.number().int().min(1).max(4).optional(),
|
|
18609
|
+
maxQuestions: z16.number().int().min(1).optional(),
|
|
18610
|
+
callback_url: z16.string().url().optional(),
|
|
18611
|
+
gl: z16.string().optional(),
|
|
18612
|
+
hl: z16.string().optional(),
|
|
18613
|
+
device: z16.enum(["desktop", "mobile"]).optional(),
|
|
18614
|
+
serpIdentity: z16.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional(),
|
|
18615
|
+
proxyMode: z16.enum(["configured", "none"]).optional(),
|
|
18616
|
+
proxyZip: z16.string().regex(/^\d{5}$/).optional(),
|
|
18617
|
+
debug: z16.boolean().optional(),
|
|
18618
|
+
serpOnly: z16.boolean().optional(),
|
|
18619
|
+
pages: z16.number().int().min(1).max(2).optional(),
|
|
18620
|
+
recency: z16.enum(["day", "week", "month", "year"]).optional(),
|
|
18621
|
+
includeAllSerpFeatures: z16.boolean().optional(),
|
|
18622
|
+
includeLocalPack: z16.boolean().optional(),
|
|
18623
|
+
includeForums: z16.boolean().optional(),
|
|
18624
|
+
includeVideos: z16.boolean().optional(),
|
|
18625
|
+
includeAiOverview: z16.boolean().optional(),
|
|
18626
|
+
includeWhatPeopleSaying: z16.boolean().optional()
|
|
19404
18627
|
});
|
|
19405
18628
|
var ExtractUrlBodyShape = {
|
|
19406
|
-
url:
|
|
19407
|
-
screenshot:
|
|
19408
|
-
screenshotDevice:
|
|
19409
|
-
extractBranding:
|
|
19410
|
-
includeFeaturedImage:
|
|
19411
|
-
downloadMedia:
|
|
19412
|
-
mediaTypes:
|
|
19413
|
-
maxMediaAssets:
|
|
19414
|
-
maxInlineImages:
|
|
19415
|
-
delivery:
|
|
19416
|
-
preserveMedia:
|
|
19417
|
-
depositToVault:
|
|
19418
|
-
vaultName:
|
|
18629
|
+
url: z16.string().min(1, "url is required"),
|
|
18630
|
+
screenshot: z16.boolean().optional(),
|
|
18631
|
+
screenshotDevice: z16.enum(["desktop", "mobile"]).optional(),
|
|
18632
|
+
extractBranding: z16.boolean().optional(),
|
|
18633
|
+
includeFeaturedImage: z16.boolean().optional(),
|
|
18634
|
+
downloadMedia: z16.boolean().optional(),
|
|
18635
|
+
mediaTypes: z16.array(z16.enum(["image", "video", "audio"])).optional(),
|
|
18636
|
+
maxMediaAssets: z16.number().int().min(1).max(250).optional(),
|
|
18637
|
+
maxInlineImages: z16.number().int().min(0).max(5).optional(),
|
|
18638
|
+
delivery: z16.enum(["auto", "inline", "artifact", "memory"]).optional(),
|
|
18639
|
+
preserveMedia: z16.boolean().optional(),
|
|
18640
|
+
depositToVault: z16.boolean().optional(),
|
|
18641
|
+
vaultName: z16.string().trim().min(1).max(120).optional()
|
|
19419
18642
|
};
|
|
19420
|
-
var ExtractUrlBodySchema =
|
|
19421
|
-
var ExtractUrlLocalBodySchema =
|
|
18643
|
+
var ExtractUrlBodySchema = z16.object(ExtractUrlBodyShape).strict();
|
|
18644
|
+
var ExtractUrlLocalBodySchema = z16.object({
|
|
19422
18645
|
...ExtractUrlBodyShape,
|
|
19423
|
-
allowLocal:
|
|
18646
|
+
allowLocal: z16.boolean().optional()
|
|
19424
18647
|
}).strict();
|
|
19425
18648
|
var DiffPageBodyShape = {
|
|
19426
|
-
url:
|
|
19427
|
-
resetBaseline:
|
|
18649
|
+
url: z16.string().min(1, "url is required"),
|
|
18650
|
+
resetBaseline: z16.boolean().optional()
|
|
19428
18651
|
};
|
|
19429
|
-
var DiffPageBodySchema =
|
|
19430
|
-
var DiffPageLocalBodySchema =
|
|
18652
|
+
var DiffPageBodySchema = z16.object(DiffPageBodyShape).strict();
|
|
18653
|
+
var DiffPageLocalBodySchema = z16.object({
|
|
19431
18654
|
...DiffPageBodyShape,
|
|
19432
|
-
allowLocal:
|
|
18655
|
+
allowLocal: z16.boolean().optional()
|
|
19433
18656
|
}).strict();
|
|
19434
|
-
var ArchiveReadBodySchema =
|
|
19435
|
-
artifactId:
|
|
19436
|
-
url:
|
|
19437
|
-
path:
|
|
19438
|
-
offset:
|
|
19439
|
-
maxBytes:
|
|
19440
|
-
maxEntries:
|
|
19441
|
-
depositToLibrary:
|
|
18657
|
+
var ArchiveReadBodySchema = z16.object({
|
|
18658
|
+
artifactId: z16.string().trim().min(1).optional(),
|
|
18659
|
+
url: z16.string().url("url must be a valid HTTPS ZIP URL").optional(),
|
|
18660
|
+
path: z16.string().trim().min(1).max(2e3).optional(),
|
|
18661
|
+
offset: z16.number().int().min(0).optional(),
|
|
18662
|
+
maxBytes: z16.number().int().min(1).max(2e5).optional(),
|
|
18663
|
+
maxEntries: z16.number().int().min(1).max(1e3).optional(),
|
|
18664
|
+
depositToLibrary: z16.boolean().optional()
|
|
19442
18665
|
}).superRefine((value, ctx) => {
|
|
19443
18666
|
if ((value.artifactId ? 1 : 0) + (value.url ? 1 : 0) !== 1) {
|
|
19444
18667
|
ctx.addIssue({
|
|
19445
|
-
code:
|
|
18668
|
+
code: z16.ZodIssueCode.custom,
|
|
19446
18669
|
path: ["artifactId"],
|
|
19447
18670
|
message: "Provide exactly one of artifactId or url"
|
|
19448
18671
|
});
|
|
19449
18672
|
}
|
|
19450
18673
|
if (value.depositToLibrary && !value.path) {
|
|
19451
18674
|
ctx.addIssue({
|
|
19452
|
-
code:
|
|
18675
|
+
code: z16.ZodIssueCode.custom,
|
|
19453
18676
|
path: ["path"],
|
|
19454
18677
|
message: "path is required when depositToLibrary is true"
|
|
19455
18678
|
});
|
|
19456
18679
|
}
|
|
19457
18680
|
});
|
|
19458
|
-
var MapUrlsBodySchema =
|
|
19459
|
-
url:
|
|
19460
|
-
maxUrls:
|
|
19461
|
-
concurrency:
|
|
19462
|
-
browserFallback:
|
|
19463
|
-
kernelFallback:
|
|
19464
|
-
delivery:
|
|
18681
|
+
var MapUrlsBodySchema = z16.object({
|
|
18682
|
+
url: z16.string().min(1, "url is required"),
|
|
18683
|
+
maxUrls: z16.number().int().min(1).max(2e3).optional(),
|
|
18684
|
+
concurrency: z16.number().int().min(1).max(20).optional(),
|
|
18685
|
+
browserFallback: z16.boolean().optional(),
|
|
18686
|
+
kernelFallback: z16.boolean().optional(),
|
|
18687
|
+
delivery: z16.enum(["auto", "inline", "artifact"]).optional()
|
|
19465
18688
|
});
|
|
19466
|
-
var WaybackInventoryBodySchema =
|
|
19467
|
-
url:
|
|
18689
|
+
var WaybackInventoryBodySchema = z16.object({
|
|
18690
|
+
url: z16.string().min(1, "url is required"),
|
|
19468
18691
|
...WaybackInventoryOptionsSchema
|
|
19469
18692
|
});
|
|
19470
|
-
var ExtractSiteBodySchema =
|
|
19471
|
-
url:
|
|
19472
|
-
maxPages:
|
|
18693
|
+
var ExtractSiteBodySchema = z16.object({
|
|
18694
|
+
url: z16.string().min(1, "url is required"),
|
|
18695
|
+
maxPages: z16.number().int().min(1).max(1e4).optional(),
|
|
19473
18696
|
wayback: WaybackTimelineSchema.optional(),
|
|
19474
|
-
browserFallback:
|
|
19475
|
-
kernelFallback:
|
|
19476
|
-
rotateProxies:
|
|
19477
|
-
rotateProxyEvery:
|
|
19478
|
-
background:
|
|
19479
|
-
downloadImages:
|
|
19480
|
-
preserveMedia:
|
|
19481
|
-
delivery:
|
|
19482
|
-
formats:
|
|
19483
|
-
renderJavaScript:
|
|
19484
|
-
captureRenderedDom:
|
|
19485
|
-
semanticSimilarity:
|
|
19486
|
-
similarityThreshold:
|
|
19487
|
-
similarityMaxPairs:
|
|
18697
|
+
browserFallback: z16.boolean().optional(),
|
|
18698
|
+
kernelFallback: z16.boolean().optional(),
|
|
18699
|
+
rotateProxies: z16.boolean().optional(),
|
|
18700
|
+
rotateProxyEvery: z16.number().int().min(1).max(100).optional(),
|
|
18701
|
+
background: z16.boolean().optional(),
|
|
18702
|
+
downloadImages: z16.boolean().optional(),
|
|
18703
|
+
preserveMedia: z16.boolean().optional(),
|
|
18704
|
+
delivery: z16.enum(["auto", "artifact"]).optional(),
|
|
18705
|
+
formats: z16.array(z16.enum(["markdown", "html", "links", "json", "images", "branding", "issues"])).optional(),
|
|
18706
|
+
renderJavaScript: z16.boolean().optional(),
|
|
18707
|
+
captureRenderedDom: z16.boolean().optional(),
|
|
18708
|
+
semanticSimilarity: z16.boolean().optional(),
|
|
18709
|
+
similarityThreshold: z16.number().min(0).max(1).optional(),
|
|
18710
|
+
similarityMaxPairs: z16.number().int().min(1).max(5e4).optional()
|
|
19488
18711
|
}).superRefine((value, ctx) => {
|
|
19489
18712
|
if (value.semanticSimilarity && (value.maxPages ?? 100) > 500) {
|
|
19490
18713
|
ctx.addIssue({
|
|
19491
|
-
code:
|
|
18714
|
+
code: z16.ZodIssueCode.custom,
|
|
19492
18715
|
path: ["maxPages"],
|
|
19493
18716
|
message: "semanticSimilarity supports at most 500 pages per analysis."
|
|
19494
18717
|
});
|
|
19495
18718
|
}
|
|
19496
18719
|
if (value.semanticSimilarity && value.wayback) {
|
|
19497
18720
|
ctx.addIssue({
|
|
19498
|
-
code:
|
|
18721
|
+
code: z16.ZodIssueCode.custom,
|
|
19499
18722
|
path: ["wayback"],
|
|
19500
18723
|
message: "semanticSimilarity analyzes one live rendered corpus and cannot be combined with a Wayback timeline."
|
|
19501
18724
|
});
|
|
19502
18725
|
}
|
|
19503
18726
|
});
|
|
19504
|
-
var SiteExportReadBodySchema =
|
|
19505
|
-
jobId:
|
|
19506
|
-
pageId:
|
|
19507
|
-
format:
|
|
19508
|
-
offset:
|
|
19509
|
-
maxBytes:
|
|
18727
|
+
var SiteExportReadBodySchema = z16.object({
|
|
18728
|
+
jobId: z16.string().trim().min(1),
|
|
18729
|
+
pageId: z16.string().regex(/^[a-f0-9]{64}$/).optional(),
|
|
18730
|
+
format: z16.enum(["manifest", "json", "html", "markdown"]).optional(),
|
|
18731
|
+
offset: z16.number().int().min(0).optional(),
|
|
18732
|
+
maxBytes: z16.number().int().min(1).max(1e6).optional()
|
|
19510
18733
|
}).superRefine((value, ctx) => {
|
|
19511
18734
|
if (value.format && value.format !== "manifest" && !value.pageId) {
|
|
19512
|
-
ctx.addIssue({ code:
|
|
18735
|
+
ctx.addIssue({ code: z16.ZodIssueCode.custom, path: ["pageId"], message: "pageId is required for page formats" });
|
|
19513
18736
|
}
|
|
19514
18737
|
});
|
|
19515
|
-
var SiteExportImageBodySchema =
|
|
19516
|
-
jobId:
|
|
19517
|
-
imageId:
|
|
18738
|
+
var SiteExportImageBodySchema = z16.object({
|
|
18739
|
+
jobId: z16.string().trim().min(1),
|
|
18740
|
+
imageId: z16.string().regex(/^[a-f0-9]{64}$/)
|
|
19518
18741
|
});
|
|
19519
|
-
var YoutubeHarvestBodySchema =
|
|
19520
|
-
mode:
|
|
19521
|
-
query:
|
|
19522
|
-
channelHandle:
|
|
19523
|
-
maxVideos:
|
|
18742
|
+
var YoutubeHarvestBodySchema = z16.object({
|
|
18743
|
+
mode: z16.enum(["search", "channel"]),
|
|
18744
|
+
query: z16.string().optional(),
|
|
18745
|
+
channelHandle: z16.string().optional(),
|
|
18746
|
+
maxVideos: z16.number().int().min(1).max(500).optional()
|
|
19524
18747
|
});
|
|
19525
|
-
var YoutubeTranscribeBodySchema =
|
|
19526
|
-
videoId:
|
|
19527
|
-
language:
|
|
18748
|
+
var YoutubeTranscribeBodySchema = z16.object({
|
|
18749
|
+
videoId: z16.string().min(1, "videoId is required"),
|
|
18750
|
+
language: z16.enum(WIZPER_LANGUAGES).optional()
|
|
19528
18751
|
});
|
|
19529
18752
|
|
|
19530
18753
|
// src/api/youtube-routes.ts
|
|
@@ -19701,11 +18924,11 @@ function fmtTs(secs) {
|
|
|
19701
18924
|
|
|
19702
18925
|
// src/api/screenshot-routes.ts
|
|
19703
18926
|
import { Hono as Hono6 } from "hono";
|
|
19704
|
-
import { z as
|
|
19705
|
-
var ScreenshotBodySchema =
|
|
19706
|
-
url:
|
|
19707
|
-
device:
|
|
19708
|
-
allowLocal:
|
|
18927
|
+
import { z as z17 } from "zod";
|
|
18928
|
+
var ScreenshotBodySchema = z17.object({
|
|
18929
|
+
url: z17.string().trim().min(1, "url is required"),
|
|
18930
|
+
device: z17.string().trim().optional(),
|
|
18931
|
+
allowLocal: z17.boolean().optional()
|
|
19709
18932
|
});
|
|
19710
18933
|
var screenshotApp = new Hono6();
|
|
19711
18934
|
screenshotApp.use("*", createApiKeyAuth());
|
|
@@ -19769,7 +18992,7 @@ screenshotApp.post("/", async (c) => {
|
|
|
19769
18992
|
|
|
19770
18993
|
// src/api/facebook-ad-routes.ts
|
|
19771
18994
|
import { Hono as Hono7 } from "hono";
|
|
19772
|
-
import { z as
|
|
18995
|
+
import { z as z18 } from "zod";
|
|
19773
18996
|
|
|
19774
18997
|
// src/extractor/FacebookAdExtractor.ts
|
|
19775
18998
|
var SPRITE_Y_TO_PLATFORM = {
|
|
@@ -20662,35 +19885,35 @@ async function extractFacebookOrganicVideoFromPage(page, sourceUrl, quality = "b
|
|
|
20662
19885
|
|
|
20663
19886
|
// src/api/facebook-ad-routes.ts
|
|
20664
19887
|
import { fal } from "@fal-ai/client";
|
|
20665
|
-
var FacebookAdBodySchema =
|
|
20666
|
-
url:
|
|
20667
|
-
libraryId:
|
|
20668
|
-
openModal:
|
|
19888
|
+
var FacebookAdBodySchema = z18.object({
|
|
19889
|
+
url: z18.string().trim().optional(),
|
|
19890
|
+
libraryId: z18.string().trim().optional(),
|
|
19891
|
+
openModal: z18.boolean().optional()
|
|
20669
19892
|
}).refine((d) => !!d.url || !!d.libraryId, { message: "url or libraryId is required" });
|
|
20670
|
-
var FacebookPageIntelBodySchema =
|
|
20671
|
-
pageId:
|
|
20672
|
-
query:
|
|
20673
|
-
libraryId:
|
|
20674
|
-
maxAds:
|
|
20675
|
-
country:
|
|
19893
|
+
var FacebookPageIntelBodySchema = z18.object({
|
|
19894
|
+
pageId: z18.string().trim().optional(),
|
|
19895
|
+
query: z18.string().trim().optional(),
|
|
19896
|
+
libraryId: z18.string().trim().optional(),
|
|
19897
|
+
maxAds: z18.number().int().min(1).max(200).optional(),
|
|
19898
|
+
country: z18.string().trim().toUpperCase().optional()
|
|
20676
19899
|
}).refine((d) => !!d.pageId || !!d.query || !!d.libraryId, {
|
|
20677
19900
|
message: "pageId, libraryId, or query is required"
|
|
20678
19901
|
});
|
|
20679
|
-
var FacebookTranscribeBodySchema =
|
|
20680
|
-
videoUrl:
|
|
19902
|
+
var FacebookTranscribeBodySchema = z18.object({
|
|
19903
|
+
videoUrl: z18.string().trim().min(1, "videoUrl is required")
|
|
20681
19904
|
});
|
|
20682
|
-
var FacebookVideoTranscribeBodySchema =
|
|
20683
|
-
url:
|
|
20684
|
-
quality:
|
|
19905
|
+
var FacebookVideoTranscribeBodySchema = z18.object({
|
|
19906
|
+
url: z18.string().trim().min(1, "url is required"),
|
|
19907
|
+
quality: z18.enum(["best", "hd", "sd"]).default("best")
|
|
20685
19908
|
});
|
|
20686
|
-
var FacebookSearchBodySchema =
|
|
20687
|
-
query:
|
|
20688
|
-
country:
|
|
20689
|
-
maxResults:
|
|
19909
|
+
var FacebookSearchBodySchema = z18.object({
|
|
19910
|
+
query: z18.string().trim().min(1, "query is required"),
|
|
19911
|
+
country: z18.string().trim().toUpperCase().optional(),
|
|
19912
|
+
maxResults: z18.number().int().min(1).max(20).optional()
|
|
20690
19913
|
});
|
|
20691
|
-
var FacebookMediaBodySchema =
|
|
20692
|
-
url:
|
|
20693
|
-
filename:
|
|
19914
|
+
var FacebookMediaBodySchema = z18.object({
|
|
19915
|
+
url: z18.string().trim().min(1, "url is required"),
|
|
19916
|
+
filename: z18.string().trim().optional()
|
|
20694
19917
|
});
|
|
20695
19918
|
function invalidRequest(message) {
|
|
20696
19919
|
return { error_code: "invalid_request", message };
|
|
@@ -21163,7 +20386,7 @@ facebookAdApp.post("/media", createApiKeyAuth(), async (c) => {
|
|
|
21163
20386
|
|
|
21164
20387
|
// src/api/google-ads-routes.ts
|
|
21165
20388
|
import { Hono as Hono8 } from "hono";
|
|
21166
|
-
import { z as
|
|
20389
|
+
import { z as z19 } from "zod";
|
|
21167
20390
|
|
|
21168
20391
|
// src/extractor/GoogleAdsExtractor.ts
|
|
21169
20392
|
var ORIGIN = "https://adstransparency.google.com";
|
|
@@ -21388,20 +20611,20 @@ var GoogleAdsExtractor = class {
|
|
|
21388
20611
|
|
|
21389
20612
|
// src/api/google-ads-routes.ts
|
|
21390
20613
|
import { fal as fal2 } from "@fal-ai/client";
|
|
21391
|
-
var GoogleAdsSearchBodySchema =
|
|
21392
|
-
query:
|
|
21393
|
-
region:
|
|
21394
|
-
maxResults:
|
|
21395
|
-
});
|
|
21396
|
-
var GoogleAdsPageIntelBodySchema =
|
|
21397
|
-
advertiserId:
|
|
21398
|
-
advertiserName:
|
|
21399
|
-
domain:
|
|
21400
|
-
region:
|
|
21401
|
-
maxAds:
|
|
20614
|
+
var GoogleAdsSearchBodySchema = z19.object({
|
|
20615
|
+
query: z19.string().trim().min(1, "query is required"),
|
|
20616
|
+
region: z19.string().trim().toUpperCase().length(2).optional(),
|
|
20617
|
+
maxResults: z19.number().int().min(1).max(20).optional()
|
|
20618
|
+
});
|
|
20619
|
+
var GoogleAdsPageIntelBodySchema = z19.object({
|
|
20620
|
+
advertiserId: z19.string().trim().optional(),
|
|
20621
|
+
advertiserName: z19.string().trim().optional(),
|
|
20622
|
+
domain: z19.string().trim().optional(),
|
|
20623
|
+
region: z19.string().trim().toUpperCase().length(2).optional(),
|
|
20624
|
+
maxAds: z19.number().int().min(1).max(200).optional()
|
|
21402
20625
|
}).refine((d) => !!d.advertiserId || !!d.domain || !!d.advertiserName, { message: "advertiserId, advertiserName, or domain is required (advertiserName and domain both come straight from google_ads_search)" });
|
|
21403
|
-
var GoogleAdsTranscribeBodySchema =
|
|
21404
|
-
videoUrl:
|
|
20626
|
+
var GoogleAdsTranscribeBodySchema = z19.object({
|
|
20627
|
+
videoUrl: z19.string().trim().min(1, "videoUrl is required")
|
|
21405
20628
|
});
|
|
21406
20629
|
function invalidRequest2(message) {
|
|
21407
20630
|
return { error_code: "invalid_request", message };
|
|
@@ -21959,7 +21182,7 @@ adminCreditsApp.get("/lookup", async (c) => {
|
|
|
21959
21182
|
|
|
21960
21183
|
// src/api/instagram-routes.ts
|
|
21961
21184
|
import { Hono as Hono10 } from "hono";
|
|
21962
|
-
import { z as
|
|
21185
|
+
import { z as z20 } from "zod";
|
|
21963
21186
|
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
21964
21187
|
import { tmpdir } from "os";
|
|
21965
21188
|
import { join as join5 } from "path";
|
|
@@ -22187,8 +21410,8 @@ function parseCount(text2) {
|
|
|
22187
21410
|
function captionFromMetadata(metadata) {
|
|
22188
21411
|
const raw = metadata.ogDescription ?? metadata.description;
|
|
22189
21412
|
if (!raw) return null;
|
|
22190
|
-
const
|
|
22191
|
-
if (
|
|
21413
|
+
const quoted = raw.match(/:\s*"([\s\S]+)"\.?\s*$/);
|
|
21414
|
+
if (quoted?.[1]) return quoted[1].trim();
|
|
22192
21415
|
return raw.trim();
|
|
22193
21416
|
}
|
|
22194
21417
|
function ownerFromMetadata(metadata) {
|
|
@@ -22409,30 +21632,30 @@ async function extractInstagramMediaFromPage(page, sourceUrl, capturedMediaUrls
|
|
|
22409
21632
|
}
|
|
22410
21633
|
|
|
22411
21634
|
// src/api/instagram-routes.ts
|
|
22412
|
-
var InstagramProfileContentBodySchema =
|
|
22413
|
-
handle:
|
|
22414
|
-
url:
|
|
22415
|
-
profile:
|
|
22416
|
-
saveProfileChanges:
|
|
22417
|
-
save_profile_changes:
|
|
22418
|
-
maxItems:
|
|
22419
|
-
maxScrolls:
|
|
22420
|
-
scrollDelayMs:
|
|
22421
|
-
stableScrollLimit:
|
|
21635
|
+
var InstagramProfileContentBodySchema = z20.object({
|
|
21636
|
+
handle: z20.string().trim().optional(),
|
|
21637
|
+
url: z20.string().trim().optional(),
|
|
21638
|
+
profile: z20.string().trim().min(1).optional(),
|
|
21639
|
+
saveProfileChanges: z20.boolean().optional(),
|
|
21640
|
+
save_profile_changes: z20.boolean().optional(),
|
|
21641
|
+
maxItems: z20.number().int().min(1).max(2e3).default(50),
|
|
21642
|
+
maxScrolls: z20.number().int().min(0).max(250).default(10),
|
|
21643
|
+
scrollDelayMs: z20.number().int().min(250).max(5e3).default(1200),
|
|
21644
|
+
stableScrollLimit: z20.number().int().min(1).max(10).default(4)
|
|
22422
21645
|
}).refine((d) => !!d.handle || !!d.url, {
|
|
22423
21646
|
message: "handle or url is required"
|
|
22424
21647
|
});
|
|
22425
|
-
var InstagramMediaTypeSchema =
|
|
22426
|
-
var InstagramMediaDownloadBodySchema =
|
|
22427
|
-
url:
|
|
22428
|
-
profile:
|
|
22429
|
-
saveProfileChanges:
|
|
22430
|
-
save_profile_changes:
|
|
22431
|
-
mediaTypes:
|
|
22432
|
-
downloadMedia:
|
|
22433
|
-
downloadAllTracks:
|
|
22434
|
-
includeTranscript:
|
|
22435
|
-
mux:
|
|
21648
|
+
var InstagramMediaTypeSchema = z20.enum(["image", "video", "audio"]);
|
|
21649
|
+
var InstagramMediaDownloadBodySchema = z20.object({
|
|
21650
|
+
url: z20.string().trim().min(1, "url is required"),
|
|
21651
|
+
profile: z20.string().trim().min(1).optional(),
|
|
21652
|
+
saveProfileChanges: z20.boolean().optional(),
|
|
21653
|
+
save_profile_changes: z20.boolean().optional(),
|
|
21654
|
+
mediaTypes: z20.array(InstagramMediaTypeSchema).default(["image", "video", "audio"]),
|
|
21655
|
+
downloadMedia: z20.boolean().default(true),
|
|
21656
|
+
downloadAllTracks: z20.boolean().default(false),
|
|
21657
|
+
includeTranscript: z20.boolean().default(false),
|
|
21658
|
+
mux: z20.boolean().default(true)
|
|
22436
21659
|
});
|
|
22437
21660
|
var instagramApp = new Hono10();
|
|
22438
21661
|
function invalidRequest3(message) {
|
|
@@ -22732,7 +21955,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
22732
21955
|
|
|
22733
21956
|
// src/api/reddit-routes.ts
|
|
22734
21957
|
import { Hono as Hono11 } from "hono";
|
|
22735
|
-
import { z as
|
|
21958
|
+
import { z as z21 } from "zod";
|
|
22736
21959
|
|
|
22737
21960
|
// src/api/reddit-trending.ts
|
|
22738
21961
|
var PARSE_REDDIT_SEARCH = `(() => {
|
|
@@ -22962,17 +22185,17 @@ function flattenQuestions(threads) {
|
|
|
22962
22185
|
}
|
|
22963
22186
|
|
|
22964
22187
|
// src/api/reddit-routes.ts
|
|
22965
|
-
var RedditThreadBodySchema =
|
|
22966
|
-
url:
|
|
22967
|
-
maxComments:
|
|
22968
|
-
});
|
|
22969
|
-
var RedditTrendingBodySchema =
|
|
22970
|
-
topic:
|
|
22971
|
-
subreddit:
|
|
22972
|
-
window:
|
|
22973
|
-
maxThreads:
|
|
22974
|
-
includeComments:
|
|
22975
|
-
maxCommentsPerThread:
|
|
22188
|
+
var RedditThreadBodySchema = z21.object({
|
|
22189
|
+
url: z21.string().trim().min(1, "url is required"),
|
|
22190
|
+
maxComments: z21.number().int().min(1).max(2e3).optional()
|
|
22191
|
+
});
|
|
22192
|
+
var RedditTrendingBodySchema = z21.object({
|
|
22193
|
+
topic: z21.string().trim().min(1, "topic is required"),
|
|
22194
|
+
subreddit: z21.string().trim().regex(/^(r\/)?[A-Za-z0-9_]{2,24}$/i, "subreddit must be a bare name like SEO").optional(),
|
|
22195
|
+
window: z21.enum(["week", "month", "all"]).default("month"),
|
|
22196
|
+
maxThreads: z21.number().int().min(1).max(40).default(20),
|
|
22197
|
+
includeComments: z21.boolean().default(true),
|
|
22198
|
+
maxCommentsPerThread: z21.number().int().min(1).max(200).default(50)
|
|
22976
22199
|
});
|
|
22977
22200
|
function invalidRequest4(message) {
|
|
22978
22201
|
return { error_code: "invalid_request", message };
|
|
@@ -23454,14 +22677,14 @@ redditApp.post("/trending", createApiKeyAuth(), async (c) => {
|
|
|
23454
22677
|
|
|
23455
22678
|
// src/api/video-routes.ts
|
|
23456
22679
|
import { Hono as Hono12 } from "hono";
|
|
23457
|
-
import { z as
|
|
22680
|
+
import { z as z22 } from "zod";
|
|
23458
22681
|
var videoApp = new Hono12();
|
|
23459
|
-
var AnalyzeBodySchema =
|
|
23460
|
-
sourceUrl:
|
|
23461
|
-
intervalS:
|
|
23462
|
-
maxFrames:
|
|
23463
|
-
detail:
|
|
23464
|
-
vault:
|
|
22682
|
+
var AnalyzeBodySchema = z22.object({
|
|
22683
|
+
sourceUrl: z22.string().trim().url("sourceUrl must be a direct video file URL"),
|
|
22684
|
+
intervalS: z22.number().min(1).max(30).optional(),
|
|
22685
|
+
maxFrames: z22.number().int().min(1).max(480).optional(),
|
|
22686
|
+
detail: z22.enum(["fast", "standard", "deep"]).optional(),
|
|
22687
|
+
vault: z22.string().trim().min(1).optional()
|
|
23465
22688
|
});
|
|
23466
22689
|
function invalidRequest5(message) {
|
|
23467
22690
|
return { error_code: "invalid_request", message };
|
|
@@ -23949,10 +23172,10 @@ mapsApp.post("/place", createApiKeyAuth(), async (c) => {
|
|
|
23949
23172
|
|
|
23950
23173
|
// src/api/trustpilot-routes.ts
|
|
23951
23174
|
import { Hono as Hono14 } from "hono";
|
|
23952
|
-
import { z as
|
|
23953
|
-
var TrustpilotReviewsBodySchema =
|
|
23954
|
-
domain:
|
|
23955
|
-
maxPages:
|
|
23175
|
+
import { z as z23 } from "zod";
|
|
23176
|
+
var TrustpilotReviewsBodySchema = z23.object({
|
|
23177
|
+
domain: z23.string().trim().min(1, "domain is required"),
|
|
23178
|
+
maxPages: z23.number().int().min(1).max(50).default(5)
|
|
23956
23179
|
});
|
|
23957
23180
|
function invalidRequest6(message) {
|
|
23958
23181
|
return { error_code: "invalid_request", message };
|
|
@@ -24111,10 +23334,10 @@ trustpilotApp.post("/reviews", createApiKeyAuth(), async (c) => {
|
|
|
24111
23334
|
|
|
24112
23335
|
// src/api/g2-routes.ts
|
|
24113
23336
|
import { Hono as Hono15 } from "hono";
|
|
24114
|
-
import { z as
|
|
24115
|
-
var G2ReviewsBodySchema =
|
|
24116
|
-
product:
|
|
24117
|
-
maxPages:
|
|
23337
|
+
import { z as z24 } from "zod";
|
|
23338
|
+
var G2ReviewsBodySchema = z24.object({
|
|
23339
|
+
product: z24.string().trim().min(1, "product is required"),
|
|
23340
|
+
maxPages: z24.number().int().min(1).max(50).default(5)
|
|
24118
23341
|
});
|
|
24119
23342
|
function invalidRequest7(message) {
|
|
24120
23343
|
return { error_code: "invalid_request", message };
|
|
@@ -25910,7 +25133,7 @@ leadListInputApp.post("/imports", async (c) => {
|
|
|
25910
25133
|
// src/api/lead-list-enrichment-routes.ts
|
|
25911
25134
|
import { createHash as createHash9, randomUUID as randomUUID12 } from "crypto";
|
|
25912
25135
|
import { Hono as Hono18 } from "hono";
|
|
25913
|
-
import { z as
|
|
25136
|
+
import { z as z25 } from "zod";
|
|
25914
25137
|
|
|
25915
25138
|
// src/api/lead-list-enrichment-dispatch.ts
|
|
25916
25139
|
import { randomUUID as randomUUID11 } from "crypto";
|
|
@@ -25939,15 +25162,15 @@ async function dispatchPendingLeadListEnrichments(limit = 25) {
|
|
|
25939
25162
|
}
|
|
25940
25163
|
|
|
25941
25164
|
// src/api/lead-list-enrichment-routes.ts
|
|
25942
|
-
var SourceSchema =
|
|
25943
|
-
|
|
25944
|
-
|
|
25165
|
+
var SourceSchema = z25.discriminatedUnion("kind", [
|
|
25166
|
+
z25.object({ kind: z25.literal("rows"), rows: LeadListRowsSchema }).strict(),
|
|
25167
|
+
z25.object({ kind: z25.literal("lead_list"), leadListId: z25.string().trim().regex(/^lst_[A-Za-z0-9]{8,80}$/) }).strict()
|
|
25945
25168
|
]);
|
|
25946
|
-
var StartSchema =
|
|
25169
|
+
var StartSchema = z25.object({
|
|
25947
25170
|
source: SourceSchema,
|
|
25948
25171
|
columnMap: LeadColumnMapSchema,
|
|
25949
25172
|
...LeadListEnrichmentOptionsSchema.shape,
|
|
25950
|
-
outputFormats:
|
|
25173
|
+
outputFormats: z25.array(z25.enum(["csv", "xlsx"])).min(1).max(2).refine((values) => new Set(values).size === values.length, "outputFormats must be unique").default(["csv", "xlsx"])
|
|
25951
25174
|
}).strict();
|
|
25952
25175
|
var TERMINAL = /* @__PURE__ */ new Set([
|
|
25953
25176
|
"complete",
|
|
@@ -26211,7 +25434,7 @@ leadListEnrichmentApp.post("/jobs", createApiKeyAuth(), async (c) => {
|
|
|
26211
25434
|
if (!gate.ok) {
|
|
26212
25435
|
return c.json(concurrencyLimitExceededResponse(gate), 429, { "Retry-After": String(gate.retryAfterSeconds) });
|
|
26213
25436
|
}
|
|
26214
|
-
const holdMc = input.rows.length * ((input.options.resolveMissingUrls ? MC_COSTS.maps_search : 0) + input.options.maxPagesPerLead * MC_COSTS.page_scrape + (input.options.emailSearchFallback === "serp_snippets" ? MC_COSTS.serp_headful : 0) + (input.options.peopleDiscovery === "off" ? 0 : (1 + input.options.maxPeoplePerLead) * MC_COSTS.serp_headful));
|
|
25437
|
+
const holdMc = input.rows.length * ((input.options.resolveMissingUrls ? MC_COSTS.maps_search : 0) + input.options.maxPagesPerLead * MC_COSTS.page_scrape + (input.options.emailSearchFallback === "serp_snippets" ? MC_COSTS.serp_headful : 0) + (input.options.peopleDiscovery === "off" ? 0 : ((input.options.peopleQueryTemplates?.length ?? 1) + input.options.maxPeoplePerLead) * MC_COSTS.serp_headful));
|
|
26215
25438
|
let creation;
|
|
26216
25439
|
try {
|
|
26217
25440
|
creation = await createOrGetLeadListEnrichmentJob({
|
|
@@ -26343,7 +25566,7 @@ leadListEnrichmentApp.post("/jobs/:jobId/cancel", createApiKeyAuth(), async (c)
|
|
|
26343
25566
|
|
|
26344
25567
|
// src/api/local-sourcebook-routes.ts
|
|
26345
25568
|
import { Hono as Hono19 } from "hono";
|
|
26346
|
-
import { z as
|
|
25569
|
+
import { z as z27 } from "zod";
|
|
26347
25570
|
|
|
26348
25571
|
// src/api/local-sourcebook-dispatch.ts
|
|
26349
25572
|
async function dispatchLocalSourcebookAcquisition(submissionId) {
|
|
@@ -26372,7 +25595,7 @@ async function redispatchQueuedLocalSourcebookAcquisitions(limit = 25) {
|
|
|
26372
25595
|
}
|
|
26373
25596
|
|
|
26374
25597
|
// src/api/local-sourcebook-governance.ts
|
|
26375
|
-
import { z as
|
|
25598
|
+
import { z as z26 } from "zod";
|
|
26376
25599
|
var DIRECTORY_TAG_SEEDS = [
|
|
26377
25600
|
{ tag: "home-services", description: "Businesses that maintain, repair, improve, or protect homes and property.", aliases: ["home service", "home and property services"] },
|
|
26378
25601
|
{ tag: "professional-services", description: "Local professional and business service providers.", aliases: ["professional service"] },
|
|
@@ -26397,28 +25620,28 @@ var CATEGORY_TAG = {
|
|
|
26397
25620
|
auto: "automotive",
|
|
26398
25621
|
wellness: "wellness"
|
|
26399
25622
|
};
|
|
26400
|
-
var DirectoryTagCandidateSchema =
|
|
26401
|
-
tag:
|
|
26402
|
-
central:
|
|
26403
|
-
reusable:
|
|
26404
|
-
description:
|
|
26405
|
-
});
|
|
26406
|
-
var DirectoryTagDecisionSchema =
|
|
26407
|
-
tag:
|
|
26408
|
-
central:
|
|
26409
|
-
reusable:
|
|
26410
|
-
description:
|
|
26411
|
-
acceptCanonical:
|
|
26412
|
-
});
|
|
26413
|
-
var DirectoryIdentitySchema =
|
|
26414
|
-
category:
|
|
26415
|
-
state:
|
|
26416
|
-
businessName:
|
|
26417
|
-
websiteUrl:
|
|
25623
|
+
var DirectoryTagCandidateSchema = z26.object({
|
|
25624
|
+
tag: z26.string().trim().min(1).max(60),
|
|
25625
|
+
central: z26.boolean().optional(),
|
|
25626
|
+
reusable: z26.boolean().optional(),
|
|
25627
|
+
description: z26.string().trim().min(8).max(240).optional()
|
|
25628
|
+
});
|
|
25629
|
+
var DirectoryTagDecisionSchema = z26.object({
|
|
25630
|
+
tag: z26.string().trim().min(1).max(60),
|
|
25631
|
+
central: z26.boolean(),
|
|
25632
|
+
reusable: z26.boolean(),
|
|
25633
|
+
description: z26.string().trim().min(8).max(240).optional(),
|
|
25634
|
+
acceptCanonical: z26.string().trim().min(1).max(60).optional()
|
|
25635
|
+
});
|
|
25636
|
+
var DirectoryIdentitySchema = z26.object({
|
|
25637
|
+
category: z26.enum(LOCAL_SOURCEBOOK_CATEGORIES),
|
|
25638
|
+
state: z26.string().trim().length(2).transform((value) => value.toLowerCase()),
|
|
25639
|
+
businessName: z26.string().trim().min(2).max(160),
|
|
25640
|
+
websiteUrl: z26.url().refine((value) => /^https?:\/\//.test(value), "websiteUrl must use http or https"),
|
|
26418
25641
|
schemaOrgType: LocalSourcebookSchemaTypeSchema.optional(),
|
|
26419
|
-
slug:
|
|
26420
|
-
tags:
|
|
26421
|
-
idempotencyKey:
|
|
25642
|
+
slug: z26.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(100).optional(),
|
|
25643
|
+
tags: z26.array(z26.string().trim().min(1).max(60)).max(20).default([]),
|
|
25644
|
+
idempotencyKey: z26.string().trim().min(8).max(200)
|
|
26422
25645
|
});
|
|
26423
25646
|
function normalizeTag(value) {
|
|
26424
25647
|
return value.toLowerCase().trim().replace(/&/g, " and ").replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60);
|
|
@@ -26603,24 +25826,24 @@ async function persistLocalSourcebookTags(submissionId, userId, tags, decisions
|
|
|
26603
25826
|
|
|
26604
25827
|
// src/api/local-sourcebook-routes.ts
|
|
26605
25828
|
var slug2 = (value) => value.toLowerCase().trim().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 100);
|
|
26606
|
-
var SubmitSchema =
|
|
26607
|
-
category:
|
|
26608
|
-
state:
|
|
26609
|
-
businessName:
|
|
26610
|
-
websiteUrl:
|
|
25829
|
+
var SubmitSchema = z27.object({
|
|
25830
|
+
category: z27.enum(LOCAL_SOURCEBOOK_CATEGORIES),
|
|
25831
|
+
state: z27.string().trim().length(2).transform((value) => value.toLowerCase()),
|
|
25832
|
+
businessName: z27.string().trim().min(2).max(160),
|
|
25833
|
+
websiteUrl: z27.url().refine((value) => /^https?:\/\//.test(value), "websiteUrl must use http or https"),
|
|
26611
25834
|
schemaOrgType: LocalSourcebookSchemaTypeSchema.optional(),
|
|
26612
|
-
slug:
|
|
26613
|
-
idempotencyKey:
|
|
25835
|
+
slug: z27.string().trim().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(100).optional(),
|
|
25836
|
+
idempotencyKey: z27.string().trim().min(8).max(200).optional()
|
|
26614
25837
|
}).strict();
|
|
26615
|
-
var PrepareWriteSchema = DirectoryIdentitySchema.extend({ tagCandidates:
|
|
26616
|
-
var ValidateWriteSchema =
|
|
25838
|
+
var PrepareWriteSchema = DirectoryIdentitySchema.extend({ tagCandidates: z27.array(DirectoryTagCandidateSchema).max(20).optional() });
|
|
25839
|
+
var ValidateWriteSchema = z27.object({
|
|
26617
25840
|
identity: DirectoryIdentitySchema.optional(),
|
|
26618
|
-
tagCandidates:
|
|
26619
|
-
tagDecisions:
|
|
25841
|
+
tagCandidates: z27.array(DirectoryTagCandidateSchema).max(20).optional(),
|
|
25842
|
+
tagDecisions: z27.array(DirectoryTagDecisionSchema).max(20).optional()
|
|
26620
25843
|
}).strict();
|
|
26621
|
-
var CaptureWriteSchema = ValidateWriteSchema.extend({ idempotencyKey:
|
|
25844
|
+
var CaptureWriteSchema = ValidateWriteSchema.extend({ idempotencyKey: z27.string().trim().min(8).max(200).optional() });
|
|
26622
25845
|
function withCanonicalLocalSourcebookUrls(listing) {
|
|
26623
|
-
const category =
|
|
25846
|
+
const category = z27.enum(LOCAL_SOURCEBOOK_CATEGORIES).safeParse(listing.category);
|
|
26624
25847
|
const state = typeof listing.state === "string" ? listing.state : "";
|
|
26625
25848
|
const listingSlug = typeof listing.slug === "string" ? listing.slug : "";
|
|
26626
25849
|
if (!category.success || !state || !listingSlug) return listing;
|
|
@@ -26642,12 +25865,12 @@ function acquisitionPlan(websiteUrl) {
|
|
|
26642
25865
|
var localSourcebookApp = new Hono19();
|
|
26643
25866
|
localSourcebookApp.get("/contract", createApiKeyAuth(), (c) => c.json({ ok: true, contract: getLocalSourcebookContract() }));
|
|
26644
25867
|
localSourcebookApp.get("/contract/:category", createApiKeyAuth(), (c) => {
|
|
26645
|
-
const category =
|
|
25868
|
+
const category = z27.enum(LOCAL_SOURCEBOOK_CATEGORIES).safeParse(c.req.param("category"));
|
|
26646
25869
|
return category.success ? c.json({ ok: true, contract: getLocalSourcebookContract(category.data) }) : c.json({ error: "invalid_category" }, 400);
|
|
26647
25870
|
});
|
|
26648
25871
|
localSourcebookApp.get("/tags", createApiKeyAuth(), async (c) => c.json({ ok: true, tags: await listLocalSourcebookTags(c.req.query("includeDeprecated") !== "false") }));
|
|
26649
25872
|
localSourcebookApp.post("/tags/resolve", createApiKeyAuth(), async (c) => {
|
|
26650
|
-
const parsed =
|
|
25873
|
+
const parsed = z27.object({ candidates: z27.array(DirectoryTagCandidateSchema).min(1).max(20) }).strict().safeParse(await c.req.json().catch(() => null));
|
|
26651
25874
|
return parsed.success ? c.json({ ok: true, resolutions: await resolveLocalSourcebookTags(parsed.data.candidates) }) : c.json({ error: "invalid_tag_candidates", issues: parsed.error.issues }, 400);
|
|
26652
25875
|
});
|
|
26653
25876
|
localSourcebookApp.post("/prepare", createApiKeyAuth(), async (c) => {
|
|
@@ -26834,25 +26057,25 @@ adminLocalSourcebookApp.post("/:id/reject", async (c) => {
|
|
|
26834
26057
|
|
|
26835
26058
|
// src/api/location-data-routes.ts
|
|
26836
26059
|
import { Hono as Hono20 } from "hono";
|
|
26837
|
-
import { z as
|
|
26060
|
+
import { z as z28 } from "zod";
|
|
26838
26061
|
var LOCATION_IMPORT_MAX_BYTES = 64 * 1024 * 1024;
|
|
26839
26062
|
var LOCATION_IMPORT_TIMEOUT_MS = 6e4;
|
|
26840
|
-
var marketQuerySchema =
|
|
26841
|
-
state:
|
|
26842
|
-
city:
|
|
26843
|
-
zip:
|
|
26844
|
-
minPopulation:
|
|
26845
|
-
populationYear:
|
|
26063
|
+
var marketQuerySchema = z28.object({
|
|
26064
|
+
state: z28.string().trim().min(2).max(40).default("TN"),
|
|
26065
|
+
city: z28.string().trim().min(1).max(120).optional(),
|
|
26066
|
+
zip: z28.string().regex(/^\d{5}$/).optional(),
|
|
26067
|
+
minPopulation: z28.coerce.number().int().min(0).max(1e8).default(0),
|
|
26068
|
+
populationYear: z28.coerce.number().int().refine(
|
|
26846
26069
|
(value) => POPULATION_YEARS.includes(value),
|
|
26847
26070
|
"populationYear must be between 2020 and 2025"
|
|
26848
26071
|
).default(2025),
|
|
26849
|
-
maxResults:
|
|
26850
|
-
includeZipGroups:
|
|
26072
|
+
maxResults: z28.coerce.number().int().min(1).max(100).default(25),
|
|
26073
|
+
includeZipGroups: z28.enum(["true", "false"]).default("true").transform((value) => value === "true")
|
|
26851
26074
|
});
|
|
26852
|
-
var importSchema =
|
|
26853
|
-
sourceUrl:
|
|
26075
|
+
var importSchema = z28.object({
|
|
26076
|
+
sourceUrl: z28.string().url().max(2e3).optional()
|
|
26854
26077
|
});
|
|
26855
|
-
var censusSyncSchema =
|
|
26078
|
+
var censusSyncSchema = z28.object({ state: z28.string().trim().min(2).max(40) });
|
|
26856
26079
|
var locationDataApp = new Hono20();
|
|
26857
26080
|
function locationAdminAuthorized(header) {
|
|
26858
26081
|
const expected = process.env.MCP_SCRAPER_LOCATION_ADMIN_KEY?.trim();
|
|
@@ -27257,34 +26480,34 @@ async function reconcileDirectoryWorkflowSettlements(limit = 25) {
|
|
|
27257
26480
|
import { createHmac as createHmac2 } from "crypto";
|
|
27258
26481
|
import { readFile as readFile2 } from "fs/promises";
|
|
27259
26482
|
import { Hono as Hono21 } from "hono";
|
|
27260
|
-
import { z as
|
|
26483
|
+
import { z as z29 } from "zod";
|
|
27261
26484
|
var workflowApp = new Hono21();
|
|
27262
|
-
var WorkflowInputSchema =
|
|
27263
|
-
var WorkflowIdSchema =
|
|
27264
|
-
var CadenceSchema =
|
|
27265
|
-
var ScheduleStatusSchema =
|
|
27266
|
-
var RunBodySchema =
|
|
26485
|
+
var WorkflowInputSchema = z29.record(z29.string(), z29.unknown()).default({});
|
|
26486
|
+
var WorkflowIdSchema = z29.string().min(1);
|
|
26487
|
+
var CadenceSchema = z29.enum(["daily", "weekly", "monthly"]);
|
|
26488
|
+
var ScheduleStatusSchema = z29.enum(["active", "paused"]);
|
|
26489
|
+
var RunBodySchema = z29.object({
|
|
27267
26490
|
workflowId: WorkflowIdSchema,
|
|
27268
26491
|
input: WorkflowInputSchema,
|
|
27269
|
-
webhookUrl:
|
|
26492
|
+
webhookUrl: z29.string().url().optional()
|
|
27270
26493
|
});
|
|
27271
|
-
var ScheduleCreateSchema =
|
|
26494
|
+
var ScheduleCreateSchema = z29.object({
|
|
27272
26495
|
workflowId: WorkflowIdSchema,
|
|
27273
|
-
name:
|
|
26496
|
+
name: z29.string().min(1).max(120).optional(),
|
|
27274
26497
|
input: WorkflowInputSchema,
|
|
27275
26498
|
cadence: CadenceSchema.default("weekly"),
|
|
27276
|
-
timezone:
|
|
27277
|
-
webhookUrl:
|
|
27278
|
-
nextRunAt:
|
|
26499
|
+
timezone: z29.string().min(1).max(64).default("UTC"),
|
|
26500
|
+
webhookUrl: z29.string().url().optional(),
|
|
26501
|
+
nextRunAt: z29.string().datetime().optional()
|
|
27279
26502
|
});
|
|
27280
|
-
var SchedulePatchSchema =
|
|
27281
|
-
name:
|
|
26503
|
+
var SchedulePatchSchema = z29.object({
|
|
26504
|
+
name: z29.string().min(1).max(120).optional(),
|
|
27282
26505
|
status: ScheduleStatusSchema.optional(),
|
|
27283
26506
|
input: WorkflowInputSchema.optional(),
|
|
27284
26507
|
cadence: CadenceSchema.optional(),
|
|
27285
|
-
timezone:
|
|
27286
|
-
webhookUrl:
|
|
27287
|
-
nextRunAt:
|
|
26508
|
+
timezone: z29.string().min(1).max(64).optional(),
|
|
26509
|
+
webhookUrl: z29.string().url().nullable().optional(),
|
|
26510
|
+
nextRunAt: z29.string().datetime().nullable().optional()
|
|
27288
26511
|
});
|
|
27289
26512
|
function hostedWorkflowOutputDir() {
|
|
27290
26513
|
return workflowOutputBaseDir(process.env.MCP_SCRAPER_WORKFLOW_OUTPUT_DIR?.trim() || "/tmp/mcp-scraper-workflows");
|
|
@@ -28101,7 +27324,7 @@ async function capturePageSnapshots(targets, options = {}) {
|
|
|
28101
27324
|
}
|
|
28102
27325
|
|
|
28103
27326
|
// src/serp-intelligence/schemas.ts
|
|
28104
|
-
import { z as
|
|
27327
|
+
import { z as z30 } from "zod";
|
|
28105
27328
|
var SerpIntelligenceDeviceValues = ["desktop", "mobile"];
|
|
28106
27329
|
var SerpIntelligenceProxyModeValues = ["location", "configured", "none"];
|
|
28107
27330
|
var SerpIntelligenceAttemptOutcomeValues = [
|
|
@@ -28116,6 +27339,7 @@ var SerpIntelligenceAttemptOutcomeValues = [
|
|
|
28116
27339
|
"location_mismatch",
|
|
28117
27340
|
"proxy_tunnel_failed",
|
|
28118
27341
|
"proxy_unavailable",
|
|
27342
|
+
"browser_session_interrupted",
|
|
28119
27343
|
"mcp_unavailable",
|
|
28120
27344
|
"error"
|
|
28121
27345
|
];
|
|
@@ -28164,181 +27388,181 @@ function isPublicHttpUrl(value) {
|
|
|
28164
27388
|
return false;
|
|
28165
27389
|
}
|
|
28166
27390
|
}
|
|
28167
|
-
var SerpIntelligencePublicHttpUrlSchema =
|
|
28168
|
-
var SerpIntelligenceCaptureBodySchema =
|
|
28169
|
-
query:
|
|
28170
|
-
location:
|
|
28171
|
-
gl:
|
|
28172
|
-
hl:
|
|
28173
|
-
device:
|
|
28174
|
-
proxyMode:
|
|
28175
|
-
proxyZip:
|
|
28176
|
-
pages:
|
|
28177
|
-
debug:
|
|
28178
|
-
includePageSnapshots:
|
|
28179
|
-
pageSnapshotLimit:
|
|
27391
|
+
var SerpIntelligencePublicHttpUrlSchema = z30.string().url().refine(isPublicHttpUrl, "url must be a public HTTP or HTTPS URL");
|
|
27392
|
+
var SerpIntelligenceCaptureBodySchema = z30.object({
|
|
27393
|
+
query: z30.string().trim().min(1, "query is required"),
|
|
27394
|
+
location: z30.string().trim().min(1).optional(),
|
|
27395
|
+
gl: z30.string().trim().length(2).default("us"),
|
|
27396
|
+
hl: z30.string().trim().length(2).default("en"),
|
|
27397
|
+
device: z30.enum(SerpIntelligenceDeviceValues).default("desktop"),
|
|
27398
|
+
proxyMode: z30.enum(SerpIntelligenceProxyModeValues).default(DEFAULT_PROXY_MODE),
|
|
27399
|
+
proxyZip: z30.string().regex(/^\d{5}$/).optional(),
|
|
27400
|
+
pages: z30.number().int().min(1).max(2).default(1),
|
|
27401
|
+
debug: z30.boolean().default(false),
|
|
27402
|
+
includePageSnapshots: z30.boolean().default(false),
|
|
27403
|
+
pageSnapshotLimit: z30.number().int().min(0).max(10).default(0)
|
|
28180
27404
|
}).strict();
|
|
28181
|
-
var SerpIntelligencePageSnapshotRequestSchema =
|
|
27405
|
+
var SerpIntelligencePageSnapshotRequestSchema = z30.object({
|
|
28182
27406
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
28183
|
-
sourceKind:
|
|
28184
|
-
sourcePosition:
|
|
27407
|
+
sourceKind: z30.enum(SerpPageSnapshotSourceKindValues).default("configured_target"),
|
|
27408
|
+
sourcePosition: z30.number().int().min(1).optional()
|
|
28185
27409
|
}).strict();
|
|
28186
|
-
var SerpIntelligencePageSnapshotsBodySchema =
|
|
28187
|
-
urls:
|
|
28188
|
-
targets:
|
|
28189
|
-
maxConcurrency:
|
|
28190
|
-
timeoutMs:
|
|
28191
|
-
debug:
|
|
27410
|
+
var SerpIntelligencePageSnapshotsBodySchema = z30.object({
|
|
27411
|
+
urls: z30.array(SerpIntelligencePublicHttpUrlSchema).min(1).max(25).optional(),
|
|
27412
|
+
targets: z30.array(SerpIntelligencePageSnapshotRequestSchema).min(1).max(25).optional(),
|
|
27413
|
+
maxConcurrency: z30.number().int().min(1).max(5).default(2),
|
|
27414
|
+
timeoutMs: z30.number().int().min(1e3).max(6e4).default(15e3),
|
|
27415
|
+
debug: z30.boolean().default(false)
|
|
28192
27416
|
}).strict().superRefine((body, ctx) => {
|
|
28193
27417
|
const hasUrls = Boolean(body.urls?.length);
|
|
28194
27418
|
const hasTargets = Boolean(body.targets?.length);
|
|
28195
27419
|
if (hasUrls === hasTargets) {
|
|
28196
27420
|
ctx.addIssue({
|
|
28197
|
-
code:
|
|
27421
|
+
code: z30.ZodIssueCode.custom,
|
|
28198
27422
|
message: "provide exactly one of urls or targets",
|
|
28199
27423
|
path: hasUrls ? ["targets"] : ["urls"]
|
|
28200
27424
|
});
|
|
28201
27425
|
}
|
|
28202
27426
|
});
|
|
28203
|
-
var SerpIntelligenceAICitationSchema =
|
|
28204
|
-
text:
|
|
28205
|
-
href:
|
|
27427
|
+
var SerpIntelligenceAICitationSchema = z30.object({
|
|
27428
|
+
text: z30.string(),
|
|
27429
|
+
href: z30.string()
|
|
28206
27430
|
}).strict();
|
|
28207
|
-
var SerpIntelligenceOrganicResultSchema =
|
|
28208
|
-
position:
|
|
28209
|
-
title:
|
|
28210
|
-
url:
|
|
28211
|
-
domain:
|
|
28212
|
-
cite:
|
|
28213
|
-
snippet:
|
|
28214
|
-
isRedditStyle:
|
|
28215
|
-
inlineRating:
|
|
28216
|
-
value:
|
|
28217
|
-
count:
|
|
27431
|
+
var SerpIntelligenceOrganicResultSchema = z30.object({
|
|
27432
|
+
position: z30.number().int().min(1),
|
|
27433
|
+
title: z30.string(),
|
|
27434
|
+
url: z30.string(),
|
|
27435
|
+
domain: z30.string(),
|
|
27436
|
+
cite: z30.string().nullable(),
|
|
27437
|
+
snippet: z30.string().nullable(),
|
|
27438
|
+
isRedditStyle: z30.boolean(),
|
|
27439
|
+
inlineRating: z30.object({
|
|
27440
|
+
value: z30.string(),
|
|
27441
|
+
count: z30.string()
|
|
28218
27442
|
}).strict().nullable()
|
|
28219
27443
|
}).strict();
|
|
28220
|
-
var SerpIntelligenceLocationEvidenceSchema =
|
|
28221
|
-
status:
|
|
28222
|
-
expected:
|
|
28223
|
-
city:
|
|
28224
|
-
regionCode:
|
|
28225
|
-
canonicalLocation:
|
|
27444
|
+
var SerpIntelligenceLocationEvidenceSchema = z30.object({
|
|
27445
|
+
status: z30.enum(SerpIntelligenceLocalizationStatusValues),
|
|
27446
|
+
expected: z30.object({
|
|
27447
|
+
city: z30.string(),
|
|
27448
|
+
regionCode: z30.string().nullable(),
|
|
27449
|
+
canonicalLocation: z30.string()
|
|
28226
27450
|
}).strict().nullable(),
|
|
28227
|
-
candidates:
|
|
28228
|
-
city:
|
|
28229
|
-
regionCode:
|
|
28230
|
-
count:
|
|
28231
|
-
examples:
|
|
27451
|
+
candidates: z30.array(z30.object({
|
|
27452
|
+
city: z30.string(),
|
|
27453
|
+
regionCode: z30.string(),
|
|
27454
|
+
count: z30.number().int().min(0),
|
|
27455
|
+
examples: z30.array(z30.string())
|
|
28232
27456
|
}).strict())
|
|
28233
27457
|
}).strict();
|
|
28234
|
-
var SerpIntelligenceHarvestResultSchema =
|
|
28235
|
-
seed:
|
|
28236
|
-
location:
|
|
28237
|
-
extractedAt:
|
|
28238
|
-
totalQuestions:
|
|
28239
|
-
surface:
|
|
28240
|
-
aiOverview:
|
|
28241
|
-
detected:
|
|
28242
|
-
text:
|
|
28243
|
-
citations:
|
|
28244
|
-
expanded:
|
|
28245
|
-
fullyExpanded:
|
|
28246
|
-
sections:
|
|
27458
|
+
var SerpIntelligenceHarvestResultSchema = z30.object({
|
|
27459
|
+
seed: z30.string(),
|
|
27460
|
+
location: z30.string().nullable(),
|
|
27461
|
+
extractedAt: z30.string(),
|
|
27462
|
+
totalQuestions: z30.number().int().min(0),
|
|
27463
|
+
surface: z30.enum(["web", "aim", "unknown"]),
|
|
27464
|
+
aiOverview: z30.object({
|
|
27465
|
+
detected: z30.boolean(),
|
|
27466
|
+
text: z30.string().nullable(),
|
|
27467
|
+
citations: z30.array(SerpIntelligenceAICitationSchema),
|
|
27468
|
+
expanded: z30.boolean().optional(),
|
|
27469
|
+
fullyExpanded: z30.boolean().optional(),
|
|
27470
|
+
sections: z30.array(z30.string()).optional()
|
|
28247
27471
|
}).strict(),
|
|
28248
|
-
aiMode:
|
|
28249
|
-
detected:
|
|
28250
|
-
text:
|
|
28251
|
-
citations:
|
|
27472
|
+
aiMode: z30.object({
|
|
27473
|
+
detected: z30.boolean(),
|
|
27474
|
+
text: z30.string().nullable(),
|
|
27475
|
+
citations: z30.array(SerpIntelligenceAICitationSchema)
|
|
28252
27476
|
}).strict(),
|
|
28253
|
-
tree:
|
|
28254
|
-
flat:
|
|
28255
|
-
videos:
|
|
28256
|
-
forums:
|
|
28257
|
-
organicResults:
|
|
28258
|
-
localPack:
|
|
28259
|
-
entityIds:
|
|
28260
|
-
entities:
|
|
28261
|
-
name:
|
|
28262
|
-
kgId:
|
|
28263
|
-
cid:
|
|
28264
|
-
gcid:
|
|
27477
|
+
tree: z30.array(z30.unknown()),
|
|
27478
|
+
flat: z30.array(z30.unknown()),
|
|
27479
|
+
videos: z30.array(z30.unknown()),
|
|
27480
|
+
forums: z30.array(z30.unknown()),
|
|
27481
|
+
organicResults: z30.array(SerpIntelligenceOrganicResultSchema),
|
|
27482
|
+
localPack: z30.array(z30.unknown()),
|
|
27483
|
+
entityIds: z30.object({
|
|
27484
|
+
entities: z30.array(z30.object({
|
|
27485
|
+
name: z30.string(),
|
|
27486
|
+
kgId: z30.string().nullable(),
|
|
27487
|
+
cid: z30.string().nullable(),
|
|
27488
|
+
gcid: z30.string().nullable()
|
|
28265
27489
|
}).strict()),
|
|
28266
|
-
kgIds:
|
|
28267
|
-
cids:
|
|
28268
|
-
gcids:
|
|
27490
|
+
kgIds: z30.array(z30.string()),
|
|
27491
|
+
cids: z30.array(z30.string()),
|
|
27492
|
+
gcids: z30.array(z30.string())
|
|
28269
27493
|
}).strict(),
|
|
28270
|
-
stats:
|
|
28271
|
-
seed:
|
|
28272
|
-
totalQuestions:
|
|
28273
|
-
maxDepthReached:
|
|
28274
|
-
durationMs:
|
|
28275
|
-
errorCount:
|
|
27494
|
+
stats: z30.object({
|
|
27495
|
+
seed: z30.string(),
|
|
27496
|
+
totalQuestions: z30.number().int().min(0),
|
|
27497
|
+
maxDepthReached: z30.number().int().min(0),
|
|
27498
|
+
durationMs: z30.number().min(0),
|
|
27499
|
+
errorCount: z30.number().int().min(0)
|
|
28276
27500
|
}).strict(),
|
|
28277
|
-
diagnostics:
|
|
28278
|
-
completionStatus:
|
|
28279
|
-
problem:
|
|
28280
|
-
warnings:
|
|
28281
|
-
debug:
|
|
27501
|
+
diagnostics: z30.object({
|
|
27502
|
+
completionStatus: z30.enum(["paa_found", "no_paa", "serp_only"]),
|
|
27503
|
+
problem: z30.null(),
|
|
27504
|
+
warnings: z30.array(z30.unknown()).optional(),
|
|
27505
|
+
debug: z30.object({
|
|
28282
27506
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.optional()
|
|
28283
27507
|
}).passthrough().optional()
|
|
28284
27508
|
}).passthrough(),
|
|
28285
|
-
whatPeopleSaying:
|
|
27509
|
+
whatPeopleSaying: z30.array(z30.unknown())
|
|
28286
27510
|
}).strict();
|
|
28287
|
-
var SerpIntelligenceCaptureAttemptSchema =
|
|
28288
|
-
attemptNumber:
|
|
28289
|
-
outcome:
|
|
28290
|
-
startedAt:
|
|
28291
|
-
completedAt:
|
|
28292
|
-
durationMs:
|
|
28293
|
-
problemCode:
|
|
28294
|
-
message:
|
|
28295
|
-
kernelSessionId:
|
|
28296
|
-
cleanupSucceeded:
|
|
27511
|
+
var SerpIntelligenceCaptureAttemptSchema = z30.object({
|
|
27512
|
+
attemptNumber: z30.number().int().min(1),
|
|
27513
|
+
outcome: z30.enum(SerpIntelligenceAttemptOutcomeValues),
|
|
27514
|
+
startedAt: z30.string().optional(),
|
|
27515
|
+
completedAt: z30.string().optional(),
|
|
27516
|
+
durationMs: z30.number().min(0).optional(),
|
|
27517
|
+
problemCode: z30.string().optional(),
|
|
27518
|
+
message: z30.string().optional(),
|
|
27519
|
+
kernelSessionId: z30.string().nullable().optional(),
|
|
27520
|
+
cleanupSucceeded: z30.boolean().nullable().optional()
|
|
28297
27521
|
}).strict();
|
|
28298
|
-
var SerpPageSnapshotCaptureSchema =
|
|
27522
|
+
var SerpPageSnapshotCaptureSchema = z30.object({
|
|
28299
27523
|
url: SerpIntelligencePublicHttpUrlSchema,
|
|
28300
27524
|
requestedUrl: SerpIntelligencePublicHttpUrlSchema,
|
|
28301
27525
|
finalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
28302
|
-
sourceKind:
|
|
28303
|
-
sourcePosition:
|
|
28304
|
-
status:
|
|
28305
|
-
fetchedVia:
|
|
28306
|
-
httpStatus:
|
|
28307
|
-
contentType:
|
|
28308
|
-
title:
|
|
27526
|
+
sourceKind: z30.enum(SerpPageSnapshotSourceKindValues),
|
|
27527
|
+
sourcePosition: z30.number().int().min(1).nullable(),
|
|
27528
|
+
status: z30.enum(SerpPageFetchStatusValues),
|
|
27529
|
+
fetchedVia: z30.enum(SerpPageFetchedViaValues).nullable(),
|
|
27530
|
+
httpStatus: z30.number().int().min(100).max(599).nullable(),
|
|
27531
|
+
contentType: z30.string().nullable(),
|
|
27532
|
+
title: z30.string().nullable(),
|
|
28309
27533
|
canonicalUrl: SerpIntelligencePublicHttpUrlSchema.nullable(),
|
|
28310
|
-
metaDescription:
|
|
28311
|
-
headings:
|
|
28312
|
-
level:
|
|
28313
|
-
text:
|
|
27534
|
+
metaDescription: z30.string().nullable(),
|
|
27535
|
+
headings: z30.array(z30.object({
|
|
27536
|
+
level: z30.number().int().min(1).max(6),
|
|
27537
|
+
text: z30.string()
|
|
28314
27538
|
}).strict()).default([]),
|
|
28315
|
-
artifact:
|
|
28316
|
-
htmlBlobUrl:
|
|
28317
|
-
textBlobUrl:
|
|
28318
|
-
markdownBlobUrl:
|
|
28319
|
-
screenshotBlobUrl:
|
|
28320
|
-
contentSha256:
|
|
28321
|
-
capturedAt:
|
|
27539
|
+
artifact: z30.object({
|
|
27540
|
+
htmlBlobUrl: z30.string().url().nullable(),
|
|
27541
|
+
textBlobUrl: z30.string().url().nullable(),
|
|
27542
|
+
markdownBlobUrl: z30.string().url().nullable(),
|
|
27543
|
+
screenshotBlobUrl: z30.string().url().nullable(),
|
|
27544
|
+
contentSha256: z30.string().nullable(),
|
|
27545
|
+
capturedAt: z30.string().nullable()
|
|
28322
27546
|
}).strict(),
|
|
28323
|
-
error:
|
|
28324
|
-
code:
|
|
28325
|
-
message:
|
|
27547
|
+
error: z30.object({
|
|
27548
|
+
code: z30.string(),
|
|
27549
|
+
message: z30.string()
|
|
28326
27550
|
}).strict().nullable()
|
|
28327
27551
|
}).strict();
|
|
28328
|
-
var SerpIntelligenceCaptureResponseSchema =
|
|
27552
|
+
var SerpIntelligenceCaptureResponseSchema = z30.object({
|
|
28329
27553
|
harvestResult: SerpIntelligenceHarvestResultSchema,
|
|
28330
|
-
attempts:
|
|
27554
|
+
attempts: z30.array(SerpIntelligenceCaptureAttemptSchema),
|
|
28331
27555
|
locationEvidence: SerpIntelligenceLocationEvidenceSchema.nullable(),
|
|
28332
|
-
pageSnapshotArtifacts:
|
|
28333
|
-
billing:
|
|
28334
|
-
creditsUsed:
|
|
28335
|
-
requestId:
|
|
28336
|
-
jobId:
|
|
27556
|
+
pageSnapshotArtifacts: z30.array(SerpPageSnapshotCaptureSchema),
|
|
27557
|
+
billing: z30.object({
|
|
27558
|
+
creditsUsed: z30.number().min(0).optional(),
|
|
27559
|
+
requestId: z30.string().optional(),
|
|
27560
|
+
jobId: z30.string().optional()
|
|
28337
27561
|
}).strict().optional()
|
|
28338
27562
|
}).strict();
|
|
28339
|
-
var SerpIntelligencePageSnapshotsResponseSchema =
|
|
28340
|
-
pageSnapshotArtifacts:
|
|
28341
|
-
attempts:
|
|
27563
|
+
var SerpIntelligencePageSnapshotsResponseSchema = z30.object({
|
|
27564
|
+
pageSnapshotArtifacts: z30.array(SerpPageSnapshotCaptureSchema),
|
|
27565
|
+
attempts: z30.array(SerpIntelligenceCaptureAttemptSchema).default([])
|
|
28342
27566
|
}).strict();
|
|
28343
27567
|
|
|
28344
27568
|
// src/serp-intelligence/serp-capture-service.ts
|
|
@@ -30146,7 +29370,7 @@ var FanoutCdpCapture = class {
|
|
|
30146
29370
|
// src/services/fanout/parse-helpers.ts
|
|
30147
29371
|
var INTERNAL = /chatgpt\.com|openai\.com|oaiusercontent|oaistatic|claude\.ai|anthropic\.com/i;
|
|
30148
29372
|
var TRACK = /^(utm_|oai|fbclid|gclid|ref$|ref_|sa$|usg$|ved$|sca_esv$|sources$)/i;
|
|
30149
|
-
function
|
|
29373
|
+
function domainOf(u) {
|
|
30150
29374
|
try {
|
|
30151
29375
|
return new URL(u).hostname.replace(/^www\./, "");
|
|
30152
29376
|
} catch {
|
|
@@ -30166,7 +29390,7 @@ function normUrl(u) {
|
|
|
30166
29390
|
}
|
|
30167
29391
|
}
|
|
30168
29392
|
function isExternal(u) {
|
|
30169
|
-
const h =
|
|
29393
|
+
const h = domainOf(u);
|
|
30170
29394
|
return Boolean(h) && !INTERNAL.test(h);
|
|
30171
29395
|
}
|
|
30172
29396
|
function unesc(s) {
|
|
@@ -30340,7 +29564,7 @@ var chatgptAdapter = {
|
|
|
30340
29564
|
const k = normUrl(url);
|
|
30341
29565
|
let s = sources.get(k);
|
|
30342
29566
|
if (!s) {
|
|
30343
|
-
s = { url, domain:
|
|
29567
|
+
s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, firstOrder: Infinity, snippet: "", round: null };
|
|
30344
29568
|
sources.set(k, s);
|
|
30345
29569
|
}
|
|
30346
29570
|
return s;
|
|
@@ -30348,7 +29572,7 @@ var chatgptAdapter = {
|
|
|
30348
29572
|
const addSnippet = (url, title, text2) => {
|
|
30349
29573
|
const clean2 = stripCitationMarkers(text2);
|
|
30350
29574
|
if (!isExternal(url) || clean2.length < 2) return;
|
|
30351
|
-
snippets.set(normUrl(url), { url, domain:
|
|
29575
|
+
snippets.set(normUrl(url), { url, domain: domainOf(url), title: title || domainOf(url), text: clean2.slice(0, 400) });
|
|
30352
29576
|
};
|
|
30353
29577
|
for (const m of raw.matchAll(/"([a-z_]*(?:quer|search)[a-z_]*)"\s*:/gi)) unmapped.add(m[1]);
|
|
30354
29578
|
for (const m of raw.matchAll(/"search_model_queries"\s*:\s*\[([^\]]*)\]/g)) {
|
|
@@ -30481,7 +29705,7 @@ var claudeAdapter = {
|
|
|
30481
29705
|
const k = normUrl(url);
|
|
30482
29706
|
let s = sources.get(k);
|
|
30483
29707
|
if (!s) {
|
|
30484
|
-
s = { url, domain:
|
|
29708
|
+
s = { url, domain: domainOf(url), title: "", cited: false, timesCited: 0, snippet: "", round: null };
|
|
30485
29709
|
sources.set(k, s);
|
|
30486
29710
|
}
|
|
30487
29711
|
return s;
|
|
@@ -30538,7 +29762,7 @@ var claudeAdapter = {
|
|
|
30538
29762
|
src.timesCited++;
|
|
30539
29763
|
if (!src.title && typeof c.title === "string") src.title = c.title.slice(0, 200);
|
|
30540
29764
|
if (typeof c.cited_text === "string" && c.cited_text) {
|
|
30541
|
-
snippets.set(normUrl(c.url), { url: c.url, domain:
|
|
29765
|
+
snippets.set(normUrl(c.url), { url: c.url, domain: domainOf(c.url), title: src.title, text: String(c.cited_text).slice(0, 400) });
|
|
30542
29766
|
}
|
|
30543
29767
|
}
|
|
30544
29768
|
}
|
|
@@ -35058,7 +34282,7 @@ resendInboundApp.post("/webhooks", async (c) => {
|
|
|
35058
34282
|
// src/api/workshop-registration-routes.ts
|
|
35059
34283
|
import { Hono as Hono31 } from "hono";
|
|
35060
34284
|
import { Resend as Resend2 } from "resend";
|
|
35061
|
-
import { z as
|
|
34285
|
+
import { z as z31 } from "zod";
|
|
35062
34286
|
|
|
35063
34287
|
// src/email/workshop-field-notes.ts
|
|
35064
34288
|
var WORKSHOP_FIELD_NOTES_URL = "https://mcp-field-notes.vercel.app/";
|
|
@@ -35442,11 +34666,11 @@ async function updateWorkshopRegistration(id, patch) {
|
|
|
35442
34666
|
|
|
35443
34667
|
// src/api/workshop-registration-routes.ts
|
|
35444
34668
|
var EVENT_SLUG = "aug-webinar";
|
|
35445
|
-
var registrationSchema =
|
|
35446
|
-
firstName:
|
|
35447
|
-
lastName:
|
|
35448
|
-
email:
|
|
35449
|
-
companyWebsite:
|
|
34669
|
+
var registrationSchema = z31.object({
|
|
34670
|
+
firstName: z31.string().trim().min(1).max(64),
|
|
34671
|
+
lastName: z31.string().trim().max(64).optional().default(""),
|
|
34672
|
+
email: z31.string().trim().email().max(320).transform((value) => value.toLowerCase()),
|
|
34673
|
+
companyWebsite: z31.string().max(200).optional().default("")
|
|
35450
34674
|
});
|
|
35451
34675
|
var workshopRegistrationApp = new Hono31();
|
|
35452
34676
|
function eventConfig() {
|
|
@@ -35842,7 +35066,7 @@ workshopRegistrationApp.post("/:event/register", async (c) => {
|
|
|
35842
35066
|
|
|
35843
35067
|
// src/api/editorial-reading-room-routes.ts
|
|
35844
35068
|
import { Hono as Hono32 } from "hono";
|
|
35845
|
-
import { z as
|
|
35069
|
+
import { z as z32 } from "zod";
|
|
35846
35070
|
|
|
35847
35071
|
// src/editorial-reading-room/render.ts
|
|
35848
35072
|
import { createHash as createHash15 } from "crypto";
|
|
@@ -36128,11 +35352,11 @@ function renewEditorialReadingRoomArtifactDownload(args) {
|
|
|
36128
35352
|
}
|
|
36129
35353
|
|
|
36130
35354
|
// src/api/editorial-reading-room-routes.ts
|
|
36131
|
-
var CreateRequestSchema =
|
|
35355
|
+
var CreateRequestSchema = z32.object({
|
|
36132
35356
|
...CreateEditorialReadingRoomInputSchema,
|
|
36133
|
-
delivery:
|
|
35357
|
+
delivery: z32.enum(["artifact", "inline"]).default("artifact")
|
|
36134
35358
|
}).strict();
|
|
36135
|
-
var RenewRequestSchema =
|
|
35359
|
+
var RenewRequestSchema = z32.object(RenewEditorialReadingRoomDownloadInputSchema).strict();
|
|
36136
35360
|
var editorialReadingRoomApp = new Hono32();
|
|
36137
35361
|
editorialReadingRoomApp.use("*", createApiKeyAuth());
|
|
36138
35362
|
editorialReadingRoomApp.post("/create", async (c) => {
|
|
@@ -36209,7 +35433,7 @@ editorialReadingRoomApp.post("/renew-download", async (c) => {
|
|
|
36209
35433
|
|
|
36210
35434
|
// src/api/commons-routes.ts
|
|
36211
35435
|
import { Hono as Hono33 } from "hono";
|
|
36212
|
-
import { z as
|
|
35436
|
+
import { z as z33 } from "zod";
|
|
36213
35437
|
|
|
36214
35438
|
// src/api/commons-repository.ts
|
|
36215
35439
|
import { createHash as createHash18, randomUUID as randomUUID20 } from "crypto";
|
|
@@ -39516,135 +38740,135 @@ function addPublicMetadata(html, canonicalUrl, publicationTitle) {
|
|
|
39516
38740
|
// src/api/commons-routes.ts
|
|
39517
38741
|
var commonsApp = new Hono33();
|
|
39518
38742
|
var auth2 = createApiKeyAuth();
|
|
39519
|
-
var SearchBodySchema =
|
|
39520
|
-
mode:
|
|
39521
|
-
query:
|
|
39522
|
-
entityType:
|
|
39523
|
-
tag:
|
|
39524
|
-
tags:
|
|
39525
|
-
keyword:
|
|
39526
|
-
keywords:
|
|
39527
|
-
relatedEntityId:
|
|
39528
|
-
trailId:
|
|
39529
|
-
sourceDomain:
|
|
39530
|
-
hasMedia:
|
|
39531
|
-
hasVideo:
|
|
39532
|
-
publishedAfter:
|
|
39533
|
-
updatedAfter:
|
|
39534
|
-
filterId:
|
|
39535
|
-
limit:
|
|
39536
|
-
offset:
|
|
38743
|
+
var SearchBodySchema = z33.object({
|
|
38744
|
+
mode: z33.enum(["lexical", "semantic", "hybrid"]).optional(),
|
|
38745
|
+
query: z33.string().max(300).optional(),
|
|
38746
|
+
entityType: z33.string().max(120).optional(),
|
|
38747
|
+
tag: z33.string().max(80).optional(),
|
|
38748
|
+
tags: z33.array(z33.string().max(80)).max(20).optional(),
|
|
38749
|
+
keyword: z33.string().max(120).optional(),
|
|
38750
|
+
keywords: z33.array(z33.string().max(120)).max(20).optional(),
|
|
38751
|
+
relatedEntityId: z33.string().max(80).optional(),
|
|
38752
|
+
trailId: z33.string().max(80).optional(),
|
|
38753
|
+
sourceDomain: z33.string().max(240).optional(),
|
|
38754
|
+
hasMedia: z33.boolean().optional(),
|
|
38755
|
+
hasVideo: z33.boolean().optional(),
|
|
38756
|
+
publishedAfter: z33.string().max(80).optional(),
|
|
38757
|
+
updatedAfter: z33.string().max(80).optional(),
|
|
38758
|
+
filterId: z33.string().max(200).optional(),
|
|
38759
|
+
limit: z33.number().int().min(1).max(100).optional(),
|
|
38760
|
+
offset: z33.number().int().min(0).max(1e4).optional()
|
|
39537
38761
|
}).strict();
|
|
39538
|
-
var FeaturedImageSchema =
|
|
39539
|
-
url:
|
|
39540
|
-
alt:
|
|
39541
|
-
caption:
|
|
39542
|
-
sourceUrl:
|
|
39543
|
-
license:
|
|
39544
|
-
width:
|
|
39545
|
-
height:
|
|
38762
|
+
var FeaturedImageSchema = z33.object({
|
|
38763
|
+
url: z33.string().url(),
|
|
38764
|
+
alt: z33.string().max(500).optional(),
|
|
38765
|
+
caption: z33.string().max(1e3).optional(),
|
|
38766
|
+
sourceUrl: z33.string().url().optional(),
|
|
38767
|
+
license: z33.string().max(240).optional(),
|
|
38768
|
+
width: z33.number().int().positive().optional(),
|
|
38769
|
+
height: z33.number().int().positive().optional()
|
|
39546
38770
|
}).strict();
|
|
39547
|
-
var MediaSchema =
|
|
39548
|
-
type:
|
|
39549
|
-
url:
|
|
39550
|
-
alt:
|
|
39551
|
-
caption:
|
|
39552
|
-
posterUrl:
|
|
39553
|
-
sourceUrl:
|
|
39554
|
-
license:
|
|
39555
|
-
width:
|
|
39556
|
-
height:
|
|
39557
|
-
durationSeconds:
|
|
38771
|
+
var MediaSchema = z33.object({
|
|
38772
|
+
type: z33.enum(["image", "video", "audio"]),
|
|
38773
|
+
url: z33.string().url(),
|
|
38774
|
+
alt: z33.string().max(500).optional(),
|
|
38775
|
+
caption: z33.string().max(1e3).optional(),
|
|
38776
|
+
posterUrl: z33.string().url().optional(),
|
|
38777
|
+
sourceUrl: z33.string().url().optional(),
|
|
38778
|
+
license: z33.string().max(240).optional(),
|
|
38779
|
+
width: z33.number().int().positive().optional(),
|
|
38780
|
+
height: z33.number().int().positive().optional(),
|
|
38781
|
+
durationSeconds: z33.number().int().positive().optional()
|
|
39558
38782
|
}).strict();
|
|
39559
|
-
var CitationSchema =
|
|
39560
|
-
title:
|
|
39561
|
-
url:
|
|
39562
|
-
source:
|
|
39563
|
-
note:
|
|
39564
|
-
accessedAt:
|
|
38783
|
+
var CitationSchema = z33.object({
|
|
38784
|
+
title: z33.string().min(1).max(240),
|
|
38785
|
+
url: z33.string().url().optional(),
|
|
38786
|
+
source: z33.string().max(240).optional(),
|
|
38787
|
+
note: z33.string().max(1e3).optional(),
|
|
38788
|
+
accessedAt: z33.string().max(80).optional()
|
|
39565
38789
|
}).strict();
|
|
39566
|
-
var RelatedLinkSchema =
|
|
39567
|
-
title:
|
|
39568
|
-
url:
|
|
39569
|
-
slug:
|
|
39570
|
-
entityId:
|
|
39571
|
-
relationship:
|
|
39572
|
-
summary:
|
|
39573
|
-
imageUrl:
|
|
39574
|
-
needsLink:
|
|
38790
|
+
var RelatedLinkSchema = z33.object({
|
|
38791
|
+
title: z33.string().min(1).max(180),
|
|
38792
|
+
url: z33.string().max(2e3).optional(),
|
|
38793
|
+
slug: z33.string().max(180).optional(),
|
|
38794
|
+
entityId: z33.string().max(80).optional(),
|
|
38795
|
+
relationship: z33.string().max(120).optional(),
|
|
38796
|
+
summary: z33.string().max(300).optional(),
|
|
38797
|
+
imageUrl: z33.string().max(2e3).optional(),
|
|
38798
|
+
needsLink: z33.boolean().optional()
|
|
39575
38799
|
}).strict();
|
|
39576
|
-
var SubmitEntitySchema =
|
|
39577
|
-
idempotencyKey:
|
|
39578
|
-
title:
|
|
39579
|
-
description:
|
|
39580
|
-
slug:
|
|
39581
|
-
entityId:
|
|
39582
|
-
entityType:
|
|
39583
|
-
disambiguationName:
|
|
38800
|
+
var SubmitEntitySchema = z33.object({
|
|
38801
|
+
idempotencyKey: z33.string().min(8).max(200),
|
|
38802
|
+
title: z33.string().min(3).max(180),
|
|
38803
|
+
description: z33.string().min(8).max(600),
|
|
38804
|
+
slug: z33.string().max(180).optional(),
|
|
38805
|
+
entityId: z33.string().max(80).optional(),
|
|
38806
|
+
entityType: z33.string().max(120).optional(),
|
|
38807
|
+
disambiguationName: z33.string().max(240).optional(),
|
|
39584
38808
|
featuredImage: FeaturedImageSchema.optional(),
|
|
39585
|
-
source:
|
|
39586
|
-
originalUrl:
|
|
39587
|
-
resolvedUrl:
|
|
39588
|
-
sourceCanonicalUrl:
|
|
39589
|
-
relCanonicalHref:
|
|
39590
|
-
sourceByline:
|
|
39591
|
-
publisher:
|
|
39592
|
-
authors:
|
|
39593
|
-
publishedAt:
|
|
39594
|
-
capturedAt:
|
|
39595
|
-
license:
|
|
39596
|
-
rightsSummary:
|
|
39597
|
-
contentHash:
|
|
38809
|
+
source: z33.object({
|
|
38810
|
+
originalUrl: z33.string().url().optional(),
|
|
38811
|
+
resolvedUrl: z33.string().url().optional(),
|
|
38812
|
+
sourceCanonicalUrl: z33.string().url().optional(),
|
|
38813
|
+
relCanonicalHref: z33.string().url().optional(),
|
|
38814
|
+
sourceByline: z33.string().max(240).optional(),
|
|
38815
|
+
publisher: z33.string().max(240).optional(),
|
|
38816
|
+
authors: z33.array(z33.string().max(120)).max(20).optional(),
|
|
38817
|
+
publishedAt: z33.string().max(80).optional(),
|
|
38818
|
+
capturedAt: z33.string().max(80).optional(),
|
|
38819
|
+
license: z33.string().max(240).optional(),
|
|
38820
|
+
rightsSummary: z33.string().max(1e3).optional(),
|
|
38821
|
+
contentHash: z33.string().max(200).optional()
|
|
39598
38822
|
}).strict().optional(),
|
|
39599
|
-
tags:
|
|
39600
|
-
keywords:
|
|
39601
|
-
relatedEntities:
|
|
39602
|
-
entityId:
|
|
39603
|
-
title:
|
|
39604
|
-
relationship:
|
|
39605
|
-
slug:
|
|
39606
|
-
url:
|
|
39607
|
-
description:
|
|
38823
|
+
tags: z33.array(z33.string().max(80)).max(24).optional(),
|
|
38824
|
+
keywords: z33.array(z33.string().max(120)).max(50).optional(),
|
|
38825
|
+
relatedEntities: z33.array(z33.object({
|
|
38826
|
+
entityId: z33.string().max(80).optional(),
|
|
38827
|
+
title: z33.string().min(1).max(180),
|
|
38828
|
+
relationship: z33.string().max(120).optional(),
|
|
38829
|
+
slug: z33.string().max(180).optional(),
|
|
38830
|
+
url: z33.string().url().optional(),
|
|
38831
|
+
description: z33.string().max(300).optional()
|
|
39608
38832
|
}).strict()).max(200).optional(),
|
|
39609
|
-
claims:
|
|
39610
|
-
bodyMarkdown:
|
|
39611
|
-
contentSections:
|
|
39612
|
-
id:
|
|
39613
|
-
heading:
|
|
39614
|
-
body:
|
|
39615
|
-
position:
|
|
39616
|
-
citations:
|
|
38833
|
+
claims: z33.array(CommonsClaimInputSchema).max(300).optional(),
|
|
38834
|
+
bodyMarkdown: z33.string().max(5e5).optional(),
|
|
38835
|
+
contentSections: z33.array(z33.object({
|
|
38836
|
+
id: z33.string().max(180).optional(),
|
|
38837
|
+
heading: z33.string().min(1).max(180),
|
|
38838
|
+
body: z33.string().min(1).max(1e5),
|
|
38839
|
+
position: z33.number().int().positive().optional(),
|
|
38840
|
+
citations: z33.array(CitationSchema).max(300).optional()
|
|
39617
38841
|
}).strict()).max(80).optional(),
|
|
39618
|
-
articleSections:
|
|
39619
|
-
relatedLinks:
|
|
39620
|
-
notes:
|
|
39621
|
-
marker:
|
|
39622
|
-
body:
|
|
38842
|
+
articleSections: z33.object({
|
|
38843
|
+
relatedLinks: z33.array(RelatedLinkSchema).max(100).optional(),
|
|
38844
|
+
notes: z33.array(z33.object({
|
|
38845
|
+
marker: z33.string().max(20).optional(),
|
|
38846
|
+
body: z33.string().min(1).max(2e3)
|
|
39623
38847
|
}).strict()).max(100).optional(),
|
|
39624
|
-
citations:
|
|
39625
|
-
externalLinks:
|
|
39626
|
-
title:
|
|
39627
|
-
url:
|
|
39628
|
-
summary:
|
|
38848
|
+
citations: z33.array(CitationSchema).max(300).optional(),
|
|
38849
|
+
externalLinks: z33.array(z33.object({
|
|
38850
|
+
title: z33.string().min(1).max(180),
|
|
38851
|
+
url: z33.string().url(),
|
|
38852
|
+
summary: z33.string().max(300).optional()
|
|
39629
38853
|
}).strict()).max(100).optional(),
|
|
39630
|
-
categories:
|
|
38854
|
+
categories: z33.array(z33.string().max(120)).max(50).optional()
|
|
39631
38855
|
}).strict().optional(),
|
|
39632
|
-
media:
|
|
39633
|
-
jsonLd:
|
|
39634
|
-
seo:
|
|
39635
|
-
canonicalUrl:
|
|
39636
|
-
relCanonical:
|
|
39637
|
-
metaTitle:
|
|
39638
|
-
metaDescription:
|
|
39639
|
-
ogImage:
|
|
39640
|
-
schemaOrgType:
|
|
39641
|
-
noIndex:
|
|
38856
|
+
media: z33.array(MediaSchema).max(300).optional(),
|
|
38857
|
+
jsonLd: z33.record(z33.string(), z33.unknown()).optional(),
|
|
38858
|
+
seo: z33.object({
|
|
38859
|
+
canonicalUrl: z33.string().url().optional(),
|
|
38860
|
+
relCanonical: z33.string().url().optional(),
|
|
38861
|
+
metaTitle: z33.string().max(240).optional(),
|
|
38862
|
+
metaDescription: z33.string().max(500).optional(),
|
|
38863
|
+
ogImage: z33.string().url().optional(),
|
|
38864
|
+
schemaOrgType: z33.string().max(80).optional(),
|
|
38865
|
+
noIndex: z33.boolean().optional()
|
|
39642
38866
|
}).strict().optional(),
|
|
39643
|
-
actorChannel:
|
|
39644
|
-
actorLabel:
|
|
39645
|
-
changeSummary:
|
|
39646
|
-
baseRevision:
|
|
39647
|
-
reviewPolicy:
|
|
38867
|
+
actorChannel: z33.enum(["mcp", "frontend", "api"]).optional(),
|
|
38868
|
+
actorLabel: z33.string().max(120).optional(),
|
|
38869
|
+
changeSummary: z33.string().max(500).optional(),
|
|
38870
|
+
baseRevision: z33.number().int().positive().optional(),
|
|
38871
|
+
reviewPolicy: z33.enum(["auto_publish_if_safe", "always_review"]).optional()
|
|
39648
38872
|
}).strict();
|
|
39649
38873
|
var PrepareEntitySchema = SubmitEntitySchema.omit({
|
|
39650
38874
|
idempotencyKey: true,
|
|
@@ -39654,33 +38878,33 @@ var PrepareEntitySchema = SubmitEntitySchema.omit({
|
|
|
39654
38878
|
baseRevision: true,
|
|
39655
38879
|
reviewPolicy: true
|
|
39656
38880
|
}).partial().extend({
|
|
39657
|
-
title:
|
|
39658
|
-
schemaOrgType:
|
|
39659
|
-
maxCandidates:
|
|
38881
|
+
title: z33.string().min(1).max(180),
|
|
38882
|
+
schemaOrgType: z33.string().max(80).optional(),
|
|
38883
|
+
maxCandidates: z33.number().int().min(1).max(20).optional()
|
|
39660
38884
|
}).strict();
|
|
39661
38885
|
var ValidateEntitySchema = SubmitEntitySchema.partial().extend({
|
|
39662
|
-
idempotencyKey:
|
|
38886
|
+
idempotencyKey: z33.string().min(8).max(200).optional()
|
|
39663
38887
|
}).strict();
|
|
39664
|
-
var SaveFilterSchema =
|
|
39665
|
-
name:
|
|
39666
|
-
description:
|
|
38888
|
+
var SaveFilterSchema = z33.object({
|
|
38889
|
+
name: z33.string().min(2).max(80),
|
|
38890
|
+
description: z33.string().max(500).optional(),
|
|
39667
38891
|
filter: SearchBodySchema
|
|
39668
38892
|
}).strict();
|
|
39669
|
-
var NeedsLinkBodySchema =
|
|
39670
|
-
query:
|
|
39671
|
-
entityType:
|
|
39672
|
-
tag:
|
|
39673
|
-
tags:
|
|
39674
|
-
sourceEntityId:
|
|
39675
|
-
sourceSlug:
|
|
39676
|
-
limit:
|
|
39677
|
-
offset:
|
|
38893
|
+
var NeedsLinkBodySchema = z33.object({
|
|
38894
|
+
query: z33.string().max(300).optional(),
|
|
38895
|
+
entityType: z33.string().max(120).optional(),
|
|
38896
|
+
tag: z33.string().max(80).optional(),
|
|
38897
|
+
tags: z33.array(z33.string().max(80)).max(20).optional(),
|
|
38898
|
+
sourceEntityId: z33.string().max(80).optional(),
|
|
38899
|
+
sourceSlug: z33.string().max(180).optional(),
|
|
38900
|
+
limit: z33.number().int().min(1).max(100).optional(),
|
|
38901
|
+
offset: z33.number().int().min(0).max(1e4).optional()
|
|
39678
38902
|
}).strict();
|
|
39679
|
-
var PreparePublicationSchema =
|
|
39680
|
-
var ValidatePublicationSchema =
|
|
39681
|
-
var ClaimPublicationSchema =
|
|
39682
|
-
var PublishEditorialSchema =
|
|
39683
|
-
var UpdateEditorialArticleSchema =
|
|
38903
|
+
var PreparePublicationSchema = z33.object(CommonsPreparePublicationInputSchema).strict();
|
|
38904
|
+
var ValidatePublicationSchema = z33.object(CommonsValidatePublicationInputSchema).strict();
|
|
38905
|
+
var ClaimPublicationSchema = z33.object(CommonsClaimPublicationInputSchema).strict();
|
|
38906
|
+
var PublishEditorialSchema = z33.object(CommonsPublishEditorialInputSchema).strict();
|
|
38907
|
+
var UpdateEditorialArticleSchema = z33.object(CommonsUpdateEditorialArticleInputSchema).strict();
|
|
39684
38908
|
commonsApp.post("/index/embed", auth2, async (c) => {
|
|
39685
38909
|
const body = await c.req.json().catch(() => ({}));
|
|
39686
38910
|
const result = await embedQueuedCommonsDocuments(Number(body.limit) || 50);
|
|
@@ -39800,12 +39024,12 @@ commonsApp.get("/publications/:subdomain", async (c) => {
|
|
|
39800
39024
|
return publicationError(c, error);
|
|
39801
39025
|
}
|
|
39802
39026
|
});
|
|
39803
|
-
var HostImageSchema =
|
|
39804
|
-
sourceUrl:
|
|
39805
|
-
imageBase64:
|
|
39806
|
-
alt:
|
|
39807
|
-
license:
|
|
39808
|
-
attribution:
|
|
39027
|
+
var HostImageSchema = z33.object({
|
|
39028
|
+
sourceUrl: z33.string().url().optional(),
|
|
39029
|
+
imageBase64: z33.string().min(16).optional(),
|
|
39030
|
+
alt: z33.string().max(500).optional(),
|
|
39031
|
+
license: z33.string().max(240).optional(),
|
|
39032
|
+
attribution: z33.string().max(500).optional()
|
|
39809
39033
|
}).strict().refine((body) => Boolean(body.sourceUrl || body.imageBase64), {
|
|
39810
39034
|
message: "Provide either sourceUrl or imageBase64."
|
|
39811
39035
|
});
|
|
@@ -40041,7 +39265,7 @@ function xmlEscape(value) {
|
|
|
40041
39265
|
// src/api/analytics-routes.ts
|
|
40042
39266
|
import { Hono as Hono34 } from "hono";
|
|
40043
39267
|
import { createMiddleware as createMiddleware4 } from "hono/factory";
|
|
40044
|
-
import { z as
|
|
39268
|
+
import { z as z35 } from "zod";
|
|
40045
39269
|
import { createHash as createHash21 } from "crypto";
|
|
40046
39270
|
import Papa4 from "papaparse";
|
|
40047
39271
|
|
|
@@ -40052,28 +39276,28 @@ import {
|
|
|
40052
39276
|
randomBytes as randomBytes4,
|
|
40053
39277
|
timingSafeEqual as timingSafeEqual2
|
|
40054
39278
|
} from "crypto";
|
|
40055
|
-
import { z as
|
|
39279
|
+
import { z as z34 } from "zod";
|
|
40056
39280
|
var THORBIT_PRODUCT_URL = "https://thorbit.ai";
|
|
40057
39281
|
var REFRESH_INTERVAL_MS = 15 * 60 * 1e3;
|
|
40058
39282
|
var ELIGIBLE_GRACE_MS = 24 * 60 * 60 * 1e3;
|
|
40059
39283
|
var TRIAL_LENGTH_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
40060
|
-
var ThorbitEntitlementSchema =
|
|
40061
|
-
product:
|
|
40062
|
-
eligible:
|
|
40063
|
-
reason:
|
|
40064
|
-
organizationId:
|
|
40065
|
-
planId:
|
|
40066
|
-
subscriptionStatus:
|
|
40067
|
-
monthlyRecurringRevenueCents:
|
|
40068
|
-
minimumMonthlyRecurringRevenueCents:
|
|
40069
|
-
checkedAt:
|
|
39284
|
+
var ThorbitEntitlementSchema = z34.object({
|
|
39285
|
+
product: z34.literal("x_ray_pixel"),
|
|
39286
|
+
eligible: z34.boolean(),
|
|
39287
|
+
reason: z34.string().min(1).max(120),
|
|
39288
|
+
organizationId: z34.string().min(1).max(64),
|
|
39289
|
+
planId: z34.string().max(32).nullable(),
|
|
39290
|
+
subscriptionStatus: z34.string().max(20).nullable(),
|
|
39291
|
+
monthlyRecurringRevenueCents: z34.number().int().min(0),
|
|
39292
|
+
minimumMonthlyRecurringRevenueCents: z34.number().int().min(5e3),
|
|
39293
|
+
checkedAt: z34.string().datetime()
|
|
40070
39294
|
}).strict();
|
|
40071
|
-
var AssertionSchema =
|
|
40072
|
-
version:
|
|
40073
|
-
issuer:
|
|
40074
|
-
audience:
|
|
40075
|
-
issuedAt:
|
|
40076
|
-
expiresAt:
|
|
39295
|
+
var AssertionSchema = z34.object({
|
|
39296
|
+
version: z34.literal(1),
|
|
39297
|
+
issuer: z34.literal("thorbit"),
|
|
39298
|
+
audience: z34.literal("mcp-scraper:x-ray"),
|
|
39299
|
+
issuedAt: z34.number().int().positive(),
|
|
39300
|
+
expiresAt: z34.number().int().positive(),
|
|
40077
39301
|
entitlement: ThorbitEntitlementSchema
|
|
40078
39302
|
}).strict();
|
|
40079
39303
|
function thorbitBaseUrl() {
|
|
@@ -40215,7 +39439,7 @@ async function requestEntitlement(method, options) {
|
|
|
40215
39439
|
const message = payload && typeof payload === "object" && "error" in payload ? String(payload.error) : `Thorbit entitlement check failed (${response.status})`;
|
|
40216
39440
|
throw new Error(message);
|
|
40217
39441
|
}
|
|
40218
|
-
const parsed =
|
|
39442
|
+
const parsed = z34.object({ entitlement: ThorbitEntitlementSchema }).safeParse(payload);
|
|
40219
39443
|
if (!parsed.success) throw new Error("Thorbit returned an invalid entitlement assertion");
|
|
40220
39444
|
return parsed.data.entitlement;
|
|
40221
39445
|
}
|
|
@@ -40350,83 +39574,83 @@ var entitlementGuard = createMiddleware4(async (c, next) => {
|
|
|
40350
39574
|
}
|
|
40351
39575
|
await next();
|
|
40352
39576
|
});
|
|
40353
|
-
var ThorbitApiKeySchema =
|
|
40354
|
-
var ThorbitCallbackSchema =
|
|
40355
|
-
state:
|
|
40356
|
-
assertion:
|
|
39577
|
+
var ThorbitApiKeySchema = z35.object({ apiKey: z35.string().trim().min(20).max(512) }).strict();
|
|
39578
|
+
var ThorbitCallbackSchema = z35.object({
|
|
39579
|
+
state: z35.string().min(20).max(300),
|
|
39580
|
+
assertion: z35.string().min(40).max(8e3)
|
|
40357
39581
|
}).strict();
|
|
40358
|
-
var SiteInputSchema =
|
|
40359
|
-
name:
|
|
40360
|
-
slug:
|
|
40361
|
-
timezone:
|
|
39582
|
+
var SiteInputSchema = z35.object({
|
|
39583
|
+
name: z35.string().trim().min(1).max(120),
|
|
39584
|
+
slug: z35.string().trim().min(1).max(80).optional(),
|
|
39585
|
+
timezone: z35.string().trim().min(1).max(80).optional()
|
|
40362
39586
|
}).strict();
|
|
40363
|
-
var PixelInputSchema =
|
|
40364
|
-
name:
|
|
40365
|
-
environment:
|
|
39587
|
+
var PixelInputSchema = z35.object({
|
|
39588
|
+
name: z35.string().trim().min(1).max(120),
|
|
39589
|
+
environment: z35.string().trim().min(1).max(60).optional()
|
|
40366
39590
|
}).strict();
|
|
40367
|
-
var PixelUpdateSchema =
|
|
40368
|
-
name:
|
|
40369
|
-
environment:
|
|
40370
|
-
status:
|
|
39591
|
+
var PixelUpdateSchema = z35.object({
|
|
39592
|
+
name: z35.string().trim().min(1).max(120).optional(),
|
|
39593
|
+
environment: z35.string().trim().min(1).max(60).optional(),
|
|
39594
|
+
status: z35.enum(["active", "disabled"]).optional()
|
|
40371
39595
|
}).strict().refine(
|
|
40372
39596
|
(value) => Object.keys(value).length > 0,
|
|
40373
39597
|
"Provide at least one change."
|
|
40374
39598
|
);
|
|
40375
|
-
var EventSchema =
|
|
40376
|
-
eventId:
|
|
40377
|
-
eventName:
|
|
40378
|
-
occurredAt:
|
|
40379
|
-
visitorId:
|
|
40380
|
-
sessionId:
|
|
40381
|
-
path:
|
|
40382
|
-
canonicalUrl:
|
|
40383
|
-
title:
|
|
40384
|
-
referrer:
|
|
40385
|
-
source:
|
|
40386
|
-
medium:
|
|
40387
|
-
campaign:
|
|
40388
|
-
clickIds:
|
|
40389
|
-
deviceClass:
|
|
40390
|
-
properties:
|
|
40391
|
-
engagedMs:
|
|
40392
|
-
scrollDepth:
|
|
39599
|
+
var EventSchema = z35.object({
|
|
39600
|
+
eventId: z35.string().trim().min(8).max(160),
|
|
39601
|
+
eventName: z35.string().trim().regex(/^[a-z][a-z0-9_]{1,79}$/),
|
|
39602
|
+
occurredAt: z35.string().datetime(),
|
|
39603
|
+
visitorId: z35.string().trim().max(160).optional(),
|
|
39604
|
+
sessionId: z35.string().trim().max(160).optional(),
|
|
39605
|
+
path: z35.string().trim().max(2e3).optional(),
|
|
39606
|
+
canonicalUrl: z35.string().url().max(3e3).optional(),
|
|
39607
|
+
title: z35.string().trim().max(500).optional(),
|
|
39608
|
+
referrer: z35.string().trim().max(3e3).optional(),
|
|
39609
|
+
source: z35.string().trim().max(180).optional(),
|
|
39610
|
+
medium: z35.string().trim().max(180).optional(),
|
|
39611
|
+
campaign: z35.string().trim().max(240).optional(),
|
|
39612
|
+
clickIds: z35.record(z35.string(), z35.string().trim().max(500)).optional(),
|
|
39613
|
+
deviceClass: z35.enum(["desktop", "tablet", "mobile", "unknown"]).optional(),
|
|
39614
|
+
properties: z35.record(z35.string(), z35.unknown()).optional(),
|
|
39615
|
+
engagedMs: z35.number().int().min(0).max(MAX_ENGAGED_MS).optional(),
|
|
39616
|
+
scrollDepth: z35.number().int().min(0).max(100).optional()
|
|
40393
39617
|
}).strict();
|
|
40394
|
-
var IngestionSchema =
|
|
40395
|
-
pixelId:
|
|
40396
|
-
events:
|
|
39618
|
+
var IngestionSchema = z35.object({
|
|
39619
|
+
pixelId: z35.string().trim().min(8).max(160),
|
|
39620
|
+
events: z35.array(EventSchema).min(1).max(20)
|
|
40397
39621
|
}).strict();
|
|
40398
|
-
var ConversionSchema =
|
|
40399
|
-
siteId:
|
|
40400
|
-
pixelId:
|
|
40401
|
-
sessionId:
|
|
40402
|
-
sourceEventId:
|
|
40403
|
-
conversionKind:
|
|
40404
|
-
valueMinor:
|
|
40405
|
-
currency:
|
|
40406
|
-
occurredAt:
|
|
39622
|
+
var ConversionSchema = z35.object({
|
|
39623
|
+
siteId: z35.string().uuid(),
|
|
39624
|
+
pixelId: z35.string().uuid().optional(),
|
|
39625
|
+
sessionId: z35.string().trim().max(160).optional(),
|
|
39626
|
+
sourceEventId: z35.string().trim().min(8).max(200),
|
|
39627
|
+
conversionKind: z35.string().trim().regex(/^[a-z][a-z0-9_]{1,79}$/),
|
|
39628
|
+
valueMinor: z35.number().int().min(0).max(2147483647).optional(),
|
|
39629
|
+
currency: z35.string().trim().regex(/^[A-Za-z]{3}$/).optional(),
|
|
39630
|
+
occurredAt: z35.string().datetime()
|
|
40407
39631
|
}).strict();
|
|
40408
|
-
var ExportSchema =
|
|
40409
|
-
report:
|
|
40410
|
-
format:
|
|
40411
|
-
idempotencyKey:
|
|
39632
|
+
var ExportSchema = z35.object({
|
|
39633
|
+
report: z35.enum(["overview", "acquisition", "content", "conversions"]).default("overview"),
|
|
39634
|
+
format: z35.enum(["csv", "json", "markdown"]).default("csv"),
|
|
39635
|
+
idempotencyKey: z35.string().trim().min(8).max(160).optional()
|
|
40412
39636
|
}).strict();
|
|
40413
|
-
var CampaignLinkSchema =
|
|
40414
|
-
pixelId:
|
|
40415
|
-
name:
|
|
40416
|
-
destinationUrl:
|
|
40417
|
-
shortCode:
|
|
40418
|
-
source:
|
|
40419
|
-
medium:
|
|
40420
|
-
campaign:
|
|
40421
|
-
term:
|
|
40422
|
-
content:
|
|
40423
|
-
adGroup:
|
|
40424
|
-
adName:
|
|
40425
|
-
creativeId:
|
|
39637
|
+
var CampaignLinkSchema = z35.object({
|
|
39638
|
+
pixelId: z35.string().uuid().optional(),
|
|
39639
|
+
name: z35.string().trim().min(1).max(120),
|
|
39640
|
+
destinationUrl: z35.string().url().max(3e3),
|
|
39641
|
+
shortCode: z35.string().trim().min(3).max(48).optional(),
|
|
39642
|
+
source: z35.string().trim().min(1).max(180),
|
|
39643
|
+
medium: z35.string().trim().min(1).max(180),
|
|
39644
|
+
campaign: z35.string().trim().min(1).max(240),
|
|
39645
|
+
term: z35.string().trim().max(240).optional(),
|
|
39646
|
+
content: z35.string().trim().max(240).optional(),
|
|
39647
|
+
adGroup: z35.string().trim().max(240).optional(),
|
|
39648
|
+
adName: z35.string().trim().max(240).optional(),
|
|
39649
|
+
creativeId: z35.string().trim().max(240).optional()
|
|
40426
39650
|
}).strict();
|
|
40427
|
-
var FormFieldSchema =
|
|
40428
|
-
id:
|
|
40429
|
-
type:
|
|
39651
|
+
var FormFieldSchema = z35.object({
|
|
39652
|
+
id: z35.string().trim().regex(/^[a-z][a-z0-9_]{0,47}$/),
|
|
39653
|
+
type: z35.enum([
|
|
40430
39654
|
"text",
|
|
40431
39655
|
"email",
|
|
40432
39656
|
"tel",
|
|
@@ -40436,45 +39660,45 @@ var FormFieldSchema = z36.object({
|
|
|
40436
39660
|
"radio",
|
|
40437
39661
|
"number"
|
|
40438
39662
|
]),
|
|
40439
|
-
label:
|
|
40440
|
-
required:
|
|
40441
|
-
options:
|
|
39663
|
+
label: z35.string().trim().min(1).max(120),
|
|
39664
|
+
required: z35.boolean().default(false),
|
|
39665
|
+
options: z35.array(z35.string().trim().min(1).max(120)).max(30).optional()
|
|
40442
39666
|
}).strict();
|
|
40443
|
-
var FormInputSchema =
|
|
40444
|
-
pixelId:
|
|
40445
|
-
name:
|
|
40446
|
-
fields:
|
|
39667
|
+
var FormInputSchema = z35.object({
|
|
39668
|
+
pixelId: z35.string().uuid(),
|
|
39669
|
+
name: z35.string().trim().min(1).max(120),
|
|
39670
|
+
fields: z35.array(FormFieldSchema).min(1).max(30).refine(
|
|
40447
39671
|
(fields) => new Set(fields.map((field) => field.id)).size === fields.length,
|
|
40448
39672
|
"Field IDs must be unique."
|
|
40449
39673
|
),
|
|
40450
|
-
brand:
|
|
40451
|
-
primaryColor:
|
|
40452
|
-
backgroundColor:
|
|
40453
|
-
textColor:
|
|
40454
|
-
radius:
|
|
39674
|
+
brand: z35.object({
|
|
39675
|
+
primaryColor: z35.string().regex(/^#[0-9a-f]{6}$/i),
|
|
39676
|
+
backgroundColor: z35.string().regex(/^#[0-9a-f]{6}$/i),
|
|
39677
|
+
textColor: z35.string().regex(/^#[0-9a-f]{6}$/i),
|
|
39678
|
+
radius: z35.number().int().min(0).max(32)
|
|
40455
39679
|
}).strict(),
|
|
40456
|
-
submitLabel:
|
|
40457
|
-
successMessage:
|
|
40458
|
-
redirectUrl:
|
|
40459
|
-
consentText:
|
|
40460
|
-
publish:
|
|
39680
|
+
submitLabel: z35.string().trim().min(1).max(80).optional(),
|
|
39681
|
+
successMessage: z35.string().trim().min(1).max(500).optional(),
|
|
39682
|
+
redirectUrl: z35.string().url().max(3e3).optional(),
|
|
39683
|
+
consentText: z35.string().trim().max(1e3).optional(),
|
|
39684
|
+
publish: z35.boolean().optional()
|
|
40461
39685
|
}).strict();
|
|
40462
|
-
var FormSubmissionSchema =
|
|
40463
|
-
data:
|
|
40464
|
-
|
|
40465
|
-
|
|
39686
|
+
var FormSubmissionSchema = z35.object({
|
|
39687
|
+
data: z35.record(
|
|
39688
|
+
z35.string(),
|
|
39689
|
+
z35.union([z35.string().max(5e3), z35.number(), z35.boolean()])
|
|
40466
39690
|
).refine((value) => Object.keys(value).length <= 30),
|
|
40467
|
-
visitorId:
|
|
40468
|
-
sessionId:
|
|
40469
|
-
source:
|
|
40470
|
-
medium:
|
|
40471
|
-
campaign:
|
|
40472
|
-
clickIds:
|
|
40473
|
-
placementUrl:
|
|
40474
|
-
website:
|
|
39691
|
+
visitorId: z35.string().trim().max(160).optional(),
|
|
39692
|
+
sessionId: z35.string().trim().max(160).optional(),
|
|
39693
|
+
source: z35.string().trim().max(180).optional(),
|
|
39694
|
+
medium: z35.string().trim().max(180).optional(),
|
|
39695
|
+
campaign: z35.string().trim().max(240).optional(),
|
|
39696
|
+
clickIds: z35.record(z35.string(), z35.string().trim().max(500)).optional(),
|
|
39697
|
+
placementUrl: z35.string().url().max(3e3),
|
|
39698
|
+
website: z35.string().max(0).optional()
|
|
40475
39699
|
}).strict();
|
|
40476
|
-
var CrmImportSchema =
|
|
40477
|
-
sourceSystem:
|
|
39700
|
+
var CrmImportSchema = z35.object({
|
|
39701
|
+
sourceSystem: z35.enum([
|
|
40478
39702
|
"hubspot",
|
|
40479
39703
|
"salesforce",
|
|
40480
39704
|
"gohighlevel",
|
|
@@ -40483,35 +39707,35 @@ var CrmImportSchema = z36.object({
|
|
|
40483
39707
|
"keap",
|
|
40484
39708
|
"other"
|
|
40485
39709
|
]),
|
|
40486
|
-
filename:
|
|
40487
|
-
csv:
|
|
40488
|
-
mapping:
|
|
40489
|
-
email:
|
|
40490
|
-
firstName:
|
|
40491
|
-
lastName:
|
|
40492
|
-
name:
|
|
40493
|
-
phone:
|
|
40494
|
-
company:
|
|
40495
|
-
externalId:
|
|
39710
|
+
filename: z35.string().trim().min(1).max(240),
|
|
39711
|
+
csv: z35.string().min(1).max(8e6),
|
|
39712
|
+
mapping: z35.object({
|
|
39713
|
+
email: z35.string().trim().min(1).max(240).optional(),
|
|
39714
|
+
firstName: z35.string().trim().min(1).max(240).optional(),
|
|
39715
|
+
lastName: z35.string().trim().min(1).max(240).optional(),
|
|
39716
|
+
name: z35.string().trim().min(1).max(240).optional(),
|
|
39717
|
+
phone: z35.string().trim().min(1).max(240).optional(),
|
|
39718
|
+
company: z35.string().trim().min(1).max(240).optional(),
|
|
39719
|
+
externalId: z35.string().trim().min(1).max(240).optional()
|
|
40496
39720
|
}).strict()
|
|
40497
39721
|
}).strict();
|
|
40498
|
-
var ActivationDestinationSchema =
|
|
40499
|
-
platform:
|
|
40500
|
-
name:
|
|
40501
|
-
connectionRef:
|
|
40502
|
-
externalDatasetId:
|
|
40503
|
-
eventMapping:
|
|
39722
|
+
var ActivationDestinationSchema = z35.object({
|
|
39723
|
+
platform: z35.enum(["meta", "google", "tiktok", "reddit"]),
|
|
39724
|
+
name: z35.string().trim().min(1).max(120),
|
|
39725
|
+
connectionRef: z35.string().trim().max(240).optional(),
|
|
39726
|
+
externalDatasetId: z35.string().trim().max(240).optional(),
|
|
39727
|
+
eventMapping: z35.record(z35.string(), z35.string().trim().max(120)).optional()
|
|
40504
39728
|
}).strict();
|
|
40505
|
-
var BusinessModelSchema =
|
|
40506
|
-
var AdSpendSchema =
|
|
40507
|
-
rows:
|
|
40508
|
-
|
|
40509
|
-
date:
|
|
40510
|
-
platform:
|
|
40511
|
-
campaign:
|
|
40512
|
-
spendMinor:
|
|
40513
|
-
currency:
|
|
40514
|
-
sourceRef:
|
|
39729
|
+
var BusinessModelSchema = z35.object({ businessModel: z35.enum(["leadgen", "saas", "ecommerce"]) }).strict();
|
|
39730
|
+
var AdSpendSchema = z35.object({
|
|
39731
|
+
rows: z35.array(
|
|
39732
|
+
z35.object({
|
|
39733
|
+
date: z35.string().date(),
|
|
39734
|
+
platform: z35.string().trim().min(1).max(80),
|
|
39735
|
+
campaign: z35.string().trim().max(240).optional(),
|
|
39736
|
+
spendMinor: z35.number().int().min(0).max(Number.MAX_SAFE_INTEGER),
|
|
39737
|
+
currency: z35.string().regex(/^[A-Za-z]{3}$/).optional(),
|
|
39738
|
+
sourceRef: z35.string().max(500).optional()
|
|
40515
39739
|
}).strict()
|
|
40516
39740
|
).min(1).max(5e3)
|
|
40517
39741
|
}).strict();
|
|
@@ -41744,7 +40968,7 @@ async function deliverPendingAnalyticsForms(limit = 10) {
|
|
|
41744
40968
|
|
|
41745
40969
|
// src/api/scheduled-result-routes.ts
|
|
41746
40970
|
import { Hono as Hono35 } from "hono";
|
|
41747
|
-
import { z as
|
|
40971
|
+
import { z as z36 } from "zod";
|
|
41748
40972
|
|
|
41749
40973
|
// src/api/scheduled-artifact-owner.ts
|
|
41750
40974
|
import { createHash as createHash22 } from "crypto";
|
|
@@ -42385,15 +41609,15 @@ var scheduledArtifactTemplatePresets = [
|
|
|
42385
41609
|
];
|
|
42386
41610
|
|
|
42387
41611
|
// src/api/scheduled-result-routes.ts
|
|
42388
|
-
var TemplateParamsSchema =
|
|
42389
|
-
var RunParamsSchema =
|
|
42390
|
-
var ViewLinkParamsSchema =
|
|
42391
|
-
runId:
|
|
42392
|
-
shareId:
|
|
41612
|
+
var TemplateParamsSchema = z36.object({ templateId: z36.string().uuid() }).strict();
|
|
41613
|
+
var RunParamsSchema = z36.object({ runId: z36.string().min(1).max(200) }).strict();
|
|
41614
|
+
var ViewLinkParamsSchema = z36.object({
|
|
41615
|
+
runId: z36.string().min(1).max(200),
|
|
41616
|
+
shareId: z36.string().uuid()
|
|
42393
41617
|
}).strict();
|
|
42394
|
-
var ArtifactTemplateListQuerySchema =
|
|
42395
|
-
status:
|
|
42396
|
-
query:
|
|
41618
|
+
var ArtifactTemplateListQuerySchema = z36.object({
|
|
41619
|
+
status: z36.enum(["active", "archived", "all"]).default("active"),
|
|
41620
|
+
query: z36.string().trim().min(1).max(120).optional(),
|
|
42397
41621
|
presetKey: ArtifactTemplatePresetKeySchema.optional()
|
|
42398
41622
|
}).strict();
|
|
42399
41623
|
function apiError(error, errorCode = "invalid_request") {
|
|
@@ -49427,8 +48651,6 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
49427
48651
|
if (checked.error) return c.json({ error: checked.error }, 400);
|
|
49428
48652
|
body.callback_url = checked.parsed?.href;
|
|
49429
48653
|
}
|
|
49430
|
-
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
49431
|
-
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
49432
48654
|
const options = {
|
|
49433
48655
|
query: body.query.trim(),
|
|
49434
48656
|
location: body.location,
|
|
@@ -49450,6 +48672,26 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
49450
48672
|
includeAiOverview: body.includeAiOverview ?? false,
|
|
49451
48673
|
includeWhatPeopleSaying: body.includeWhatPeopleSaying ?? false
|
|
49452
48674
|
};
|
|
48675
|
+
const suppliedIdempotencyKey = c.req.header("idempotency-key") ?? c.req.header("x-idempotency-key");
|
|
48676
|
+
let derivedJobId = null;
|
|
48677
|
+
let requestFingerprint3 = null;
|
|
48678
|
+
if (!options.serpOnly && suppliedIdempotencyKey != null) {
|
|
48679
|
+
const trimmedKey = suppliedIdempotencyKey.trim();
|
|
48680
|
+
if (!trimmedKey || trimmedKey.length > 200) {
|
|
48681
|
+
return c.json({ error: "Idempotency-Key must contain 1-200 characters.", errorCode: "invalid_idempotency_key", retryable: false }, 400);
|
|
48682
|
+
}
|
|
48683
|
+
derivedJobId = `paa-${sha256Hex(`${user.id}\0${trimmedKey}`).slice(0, 24)}`;
|
|
48684
|
+
requestFingerprint3 = sha256Hex(JSON.stringify({ options, callbackUrl: body.callback_url ?? null }));
|
|
48685
|
+
const existing = await getJob(derivedJobId, user.id);
|
|
48686
|
+
if (existing) {
|
|
48687
|
+
if (existing.options.requestFingerprint !== requestFingerprint3) {
|
|
48688
|
+
return c.json({ error: "Idempotency-Key was already used for a different asynchronous harvest_paa call.", errorCode: "idempotency_conflict", retryable: false }, 409);
|
|
48689
|
+
}
|
|
48690
|
+
return c.json({ job_id: existing.id, status: existing.status, replayed: true }, ["done", "failed", "cancelled"].includes(existing.status) ? 200 : 202);
|
|
48691
|
+
}
|
|
48692
|
+
}
|
|
48693
|
+
const limitErr = await checkHarvestLimits(user, c.req.header("x-mcp-scraper-concurrency-lock"));
|
|
48694
|
+
if (limitErr) return c.json(limitErr, 429, { "Retry-After": "30" });
|
|
49453
48695
|
const harvestCost = options.serpOnly ? MC_COSTS.serp_headful : paaCostForQuestionCount(options.maxQuestions);
|
|
49454
48696
|
let jobId2;
|
|
49455
48697
|
if (options.serpOnly) {
|
|
@@ -49462,7 +48704,7 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
49462
48704
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
49463
48705
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
49464
48706
|
} else {
|
|
49465
|
-
jobId2 = randomUUID28();
|
|
48707
|
+
jobId2 = derivedJobId ?? randomUUID28();
|
|
49466
48708
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
49467
48709
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
49468
48710
|
const hold = await debitMcIdempotent(
|
|
@@ -49478,9 +48720,14 @@ app.post("/harvest", auth4, async (c) => {
|
|
|
49478
48720
|
...options,
|
|
49479
48721
|
executionOwner: "inngest",
|
|
49480
48722
|
billingHoldMc: harvestCost,
|
|
49481
|
-
billingDebitKey
|
|
48723
|
+
billingDebitKey,
|
|
48724
|
+
...requestFingerprint3 ? { requestFingerprint: requestFingerprint3 } : {}
|
|
49482
48725
|
}, body.callback_url);
|
|
49483
48726
|
} catch (error) {
|
|
48727
|
+
const existing = derivedJobId ? await getJob(derivedJobId, user.id).catch(() => null) : null;
|
|
48728
|
+
if (existing && existing.options.requestFingerprint === requestFingerprint3) {
|
|
48729
|
+
return c.json({ job_id: existing.id, status: existing.status, replayed: true }, 202);
|
|
48730
|
+
}
|
|
49484
48731
|
await settleDebitMcIdempotent(
|
|
49485
48732
|
user.id,
|
|
49486
48733
|
billingDebitKey,
|
|
@@ -51013,7 +50260,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
51013
50260
|
return c.json({ error: "Unauthorized" }, 401);
|
|
51014
50261
|
}
|
|
51015
50262
|
const startedAt = Date.now();
|
|
51016
|
-
const { drainQueue } = await import("./worker-
|
|
50263
|
+
const { drainQueue } = await import("./worker-MAVYYWM7.js");
|
|
51017
50264
|
const budget = { maxJobs: 10, deadlineMs: startedAt + CRON_TICK_DRAIN_BUDGET_MS };
|
|
51018
50265
|
const origin = `${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`;
|
|
51019
50266
|
const unfinished = [];
|
|
@@ -51093,8 +50340,8 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
51093
50340
|
finishExtractJob: finishExtractJob2,
|
|
51094
50341
|
terminalExtractJobStatus: terminalExtractJobStatus2,
|
|
51095
50342
|
extractJobLimitInfo: extractJobLimitInfo2
|
|
51096
|
-
} = await import("./site-extract-repository-
|
|
51097
|
-
const { assembleExtractArtifacts } = await import("./extract-bundle-
|
|
50343
|
+
} = await import("./site-extract-repository-ASYEJO7Z.js");
|
|
50344
|
+
const { assembleExtractArtifacts } = await import("./extract-bundle-SIBFUA6I.js");
|
|
51098
50345
|
const existing = await getExtractJob2(jobId2);
|
|
51099
50346
|
if (!existing) return c.json({ error: "job not found" }, 404);
|
|
51100
50347
|
if (existing.status !== "failed") {
|
|
@@ -51121,7 +50368,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
51121
50368
|
if (!updated) return c.json({ error: "job state changed during re-finalization" }, 409);
|
|
51122
50369
|
let settlement = "already_settled_or_refunded";
|
|
51123
50370
|
if (job.billedMc == null && job.userId != null) {
|
|
51124
|
-
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
50371
|
+
const { settleExtractJob: settleExtractJob3, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-ASYEJO7Z.js");
|
|
51125
50372
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
51126
50373
|
const successful = await countSuccessfulPages2(jobId2);
|
|
51127
50374
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -51308,4 +50555,3 @@ app.get("/blog/:slug/", (c) => {
|
|
|
51308
50555
|
export {
|
|
51309
50556
|
app
|
|
51310
50557
|
};
|
|
51311
|
-
//# sourceMappingURL=server-RSHSZBUZ.js.map
|