mcp-scraper 0.65.1 → 0.66.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -1
- package/README.md +11 -6
- package/dist/{analytics-repository-IKE53S3E.js → analytics-repository-RUXBFSFX.js} +1 -2
- package/dist/bin/api-server.cjs +1844 -71
- package/dist/bin/api-server.js +4 -5
- package/dist/bin/mcp-scraper-cli.cjs +1 -2
- package/dist/bin/mcp-scraper-cli.js +6 -7
- package/dist/bin/mcp-scraper-install.cjs +7 -3
- package/dist/bin/mcp-scraper-install.js +2 -3
- package/dist/bin/mcp-stdio-server.cjs +5150 -3467
- package/dist/bin/mcp-stdio-server.js +11 -11
- package/dist/bin/paa-harvest.cjs +8 -3
- package/dist/bin/paa-harvest.js +4 -5
- package/dist/{chunk-5PIO7QBG.js → chunk-3PIXKLGZ.js} +3 -4
- package/dist/{chunk-VIWT35S6.js → chunk-4OT7BAIE.js} +1 -2
- package/dist/{chunk-ORKXE7VX.js → chunk-4P3PHFPQ.js} +11 -6
- package/dist/{chunk-AMLFKPLL.js → chunk-75VW7CWE.js} +4 -7
- package/dist/{chunk-FG6LJGJ7.js → chunk-77LQV73B.js} +0 -1
- package/dist/{chunk-DKIG2TEE.js → chunk-AQKJOSIB.js} +2 -3
- package/dist/{chunk-2XUXVKT4.js → chunk-AYJHI2FK.js} +0 -1
- package/dist/{chunk-5BEPOFKG.js → chunk-CNLLTD5B.js} +0 -1
- package/dist/{chunk-EKJ65E3Z.js → chunk-DNM65UCK.js} +0 -1
- package/dist/{chunk-PDAKCYHZ.js → chunk-E4LMY5OJ.js} +6258 -3747
- package/dist/{chunk-MQGBCWZD.js → chunk-GGZEC22A.js} +64 -1
- package/dist/{chunk-4VG3NUUY.js → chunk-GTGSD4YS.js} +2 -2
- package/dist/{chunk-MQQS7ONH.js → chunk-KBTUPRTF.js} +8 -296
- package/dist/{chunk-XPEN6OG5.js → chunk-NS754CRS.js} +15 -3
- package/dist/{chunk-UN4LZVCB.js → chunk-OZJMVCDK.js} +1 -2
- package/dist/{chunk-342OUCSG.js → chunk-PGJQDMC2.js} +1 -2
- package/dist/{chunk-G3P3ZDB4.js → chunk-RJ7JVYKU.js} +0 -1
- package/dist/{chunk-N54JS5WP.js → chunk-TNJIPV74.js} +1 -2
- package/dist/{chunk-4LINSJNI.js → chunk-TNLGHNLX.js} +1 -2
- package/dist/chunk-VRJ7CO6L.js +6 -0
- package/dist/chunk-VXLU74YZ.js +293 -0
- package/dist/{chunk-DCWXVAQT.js → chunk-WID3MFRS.js} +6 -2
- package/dist/{db-BRI465XQ.js → db-PO7N7ZP3.js} +1 -2
- package/dist/{extract-bundle-UWKJT4MU.js → extract-bundle-SIBFUA6I.js} +8 -10
- package/dist/index.cjs +8 -3
- package/dist/index.js +4 -5
- package/dist/{lead-list-enrichment-repository-ODWGRVCQ.js → lead-list-enrichment-repository-OTAUQONJ.js} +2 -3
- package/dist/{location-data-repository-67MMEVZQ.js → location-data-repository-RM2KJ3H5.js} +3 -4
- package/dist/{server-RSHSZBUZ.js → server-ELEJNRRE.js} +875 -1629
- package/dist/{site-extract-repository-J3RH3MOS.js → site-extract-repository-ASYEJO7Z.js} +4 -5
- package/dist/{worker-YKRIK5LS.js → worker-MAVYYWM7.js} +8 -9
- package/package.json +3 -1
- package/dist/analytics-repository-IKE53S3E.js.map +0 -1
- package/dist/bin/api-server.cjs.map +0 -1
- package/dist/bin/api-server.js.map +0 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +0 -1
- package/dist/bin/mcp-scraper-cli.js.map +0 -1
- package/dist/bin/mcp-scraper-install.cjs.map +0 -1
- package/dist/bin/mcp-scraper-install.js.map +0 -1
- package/dist/bin/mcp-stdio-server.cjs.map +0 -1
- package/dist/bin/mcp-stdio-server.js.map +0 -1
- package/dist/bin/paa-harvest.cjs.map +0 -1
- package/dist/bin/paa-harvest.js.map +0 -1
- package/dist/chunk-2XUXVKT4.js.map +0 -1
- package/dist/chunk-342OUCSG.js.map +0 -1
- package/dist/chunk-4LINSJNI.js.map +0 -1
- package/dist/chunk-4VG3NUUY.js.map +0 -1
- package/dist/chunk-5BEPOFKG.js.map +0 -1
- package/dist/chunk-5PIO7QBG.js.map +0 -1
- package/dist/chunk-AMLFKPLL.js.map +0 -1
- package/dist/chunk-CGKWNXJR.js +0 -7
- package/dist/chunk-CGKWNXJR.js.map +0 -1
- package/dist/chunk-DCWXVAQT.js.map +0 -1
- package/dist/chunk-DKIG2TEE.js.map +0 -1
- package/dist/chunk-EKJ65E3Z.js.map +0 -1
- package/dist/chunk-FG6LJGJ7.js.map +0 -1
- package/dist/chunk-G3P3ZDB4.js.map +0 -1
- package/dist/chunk-MQGBCWZD.js.map +0 -1
- package/dist/chunk-MQQS7ONH.js.map +0 -1
- package/dist/chunk-N54JS5WP.js.map +0 -1
- package/dist/chunk-ORKXE7VX.js.map +0 -1
- package/dist/chunk-PDAKCYHZ.js.map +0 -1
- package/dist/chunk-UN4LZVCB.js.map +0 -1
- package/dist/chunk-UVFB24RZ.js +0 -67
- package/dist/chunk-UVFB24RZ.js.map +0 -1
- package/dist/chunk-VIWT35S6.js.map +0 -1
- package/dist/chunk-XPEN6OG5.js.map +0 -1
- package/dist/db-BRI465XQ.js.map +0 -1
- package/dist/extract-bundle-UWKJT4MU.js.map +0 -1
- package/dist/index.cjs.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lead-list-enrichment-repository-ODWGRVCQ.js.map +0 -1
- package/dist/location-data-repository-67MMEVZQ.js.map +0 -1
- package/dist/server-RSHSZBUZ.js.map +0 -1
- package/dist/site-extract-repository-J3RH3MOS.js.map +0 -1
- package/dist/worker-YKRIK5LS.js.map +0 -1
|
@@ -1,3 +1,65 @@
|
|
|
1
|
+
// src/api/url-utils.ts
|
|
2
|
+
import { isIP } from "net";
|
|
3
|
+
import { lookup } from "dns/promises";
|
|
4
|
+
import ipaddr from "ipaddr.js";
|
|
5
|
+
function unbracketIpLiteral(value) {
|
|
6
|
+
return value.startsWith("[") && value.endsWith("]") ? value.slice(1, -1) : value;
|
|
7
|
+
}
|
|
8
|
+
function isPrivateIpAddress(address) {
|
|
9
|
+
const normalized = unbracketIpLiteral(address);
|
|
10
|
+
if (!isIP(normalized)) return false;
|
|
11
|
+
try {
|
|
12
|
+
const parsed = ipaddr.parse(normalized);
|
|
13
|
+
const isExtraNonGlobalIpv4 = (ipv4) => {
|
|
14
|
+
const extraNonGlobal = [
|
|
15
|
+
["192.0.0.0", 24],
|
|
16
|
+
["192.0.2.0", 24],
|
|
17
|
+
["198.18.0.0", 15],
|
|
18
|
+
["198.51.100.0", 24],
|
|
19
|
+
["203.0.113.0", 24]
|
|
20
|
+
];
|
|
21
|
+
return extraNonGlobal.some(([network, bits]) => ipv4.match(ipaddr.parse(network), bits));
|
|
22
|
+
};
|
|
23
|
+
if (parsed instanceof ipaddr.IPv6 && parsed.isIPv4MappedAddress()) {
|
|
24
|
+
const mapped = parsed.toIPv4Address();
|
|
25
|
+
return mapped.range() !== "unicast" || isExtraNonGlobalIpv4(mapped);
|
|
26
|
+
}
|
|
27
|
+
if (parsed instanceof ipaddr.IPv4) {
|
|
28
|
+
if (isExtraNonGlobalIpv4(parsed)) return true;
|
|
29
|
+
}
|
|
30
|
+
return parsed.range() !== "unicast";
|
|
31
|
+
} catch {
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
async function resolvesToPrivateAddress(hostname) {
|
|
36
|
+
const host = unbracketIpLiteral(hostname.toLowerCase());
|
|
37
|
+
if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) return true;
|
|
38
|
+
if (isPrivateIpAddress(host)) return true;
|
|
39
|
+
try {
|
|
40
|
+
const addresses = await lookup(host, { all: true, verbatim: true });
|
|
41
|
+
return addresses.length === 0 || addresses.some((entry) => isPrivateIpAddress(entry.address));
|
|
42
|
+
} catch {
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
async function validatePublicHttpUrl(raw, opts) {
|
|
47
|
+
let parsed;
|
|
48
|
+
try {
|
|
49
|
+
parsed = new URL(raw.trim());
|
|
50
|
+
} catch {
|
|
51
|
+
return { error: `Invalid ${opts.field}` };
|
|
52
|
+
}
|
|
53
|
+
const allowedProtocols = opts.requireHttps ? ["https:"] : ["http:", "https:"];
|
|
54
|
+
if (!allowedProtocols.includes(parsed.protocol)) {
|
|
55
|
+
return { error: opts.requireHttps ? `${opts.field} must use https` : `${opts.field} must use http or https` };
|
|
56
|
+
}
|
|
57
|
+
if (await resolvesToPrivateAddress(parsed.hostname)) {
|
|
58
|
+
return { error: `${opts.field} must resolve to a public internet host` };
|
|
59
|
+
}
|
|
60
|
+
return { parsed };
|
|
61
|
+
}
|
|
62
|
+
|
|
1
63
|
// src/lib/html-document.ts
|
|
2
64
|
import { load } from "cheerio";
|
|
3
65
|
var REMOVED_CONTENT = "script,style,noscript,svg,iframe,canvas,nav,header,footer,aside";
|
|
@@ -147,6 +209,7 @@ function loadHtmlDocument(sourceHtml) {
|
|
|
147
209
|
}
|
|
148
210
|
|
|
149
211
|
export {
|
|
212
|
+
isPrivateIpAddress,
|
|
213
|
+
validatePublicHttpUrl,
|
|
150
214
|
loadHtmlDocument
|
|
151
215
|
};
|
|
152
|
-
//# sourceMappingURL=chunk-MQGBCWZD.js.map
|
|
@@ -186,8 +186,9 @@ function publicErrorMessage(code) {
|
|
|
186
186
|
case "proxy_unavailable":
|
|
187
187
|
return "The web request could not establish a working connection. Please retry later.";
|
|
188
188
|
case "harvest_timeout":
|
|
189
|
-
case "mcp_request_timeout":
|
|
190
189
|
return "The request timed out before it completed. Please retry with a smaller request.";
|
|
190
|
+
case "mcp_request_timeout":
|
|
191
|
+
return "The MCP caller stopped waiting before a final response arrived. Durable work may still be running; continue with the returned Task or job handle instead of restarting it.";
|
|
191
192
|
case "concurrency_limit_exceeded":
|
|
192
193
|
return "The service is available, but this account has reached its parallel-operation limit. Wait for an active operation to finish or add capacity.";
|
|
193
194
|
case "unauthorized":
|
|
@@ -3573,4 +3574,3 @@ export {
|
|
|
3573
3574
|
insertInboxMessage,
|
|
3574
3575
|
listActiveUserIds
|
|
3575
3576
|
};
|
|
3576
|
-
//# sourceMappingURL=chunk-4VG3NUUY.js.map
|
|
@@ -1,158 +1,17 @@
|
|
|
1
|
-
import {
|
|
2
|
-
loadHtmlDocument
|
|
3
|
-
} from "./chunk-MQGBCWZD.js";
|
|
4
1
|
import {
|
|
5
2
|
rowsToCsv
|
|
6
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-RJ7JVYKU.js";
|
|
7
4
|
import {
|
|
8
5
|
DEFAULT_MAPS_PROXY_MODE,
|
|
9
6
|
postToMemoryLibrary
|
|
10
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-77LQV73B.js";
|
|
11
8
|
import {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
var MAX_CONTACT_FORMS = 10;
|
|
19
|
-
var MAX_CANDIDATE_PAGE_URLS = 20;
|
|
20
|
-
var MAX_CONTEXT_CHARS = 280;
|
|
21
|
-
var EMAIL_PATTERN = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
|
|
22
|
-
var EMAIL_NOISE = [
|
|
23
|
-
".png",
|
|
24
|
-
".jpg",
|
|
25
|
-
".jpeg",
|
|
26
|
-
".gif",
|
|
27
|
-
".svg",
|
|
28
|
-
".webp",
|
|
29
|
-
".ico",
|
|
30
|
-
"@2x",
|
|
31
|
-
"sentry",
|
|
32
|
-
"wix.com",
|
|
33
|
-
"wixpress",
|
|
34
|
-
"example.",
|
|
35
|
-
"godaddy",
|
|
36
|
-
"squarespace",
|
|
37
|
-
"schema.org",
|
|
38
|
-
"u003e",
|
|
39
|
-
"core-js"
|
|
40
|
-
];
|
|
41
|
-
var DEVELOPER_CONTEXT = /\b(?:website|site|web)\s+(?:by|built|design(?:ed)?|develop(?:ed)?|powered)\b|\b(?:developer|webmaster)\b/i;
|
|
42
|
-
var CONTACT_PATH = /(?:^|\/)(?:about|bio|contact|contact-us|meet|our-team|people|profile|staff|team)(?:\/|$)/i;
|
|
43
|
-
var SOCIAL_HOSTS = {
|
|
44
|
-
"facebook.com": "facebook",
|
|
45
|
-
"fb.com": "facebook",
|
|
46
|
-
"instagram.com": "instagram",
|
|
47
|
-
"linkedin.com": "linkedin",
|
|
48
|
-
"twitter.com": "twitter",
|
|
49
|
-
"x.com": "x",
|
|
50
|
-
"youtube.com": "youtube",
|
|
51
|
-
"youtu.be": "youtube",
|
|
52
|
-
"tiktok.com": "tiktok",
|
|
53
|
-
"pinterest.com": "pinterest"
|
|
54
|
-
};
|
|
55
|
-
var SOCIAL_NOISE = /(?:\/shar(?:e|er)|\/intent\/|\/dialog\/|\/plugins\/|\/share\?|\/developers?\/|platform\.)/i;
|
|
56
|
-
function compact(value) {
|
|
57
|
-
return (value ?? "").replace(/\s+/g, " ").trim();
|
|
58
|
-
}
|
|
59
|
-
function safeHttpUrl(value, base) {
|
|
60
|
-
try {
|
|
61
|
-
const parsed = new URL(value, base);
|
|
62
|
-
if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
|
|
63
|
-
return parsed;
|
|
64
|
-
} catch {
|
|
65
|
-
return null;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
function normalizeEmail(value) {
|
|
69
|
-
let decoded = value;
|
|
70
|
-
try {
|
|
71
|
-
decoded = decodeURIComponent(value);
|
|
72
|
-
} catch {
|
|
73
|
-
}
|
|
74
|
-
const email = decoded.split("?")[0].trim().toLowerCase().replace(/[),.;:]+$/, "");
|
|
75
|
-
if (email.length > 254 || email.split("@").length !== 2) return null;
|
|
76
|
-
if (!/^[a-z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-z0-9.-]+\.[a-z]{2,}$/i.test(email)) return null;
|
|
77
|
-
if (EMAIL_NOISE.some((part) => email.includes(part))) return null;
|
|
78
|
-
return email;
|
|
79
|
-
}
|
|
80
|
-
function nearbyContext(node) {
|
|
81
|
-
const parent = node.closest("p,li,address,section,article,div").first();
|
|
82
|
-
return compact(parent.length ? parent.text() : node.text()).slice(0, MAX_CONTEXT_CHARS);
|
|
83
|
-
}
|
|
84
|
-
function addEmail(target, raw, sourceUrl, context) {
|
|
85
|
-
const value = normalizeEmail(raw);
|
|
86
|
-
if (!value || DEVELOPER_CONTEXT.test(context)) return;
|
|
87
|
-
if (!target.has(value)) {
|
|
88
|
-
target.set(value, {
|
|
89
|
-
value,
|
|
90
|
-
sourceUrl,
|
|
91
|
-
context: compact(context).slice(0, MAX_CONTEXT_CHARS),
|
|
92
|
-
sourceType: "first_party_page"
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
function normalizedSocialUrl(parsed) {
|
|
97
|
-
parsed.hash = "";
|
|
98
|
-
for (const key of [...parsed.searchParams.keys()]) {
|
|
99
|
-
if (/^(?:utm_|fbclid|gclid)/i.test(key)) parsed.searchParams.delete(key);
|
|
100
|
-
}
|
|
101
|
-
return parsed.href;
|
|
102
|
-
}
|
|
103
|
-
function socialNetwork(hostname) {
|
|
104
|
-
const host = hostname.toLowerCase().replace(/^www\./, "");
|
|
105
|
-
const match = Object.entries(SOCIAL_HOSTS).find(([domain]) => host === domain || host.endsWith(`.${domain}`));
|
|
106
|
-
return match?.[1] ?? null;
|
|
107
|
-
}
|
|
108
|
-
function extractContactEvidence(html, pageUrl) {
|
|
109
|
-
const source = safeHttpUrl(pageUrl);
|
|
110
|
-
if (!source) throw new Error("pageUrl must be a public HTTP(S) URL");
|
|
111
|
-
const { $ } = loadHtmlDocument(html);
|
|
112
|
-
const emails = /* @__PURE__ */ new Map();
|
|
113
|
-
const socialProfiles = /* @__PURE__ */ new Map();
|
|
114
|
-
const contactForms = /* @__PURE__ */ new Map();
|
|
115
|
-
const candidatePageUrls = /* @__PURE__ */ new Map();
|
|
116
|
-
$('a[href^="mailto:" i]').each((_index, element) => {
|
|
117
|
-
const node = $(element);
|
|
118
|
-
addEmail(emails, (node.attr("href") ?? "").slice(7), source.href, nearbyContext(node));
|
|
119
|
-
});
|
|
120
|
-
$("body").find("*").addBack().contents().filter((_index, node) => node.type === "text").each((_index, node) => {
|
|
121
|
-
const parent = $(node.parent ?? node);
|
|
122
|
-
const text = String(node.data ?? "");
|
|
123
|
-
for (const match of text.match(EMAIL_PATTERN) ?? []) addEmail(emails, match, source.href, nearbyContext(parent));
|
|
124
|
-
});
|
|
125
|
-
$("a[href]").each((_index, element) => {
|
|
126
|
-
const node = $(element);
|
|
127
|
-
const parsed = safeHttpUrl(node.attr("href") ?? "", source.href);
|
|
128
|
-
if (!parsed) return;
|
|
129
|
-
const context = compact(`${node.text()} ${nearbyContext(node)}`).slice(0, MAX_CONTEXT_CHARS);
|
|
130
|
-
const network = socialNetwork(parsed.hostname);
|
|
131
|
-
if (network && !SOCIAL_NOISE.test(`${parsed.pathname}${parsed.search}`)) {
|
|
132
|
-
const url2 = normalizedSocialUrl(parsed);
|
|
133
|
-
if (!socialProfiles.has(url2)) socialProfiles.set(url2, { url: url2, network, sourceUrl: source.href, context });
|
|
134
|
-
return;
|
|
135
|
-
}
|
|
136
|
-
if (parsed.origin !== source.origin || !CONTACT_PATH.test(parsed.pathname)) return;
|
|
137
|
-
parsed.hash = "";
|
|
138
|
-
const url = parsed.href;
|
|
139
|
-
if (!candidatePageUrls.has(url)) candidatePageUrls.set(url, { url, sourceUrl: source.href, context });
|
|
140
|
-
});
|
|
141
|
-
$("form").each((_index, element) => {
|
|
142
|
-
const node = $(element);
|
|
143
|
-
const action = safeHttpUrl(node.attr("action") || source.href, source.href);
|
|
144
|
-
if (!action || action.origin !== source.origin) return;
|
|
145
|
-
action.hash = "";
|
|
146
|
-
const url = action.href;
|
|
147
|
-
if (!contactForms.has(url)) contactForms.set(url, { url, sourceUrl: source.href, context: nearbyContext(node) });
|
|
148
|
-
});
|
|
149
|
-
return {
|
|
150
|
-
emails: [...emails.values()].slice(0, MAX_EMAILS),
|
|
151
|
-
socialProfiles: [...socialProfiles.values()].slice(0, MAX_SOCIAL_PROFILES),
|
|
152
|
-
contactForms: [...contactForms.values()].slice(0, MAX_CONTACT_FORMS),
|
|
153
|
-
candidatePageUrls: [...candidatePageUrls.values()].slice(0, MAX_CANDIDATE_PAGE_URLS)
|
|
154
|
-
};
|
|
155
|
-
}
|
|
9
|
+
buildSerpEmailQuery,
|
|
10
|
+
contactAttemptUrls,
|
|
11
|
+
extractContactEvidence,
|
|
12
|
+
extractSerpEmailEvidence,
|
|
13
|
+
selectPrimaryEmail
|
|
14
|
+
} from "./chunk-VXLU74YZ.js";
|
|
156
15
|
|
|
157
16
|
// src/lib/slugify.ts
|
|
158
17
|
function slugify(s) {
|
|
@@ -753,146 +612,6 @@ ${summary}
|
|
|
753
612
|
|
|
754
613
|
// src/workflows/workflows/get-leads.ts
|
|
755
614
|
import { z as z3 } from "zod";
|
|
756
|
-
|
|
757
|
-
// src/leads/contact-enrichment.ts
|
|
758
|
-
var GENERIC_EMAIL_PREFIXES = /* @__PURE__ */ new Set([
|
|
759
|
-
"admin",
|
|
760
|
-
"contact",
|
|
761
|
-
"customerservice",
|
|
762
|
-
"hello",
|
|
763
|
-
"help",
|
|
764
|
-
"info",
|
|
765
|
-
"inquiries",
|
|
766
|
-
"mail",
|
|
767
|
-
"office",
|
|
768
|
-
"reception",
|
|
769
|
-
"sales",
|
|
770
|
-
"service",
|
|
771
|
-
"support",
|
|
772
|
-
"team"
|
|
773
|
-
]);
|
|
774
|
-
var CONTACT_PATH2 = /(?:^|\/)(?:about|bio|contact|contact-us|meet|our-team|people|profile|staff|team)(?:\/|$)/i;
|
|
775
|
-
var TEAM_CONTEXT = /\b(?:our team|meet the team|staff|agents?|brokers?|realtors?|people)\b/i;
|
|
776
|
-
var OFFICE_CONTEXT = /\b(?:brokerage|company|corporate|front desk|main office|office|organization|reception)\b/i;
|
|
777
|
-
function normalizeWords(value) {
|
|
778
|
-
return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().split(/\s+/).filter(Boolean);
|
|
779
|
-
}
|
|
780
|
-
function safeHttpUrl2(value, base) {
|
|
781
|
-
try {
|
|
782
|
-
const parsed = new URL(value, base);
|
|
783
|
-
if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
|
|
784
|
-
return parsed;
|
|
785
|
-
} catch {
|
|
786
|
-
return null;
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
function classifyEmailScope(evidence, entity) {
|
|
790
|
-
const local = evidence.value.split("@")[0].toLowerCase().replace(/[^a-z0-9]+/g, " ");
|
|
791
|
-
const nameWords = normalizeWords(entity.name).filter((word) => word.length > 1);
|
|
792
|
-
const pagePath = safeHttpUrl2(evidence.sourceUrl)?.pathname.toLowerCase() ?? "";
|
|
793
|
-
const combinedContext = `${evidence.context} ${pagePath}`.toLowerCase();
|
|
794
|
-
const generic = GENERIC_EMAIL_PREFIXES.has(local.replace(/\s+/g, ""));
|
|
795
|
-
const nameHits = nameWords.filter((word) => new RegExp(`(?:^|[^a-z0-9])${word}(?:$|[^a-z0-9])`, "i").test(`${local} ${combinedContext}`)).length;
|
|
796
|
-
const hasExactPersonContext = entity.entityType === "person" && nameWords.length >= 2 && nameHits >= Math.min(2, nameWords.length) && (local.includes(nameWords[0]) || pagePath.includes(nameWords.join("-")) || pagePath.includes(nameWords.join("/")));
|
|
797
|
-
if (!generic && hasExactPersonContext) return { scope: "person", reasons: ["named_person_page_context"] };
|
|
798
|
-
if (generic || OFFICE_CONTEXT.test(combinedContext)) return { scope: "office", reasons: generic ? ["generic_mailbox_prefix"] : ["organization_page_context"] };
|
|
799
|
-
if (TEAM_CONTEXT.test(combinedContext)) return { scope: "team", reasons: ["team_page_context"] };
|
|
800
|
-
if (entity.entityType === "business") {
|
|
801
|
-
const organizationWords = normalizeWords(entity.organization || entity.name).filter((word) => word.length > 2);
|
|
802
|
-
if (organizationWords.some((word) => combinedContext.includes(word))) return { scope: "team", reasons: ["business_context"] };
|
|
803
|
-
}
|
|
804
|
-
return { scope: "unknown", reasons: ["public_occurrence_without_identity_evidence"] };
|
|
805
|
-
}
|
|
806
|
-
function selectPrimaryEmail(evidence, entity) {
|
|
807
|
-
const scoped = evidence.map((item) => ({ ...item, ...classifyEmailScope(item, entity) }));
|
|
808
|
-
const allowed = entity.entityType === "person" ? ["person", "team"] : ["person", "team", "office"];
|
|
809
|
-
const sourcePriority = ["provided", "first_party_page", "structured_data", "serp_snippet"];
|
|
810
|
-
const priority = ["person", "team", "office"];
|
|
811
|
-
return scoped.filter((item) => allowed.includes(item.scope)).sort((a, b) => {
|
|
812
|
-
const sourceDelta = sourcePriority.indexOf(a.sourceType) - sourcePriority.indexOf(b.sourceType);
|
|
813
|
-
return sourceDelta || priority.indexOf(a.scope) - priority.indexOf(b.scope);
|
|
814
|
-
})[0] ?? null;
|
|
815
|
-
}
|
|
816
|
-
function contactAttemptUrls(seedUrl, discoveredLinks, maxPages) {
|
|
817
|
-
if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 4) throw new Error("maxPages must be an integer from 1 to 4");
|
|
818
|
-
const seed = safeHttpUrl2(seedUrl);
|
|
819
|
-
if (!seed) return [];
|
|
820
|
-
seed.hash = "";
|
|
821
|
-
const urls = /* @__PURE__ */ new Set([seed.href]);
|
|
822
|
-
const discovered = discoveredLinks.map((item) => typeof item === "string" ? item : item.url).map((item) => safeHttpUrl2(item, seed.href)).filter((item) => Boolean(item && item.origin === seed.origin && CONTACT_PATH2.test(item.pathname)));
|
|
823
|
-
for (const item of discovered) {
|
|
824
|
-
item.hash = "";
|
|
825
|
-
urls.add(item.href);
|
|
826
|
-
}
|
|
827
|
-
for (const pathname of ["/contact", "/contact-us"]) urls.add(new URL(pathname, seed.origin).href);
|
|
828
|
-
return [...urls].slice(0, maxPages);
|
|
829
|
-
}
|
|
830
|
-
|
|
831
|
-
// src/leads/serp-email-evidence.ts
|
|
832
|
-
var EMAIL_CANDIDATE = /[A-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
|
|
833
|
-
var DEVELOPER_CONTEXT2 = /\b(?:website|site|web)\s+(?:by|built|design(?:ed)?|develop(?:ed)?|powered)\b|\b(?:developer|webmaster)\b/i;
|
|
834
|
-
var MAX_CONTEXT_CHARS2 = 280;
|
|
835
|
-
var MAX_EMAILS2 = 10;
|
|
836
|
-
function normalizeHostname(hostname) {
|
|
837
|
-
return hostname.toLowerCase().replace(/^www\./, "").replace(/\.$/, "");
|
|
838
|
-
}
|
|
839
|
-
function isLocalHostname(hostname) {
|
|
840
|
-
const host = normalizeHostname(hostname);
|
|
841
|
-
return host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") || !host.includes(".") && !host.includes(":");
|
|
842
|
-
}
|
|
843
|
-
function parsePublicLiteralUrl(websiteUrl) {
|
|
844
|
-
try {
|
|
845
|
-
const parsed = new URL(websiteUrl.trim());
|
|
846
|
-
if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
|
|
847
|
-
if (isLocalHostname(parsed.hostname) || isPrivateIpAddress(parsed.hostname)) return null;
|
|
848
|
-
return parsed;
|
|
849
|
-
} catch {
|
|
850
|
-
return null;
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
function compactContext(title, snippet) {
|
|
854
|
-
return `${title} ${snippet ?? ""}`.replace(/\s+/g, " ").trim().slice(0, MAX_CONTEXT_CHARS2);
|
|
855
|
-
}
|
|
856
|
-
function buildSerpEmailQuery(websiteUrl) {
|
|
857
|
-
const parsed = parsePublicLiteralUrl(websiteUrl);
|
|
858
|
-
if (!parsed) return null;
|
|
859
|
-
parsed.hostname = normalizeHostname(parsed.hostname);
|
|
860
|
-
return `email for ${parsed.protocol}//${parsed.host}/`;
|
|
861
|
-
}
|
|
862
|
-
function extractSerpEmailEvidence({
|
|
863
|
-
websiteUrl,
|
|
864
|
-
results
|
|
865
|
-
}) {
|
|
866
|
-
const parsedWebsite = parsePublicLiteralUrl(websiteUrl);
|
|
867
|
-
if (!parsedWebsite) return [];
|
|
868
|
-
const websiteDomain = normalizeHostname(parsedWebsite.hostname);
|
|
869
|
-
const accepted = /* @__PURE__ */ new Map();
|
|
870
|
-
const orderedResults = results.map((result, index) => ({ result, index })).sort((left, right) => {
|
|
871
|
-
const leftPosition = Number.isFinite(left.result.position) ? left.result.position : Number.MAX_SAFE_INTEGER;
|
|
872
|
-
const rightPosition = Number.isFinite(right.result.position) ? right.result.position : Number.MAX_SAFE_INTEGER;
|
|
873
|
-
return leftPosition - rightPosition || left.index - right.index;
|
|
874
|
-
});
|
|
875
|
-
for (const { result } of orderedResults) {
|
|
876
|
-
const searchableText = `${result.title} ${result.snippet ?? ""}`;
|
|
877
|
-
const context = compactContext(result.title, result.snippet);
|
|
878
|
-
for (const rawCandidate of searchableText.match(EMAIL_CANDIDATE) ?? []) {
|
|
879
|
-
const value = normalizeEmail(rawCandidate);
|
|
880
|
-
if (!value || DEVELOPER_CONTEXT2.test(searchableText) || normalizeHostname(value.slice(value.lastIndexOf("@") + 1)) !== websiteDomain) continue;
|
|
881
|
-
if (accepted.has(value)) continue;
|
|
882
|
-
accepted.set(value, {
|
|
883
|
-
value,
|
|
884
|
-
sourceUrl: result.url,
|
|
885
|
-
sourceTitle: result.title,
|
|
886
|
-
sourcePosition: result.position,
|
|
887
|
-
context,
|
|
888
|
-
sourceType: "serp_snippet"
|
|
889
|
-
});
|
|
890
|
-
}
|
|
891
|
-
}
|
|
892
|
-
return [...accepted.values()].slice(0, MAX_EMAILS2);
|
|
893
|
-
}
|
|
894
|
-
|
|
895
|
-
// src/workflows/workflows/get-leads.ts
|
|
896
615
|
var GetLeadsInputSchema = z3.object({
|
|
897
616
|
query: z3.string().min(1).describe('Business category, niche, or keyword to search on Google Maps, e.g. "roofers", "med spas", "dentists". Do not include the city here.'),
|
|
898
617
|
location: z3.string().min(1).describe('City / market to search, e.g. "Houston, TX" or "Austin, Texas".'),
|
|
@@ -2182,12 +1901,6 @@ async function runWorkflowStep(id, rawInput, opts) {
|
|
|
2182
1901
|
}
|
|
2183
1902
|
|
|
2184
1903
|
export {
|
|
2185
|
-
extractContactEvidence,
|
|
2186
|
-
classifyEmailScope,
|
|
2187
|
-
selectPrimaryEmail,
|
|
2188
|
-
contactAttemptUrls,
|
|
2189
|
-
buildSerpEmailQuery,
|
|
2190
|
-
extractSerpEmailEvidence,
|
|
2191
1904
|
slugify,
|
|
2192
1905
|
workflowOutputBaseDir,
|
|
2193
1906
|
listWorkflowReports,
|
|
@@ -2200,4 +1913,3 @@ export {
|
|
|
2200
1913
|
runWorkflow,
|
|
2201
1914
|
runWorkflowStep
|
|
2202
1915
|
};
|
|
2203
|
-
//# sourceMappingURL=chunk-MQQS7ONH.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
isVendorUnavailableError
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-OZJMVCDK.js";
|
|
4
4
|
import {
|
|
5
5
|
CaptchaError,
|
|
6
6
|
LocationMismatchError,
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
finishHarvestAttempt,
|
|
11
11
|
publicErrorMessage,
|
|
12
12
|
startHarvestAttempt
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-GTGSD4YS.js";
|
|
14
14
|
|
|
15
15
|
// src/api/webhook.ts
|
|
16
16
|
async function deliverWebhook(url, payload, retries = 3) {
|
|
@@ -52,6 +52,9 @@ function looksLikeProxyUnavailable(message) {
|
|
|
52
52
|
function looksLikeInternalServiceUnavailable(message) {
|
|
53
53
|
return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
|
|
54
54
|
}
|
|
55
|
+
function looksLikeBrowserSessionInterrupted(message) {
|
|
56
|
+
return /browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message);
|
|
57
|
+
}
|
|
55
58
|
function classifyHarvestProblem(err) {
|
|
56
59
|
const message = errorMessage(err);
|
|
57
60
|
if (err instanceof RequestAbortedError) {
|
|
@@ -114,6 +117,16 @@ function classifyHarvestProblem(err) {
|
|
|
114
117
|
terminalStatus: "failed"
|
|
115
118
|
};
|
|
116
119
|
}
|
|
120
|
+
if (looksLikeBrowserSessionInterrupted(message)) {
|
|
121
|
+
return {
|
|
122
|
+
error_code: "browser_session_interrupted",
|
|
123
|
+
error_type: "extraction",
|
|
124
|
+
message: publicErrorMessage("browser_session_interrupted"),
|
|
125
|
+
retryable: true,
|
|
126
|
+
httpStatus: 503,
|
|
127
|
+
terminalStatus: "failed"
|
|
128
|
+
};
|
|
129
|
+
}
|
|
117
130
|
if (looksLikeTimeout(err, message)) {
|
|
118
131
|
return {
|
|
119
132
|
error_code: "harvest_timeout",
|
|
@@ -222,4 +235,3 @@ export {
|
|
|
222
235
|
publicErrorBoundary,
|
|
223
236
|
createHarvestAttemptRecorder
|
|
224
237
|
};
|
|
225
|
-
//# sourceMappingURL=chunk-XPEN6OG5.js.map
|
|
@@ -8,10 +8,9 @@ import {
|
|
|
8
8
|
} from "@onkernel/sdk";
|
|
9
9
|
var BRIGHTDATA_NOT_CONFIGURED_MESSAGE = "SERP browser service is not configured";
|
|
10
10
|
function isVendorUnavailableError(err, message) {
|
|
11
|
-
return err instanceof KernelAuthenticationError || err instanceof KernelPermissionDeniedError || err instanceof KernelAPIConnectionError || err instanceof KernelAPIConnectionTimeoutError || err instanceof KernelInternalServerError || message === BRIGHTDATA_NOT_CONFIGURED_MESSAGE;
|
|
11
|
+
return err instanceof KernelAuthenticationError || err instanceof KernelPermissionDeniedError || err instanceof KernelAPIConnectionError || err instanceof KernelAPIConnectionTimeoutError || err instanceof KernelInternalServerError || /^401\s+Invalid or disabled API key\s*$/i.test(message.trim()) || message === BRIGHTDATA_NOT_CONFIGURED_MESSAGE;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
export {
|
|
15
15
|
isVendorUnavailableError
|
|
16
16
|
};
|
|
17
|
-
//# sourceMappingURL=chunk-UN4LZVCB.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
validatePublicHttpUrl
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-GGZEC22A.js";
|
|
4
4
|
|
|
5
5
|
// src/api/image-audit.ts
|
|
6
6
|
var OVER_BYTES = 100 * 1024;
|
|
@@ -381,4 +381,3 @@ export {
|
|
|
381
381
|
computeIssues,
|
|
382
382
|
renderIssueReport
|
|
383
383
|
};
|
|
384
|
-
//# sourceMappingURL=chunk-342OUCSG.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getDb
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-GTGSD4YS.js";
|
|
4
4
|
|
|
5
5
|
// src/api/lead-list-enrichment-repository.ts
|
|
6
6
|
import { createHash, randomUUID } from "crypto";
|
|
@@ -590,4 +590,3 @@ export {
|
|
|
590
590
|
markLeadListEnrichmentOutboxDispatched,
|
|
591
591
|
markLeadListEnrichmentOutboxFailed
|
|
592
592
|
};
|
|
593
|
-
//# sourceMappingURL=chunk-N54JS5WP.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getDb
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-GTGSD4YS.js";
|
|
4
4
|
|
|
5
5
|
// src/lib/browser-service-env.ts
|
|
6
6
|
function browserServiceApiKey() {
|
|
@@ -275,4 +275,3 @@ export {
|
|
|
275
275
|
recordKernelSession,
|
|
276
276
|
recordVendorUsage
|
|
277
277
|
};
|
|
278
|
-
//# sourceMappingURL=chunk-4LINSJNI.js.map
|