mcp-scraper 0.65.1 → 0.66.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +31 -1
  2. package/README.md +11 -6
  3. package/dist/{analytics-repository-IKE53S3E.js → analytics-repository-RUXBFSFX.js} +1 -2
  4. package/dist/bin/api-server.cjs +1844 -71
  5. package/dist/bin/api-server.js +4 -5
  6. package/dist/bin/mcp-scraper-cli.cjs +1 -2
  7. package/dist/bin/mcp-scraper-cli.js +6 -7
  8. package/dist/bin/mcp-scraper-install.cjs +7 -3
  9. package/dist/bin/mcp-scraper-install.js +2 -3
  10. package/dist/bin/mcp-stdio-server.cjs +5150 -3467
  11. package/dist/bin/mcp-stdio-server.js +11 -11
  12. package/dist/bin/paa-harvest.cjs +8 -3
  13. package/dist/bin/paa-harvest.js +4 -5
  14. package/dist/{chunk-5PIO7QBG.js → chunk-3PIXKLGZ.js} +3 -4
  15. package/dist/{chunk-VIWT35S6.js → chunk-4OT7BAIE.js} +1 -2
  16. package/dist/{chunk-ORKXE7VX.js → chunk-4P3PHFPQ.js} +11 -6
  17. package/dist/{chunk-AMLFKPLL.js → chunk-75VW7CWE.js} +4 -7
  18. package/dist/{chunk-FG6LJGJ7.js → chunk-77LQV73B.js} +0 -1
  19. package/dist/{chunk-DKIG2TEE.js → chunk-AQKJOSIB.js} +2 -3
  20. package/dist/{chunk-2XUXVKT4.js → chunk-AYJHI2FK.js} +0 -1
  21. package/dist/{chunk-5BEPOFKG.js → chunk-CNLLTD5B.js} +0 -1
  22. package/dist/{chunk-EKJ65E3Z.js → chunk-DNM65UCK.js} +0 -1
  23. package/dist/{chunk-PDAKCYHZ.js → chunk-E4LMY5OJ.js} +6258 -3747
  24. package/dist/{chunk-MQGBCWZD.js → chunk-GGZEC22A.js} +64 -1
  25. package/dist/{chunk-4VG3NUUY.js → chunk-GTGSD4YS.js} +2 -2
  26. package/dist/{chunk-MQQS7ONH.js → chunk-KBTUPRTF.js} +8 -296
  27. package/dist/{chunk-XPEN6OG5.js → chunk-NS754CRS.js} +15 -3
  28. package/dist/{chunk-UN4LZVCB.js → chunk-OZJMVCDK.js} +1 -2
  29. package/dist/{chunk-342OUCSG.js → chunk-PGJQDMC2.js} +1 -2
  30. package/dist/{chunk-G3P3ZDB4.js → chunk-RJ7JVYKU.js} +0 -1
  31. package/dist/{chunk-N54JS5WP.js → chunk-TNJIPV74.js} +1 -2
  32. package/dist/{chunk-4LINSJNI.js → chunk-TNLGHNLX.js} +1 -2
  33. package/dist/chunk-VRJ7CO6L.js +6 -0
  34. package/dist/chunk-VXLU74YZ.js +293 -0
  35. package/dist/{chunk-DCWXVAQT.js → chunk-WID3MFRS.js} +6 -2
  36. package/dist/{db-BRI465XQ.js → db-PO7N7ZP3.js} +1 -2
  37. package/dist/{extract-bundle-UWKJT4MU.js → extract-bundle-SIBFUA6I.js} +8 -10
  38. package/dist/index.cjs +8 -3
  39. package/dist/index.js +4 -5
  40. package/dist/{lead-list-enrichment-repository-ODWGRVCQ.js → lead-list-enrichment-repository-OTAUQONJ.js} +2 -3
  41. package/dist/{location-data-repository-67MMEVZQ.js → location-data-repository-RM2KJ3H5.js} +3 -4
  42. package/dist/{server-RSHSZBUZ.js → server-ELEJNRRE.js} +875 -1629
  43. package/dist/{site-extract-repository-J3RH3MOS.js → site-extract-repository-ASYEJO7Z.js} +4 -5
  44. package/dist/{worker-YKRIK5LS.js → worker-MAVYYWM7.js} +8 -9
  45. package/package.json +3 -1
  46. package/dist/analytics-repository-IKE53S3E.js.map +0 -1
  47. package/dist/bin/api-server.cjs.map +0 -1
  48. package/dist/bin/api-server.js.map +0 -1
  49. package/dist/bin/mcp-scraper-cli.cjs.map +0 -1
  50. package/dist/bin/mcp-scraper-cli.js.map +0 -1
  51. package/dist/bin/mcp-scraper-install.cjs.map +0 -1
  52. package/dist/bin/mcp-scraper-install.js.map +0 -1
  53. package/dist/bin/mcp-stdio-server.cjs.map +0 -1
  54. package/dist/bin/mcp-stdio-server.js.map +0 -1
  55. package/dist/bin/paa-harvest.cjs.map +0 -1
  56. package/dist/bin/paa-harvest.js.map +0 -1
  57. package/dist/chunk-2XUXVKT4.js.map +0 -1
  58. package/dist/chunk-342OUCSG.js.map +0 -1
  59. package/dist/chunk-4LINSJNI.js.map +0 -1
  60. package/dist/chunk-4VG3NUUY.js.map +0 -1
  61. package/dist/chunk-5BEPOFKG.js.map +0 -1
  62. package/dist/chunk-5PIO7QBG.js.map +0 -1
  63. package/dist/chunk-AMLFKPLL.js.map +0 -1
  64. package/dist/chunk-CGKWNXJR.js +0 -7
  65. package/dist/chunk-CGKWNXJR.js.map +0 -1
  66. package/dist/chunk-DCWXVAQT.js.map +0 -1
  67. package/dist/chunk-DKIG2TEE.js.map +0 -1
  68. package/dist/chunk-EKJ65E3Z.js.map +0 -1
  69. package/dist/chunk-FG6LJGJ7.js.map +0 -1
  70. package/dist/chunk-G3P3ZDB4.js.map +0 -1
  71. package/dist/chunk-MQGBCWZD.js.map +0 -1
  72. package/dist/chunk-MQQS7ONH.js.map +0 -1
  73. package/dist/chunk-N54JS5WP.js.map +0 -1
  74. package/dist/chunk-ORKXE7VX.js.map +0 -1
  75. package/dist/chunk-PDAKCYHZ.js.map +0 -1
  76. package/dist/chunk-UN4LZVCB.js.map +0 -1
  77. package/dist/chunk-UVFB24RZ.js +0 -67
  78. package/dist/chunk-UVFB24RZ.js.map +0 -1
  79. package/dist/chunk-VIWT35S6.js.map +0 -1
  80. package/dist/chunk-XPEN6OG5.js.map +0 -1
  81. package/dist/db-BRI465XQ.js.map +0 -1
  82. package/dist/extract-bundle-UWKJT4MU.js.map +0 -1
  83. package/dist/index.cjs.map +0 -1
  84. package/dist/index.js.map +0 -1
  85. package/dist/lead-list-enrichment-repository-ODWGRVCQ.js.map +0 -1
  86. package/dist/location-data-repository-67MMEVZQ.js.map +0 -1
  87. package/dist/server-RSHSZBUZ.js.map +0 -1
  88. package/dist/site-extract-repository-J3RH3MOS.js.map +0 -1
  89. package/dist/worker-YKRIK5LS.js.map +0 -1
@@ -1,3 +1,65 @@
1
+ // src/api/url-utils.ts
2
+ import { isIP } from "net";
3
+ import { lookup } from "dns/promises";
4
+ import ipaddr from "ipaddr.js";
5
+ function unbracketIpLiteral(value) {
6
+ return value.startsWith("[") && value.endsWith("]") ? value.slice(1, -1) : value;
7
+ }
8
+ function isPrivateIpAddress(address) {
9
+ const normalized = unbracketIpLiteral(address);
10
+ if (!isIP(normalized)) return false;
11
+ try {
12
+ const parsed = ipaddr.parse(normalized);
13
+ const isExtraNonGlobalIpv4 = (ipv4) => {
14
+ const extraNonGlobal = [
15
+ ["192.0.0.0", 24],
16
+ ["192.0.2.0", 24],
17
+ ["198.18.0.0", 15],
18
+ ["198.51.100.0", 24],
19
+ ["203.0.113.0", 24]
20
+ ];
21
+ return extraNonGlobal.some(([network, bits]) => ipv4.match(ipaddr.parse(network), bits));
22
+ };
23
+ if (parsed instanceof ipaddr.IPv6 && parsed.isIPv4MappedAddress()) {
24
+ const mapped = parsed.toIPv4Address();
25
+ return mapped.range() !== "unicast" || isExtraNonGlobalIpv4(mapped);
26
+ }
27
+ if (parsed instanceof ipaddr.IPv4) {
28
+ if (isExtraNonGlobalIpv4(parsed)) return true;
29
+ }
30
+ return parsed.range() !== "unicast";
31
+ } catch {
32
+ return true;
33
+ }
34
+ }
35
+ async function resolvesToPrivateAddress(hostname) {
36
+ const host = unbracketIpLiteral(hostname.toLowerCase());
37
+ if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) return true;
38
+ if (isPrivateIpAddress(host)) return true;
39
+ try {
40
+ const addresses = await lookup(host, { all: true, verbatim: true });
41
+ return addresses.length === 0 || addresses.some((entry) => isPrivateIpAddress(entry.address));
42
+ } catch {
43
+ return true;
44
+ }
45
+ }
46
+ async function validatePublicHttpUrl(raw, opts) {
47
+ let parsed;
48
+ try {
49
+ parsed = new URL(raw.trim());
50
+ } catch {
51
+ return { error: `Invalid ${opts.field}` };
52
+ }
53
+ const allowedProtocols = opts.requireHttps ? ["https:"] : ["http:", "https:"];
54
+ if (!allowedProtocols.includes(parsed.protocol)) {
55
+ return { error: opts.requireHttps ? `${opts.field} must use https` : `${opts.field} must use http or https` };
56
+ }
57
+ if (await resolvesToPrivateAddress(parsed.hostname)) {
58
+ return { error: `${opts.field} must resolve to a public internet host` };
59
+ }
60
+ return { parsed };
61
+ }
62
+
1
63
  // src/lib/html-document.ts
2
64
  import { load } from "cheerio";
3
65
  var REMOVED_CONTENT = "script,style,noscript,svg,iframe,canvas,nav,header,footer,aside";
@@ -147,6 +209,7 @@ function loadHtmlDocument(sourceHtml) {
147
209
  }
148
210
 
149
211
  export {
212
+ isPrivateIpAddress,
213
+ validatePublicHttpUrl,
150
214
  loadHtmlDocument
151
215
  };
152
- //# sourceMappingURL=chunk-MQGBCWZD.js.map
@@ -186,8 +186,9 @@ function publicErrorMessage(code) {
186
186
  case "proxy_unavailable":
187
187
  return "The web request could not establish a working connection. Please retry later.";
188
188
  case "harvest_timeout":
189
- case "mcp_request_timeout":
190
189
  return "The request timed out before it completed. Please retry with a smaller request.";
190
+ case "mcp_request_timeout":
191
+ return "The MCP caller stopped waiting before a final response arrived. Durable work may still be running; continue with the returned Task or job handle instead of restarting it.";
191
192
  case "concurrency_limit_exceeded":
192
193
  return "The service is available, but this account has reached its parallel-operation limit. Wait for an active operation to finish or add capacity.";
193
194
  case "unauthorized":
@@ -3573,4 +3574,3 @@ export {
3573
3574
  insertInboxMessage,
3574
3575
  listActiveUserIds
3575
3576
  };
3576
- //# sourceMappingURL=chunk-4VG3NUUY.js.map
@@ -1,158 +1,17 @@
1
- import {
2
- loadHtmlDocument
3
- } from "./chunk-MQGBCWZD.js";
4
1
  import {
5
2
  rowsToCsv
6
- } from "./chunk-G3P3ZDB4.js";
3
+ } from "./chunk-RJ7JVYKU.js";
7
4
  import {
8
5
  DEFAULT_MAPS_PROXY_MODE,
9
6
  postToMemoryLibrary
10
- } from "./chunk-FG6LJGJ7.js";
7
+ } from "./chunk-77LQV73B.js";
11
8
  import {
12
- isPrivateIpAddress
13
- } from "./chunk-UVFB24RZ.js";
14
-
15
- // src/lib/contact-evidence.ts
16
- var MAX_EMAILS = 20;
17
- var MAX_SOCIAL_PROFILES = 20;
18
- var MAX_CONTACT_FORMS = 10;
19
- var MAX_CANDIDATE_PAGE_URLS = 20;
20
- var MAX_CONTEXT_CHARS = 280;
21
- var EMAIL_PATTERN = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
22
- var EMAIL_NOISE = [
23
- ".png",
24
- ".jpg",
25
- ".jpeg",
26
- ".gif",
27
- ".svg",
28
- ".webp",
29
- ".ico",
30
- "@2x",
31
- "sentry",
32
- "wix.com",
33
- "wixpress",
34
- "example.",
35
- "godaddy",
36
- "squarespace",
37
- "schema.org",
38
- "u003e",
39
- "core-js"
40
- ];
41
- var DEVELOPER_CONTEXT = /\b(?:website|site|web)\s+(?:by|built|design(?:ed)?|develop(?:ed)?|powered)\b|\b(?:developer|webmaster)\b/i;
42
- var CONTACT_PATH = /(?:^|\/)(?:about|bio|contact|contact-us|meet|our-team|people|profile|staff|team)(?:\/|$)/i;
43
- var SOCIAL_HOSTS = {
44
- "facebook.com": "facebook",
45
- "fb.com": "facebook",
46
- "instagram.com": "instagram",
47
- "linkedin.com": "linkedin",
48
- "twitter.com": "twitter",
49
- "x.com": "x",
50
- "youtube.com": "youtube",
51
- "youtu.be": "youtube",
52
- "tiktok.com": "tiktok",
53
- "pinterest.com": "pinterest"
54
- };
55
- var SOCIAL_NOISE = /(?:\/shar(?:e|er)|\/intent\/|\/dialog\/|\/plugins\/|\/share\?|\/developers?\/|platform\.)/i;
56
- function compact(value) {
57
- return (value ?? "").replace(/\s+/g, " ").trim();
58
- }
59
- function safeHttpUrl(value, base) {
60
- try {
61
- const parsed = new URL(value, base);
62
- if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
63
- return parsed;
64
- } catch {
65
- return null;
66
- }
67
- }
68
- function normalizeEmail(value) {
69
- let decoded = value;
70
- try {
71
- decoded = decodeURIComponent(value);
72
- } catch {
73
- }
74
- const email = decoded.split("?")[0].trim().toLowerCase().replace(/[),.;:]+$/, "");
75
- if (email.length > 254 || email.split("@").length !== 2) return null;
76
- if (!/^[a-z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-z0-9.-]+\.[a-z]{2,}$/i.test(email)) return null;
77
- if (EMAIL_NOISE.some((part) => email.includes(part))) return null;
78
- return email;
79
- }
80
- function nearbyContext(node) {
81
- const parent = node.closest("p,li,address,section,article,div").first();
82
- return compact(parent.length ? parent.text() : node.text()).slice(0, MAX_CONTEXT_CHARS);
83
- }
84
- function addEmail(target, raw, sourceUrl, context) {
85
- const value = normalizeEmail(raw);
86
- if (!value || DEVELOPER_CONTEXT.test(context)) return;
87
- if (!target.has(value)) {
88
- target.set(value, {
89
- value,
90
- sourceUrl,
91
- context: compact(context).slice(0, MAX_CONTEXT_CHARS),
92
- sourceType: "first_party_page"
93
- });
94
- }
95
- }
96
- function normalizedSocialUrl(parsed) {
97
- parsed.hash = "";
98
- for (const key of [...parsed.searchParams.keys()]) {
99
- if (/^(?:utm_|fbclid|gclid)/i.test(key)) parsed.searchParams.delete(key);
100
- }
101
- return parsed.href;
102
- }
103
- function socialNetwork(hostname) {
104
- const host = hostname.toLowerCase().replace(/^www\./, "");
105
- const match = Object.entries(SOCIAL_HOSTS).find(([domain]) => host === domain || host.endsWith(`.${domain}`));
106
- return match?.[1] ?? null;
107
- }
108
- function extractContactEvidence(html, pageUrl) {
109
- const source = safeHttpUrl(pageUrl);
110
- if (!source) throw new Error("pageUrl must be a public HTTP(S) URL");
111
- const { $ } = loadHtmlDocument(html);
112
- const emails = /* @__PURE__ */ new Map();
113
- const socialProfiles = /* @__PURE__ */ new Map();
114
- const contactForms = /* @__PURE__ */ new Map();
115
- const candidatePageUrls = /* @__PURE__ */ new Map();
116
- $('a[href^="mailto:" i]').each((_index, element) => {
117
- const node = $(element);
118
- addEmail(emails, (node.attr("href") ?? "").slice(7), source.href, nearbyContext(node));
119
- });
120
- $("body").find("*").addBack().contents().filter((_index, node) => node.type === "text").each((_index, node) => {
121
- const parent = $(node.parent ?? node);
122
- const text = String(node.data ?? "");
123
- for (const match of text.match(EMAIL_PATTERN) ?? []) addEmail(emails, match, source.href, nearbyContext(parent));
124
- });
125
- $("a[href]").each((_index, element) => {
126
- const node = $(element);
127
- const parsed = safeHttpUrl(node.attr("href") ?? "", source.href);
128
- if (!parsed) return;
129
- const context = compact(`${node.text()} ${nearbyContext(node)}`).slice(0, MAX_CONTEXT_CHARS);
130
- const network = socialNetwork(parsed.hostname);
131
- if (network && !SOCIAL_NOISE.test(`${parsed.pathname}${parsed.search}`)) {
132
- const url2 = normalizedSocialUrl(parsed);
133
- if (!socialProfiles.has(url2)) socialProfiles.set(url2, { url: url2, network, sourceUrl: source.href, context });
134
- return;
135
- }
136
- if (parsed.origin !== source.origin || !CONTACT_PATH.test(parsed.pathname)) return;
137
- parsed.hash = "";
138
- const url = parsed.href;
139
- if (!candidatePageUrls.has(url)) candidatePageUrls.set(url, { url, sourceUrl: source.href, context });
140
- });
141
- $("form").each((_index, element) => {
142
- const node = $(element);
143
- const action = safeHttpUrl(node.attr("action") || source.href, source.href);
144
- if (!action || action.origin !== source.origin) return;
145
- action.hash = "";
146
- const url = action.href;
147
- if (!contactForms.has(url)) contactForms.set(url, { url, sourceUrl: source.href, context: nearbyContext(node) });
148
- });
149
- return {
150
- emails: [...emails.values()].slice(0, MAX_EMAILS),
151
- socialProfiles: [...socialProfiles.values()].slice(0, MAX_SOCIAL_PROFILES),
152
- contactForms: [...contactForms.values()].slice(0, MAX_CONTACT_FORMS),
153
- candidatePageUrls: [...candidatePageUrls.values()].slice(0, MAX_CANDIDATE_PAGE_URLS)
154
- };
155
- }
9
+ buildSerpEmailQuery,
10
+ contactAttemptUrls,
11
+ extractContactEvidence,
12
+ extractSerpEmailEvidence,
13
+ selectPrimaryEmail
14
+ } from "./chunk-VXLU74YZ.js";
156
15
 
157
16
  // src/lib/slugify.ts
158
17
  function slugify(s) {
@@ -753,146 +612,6 @@ ${summary}
753
612
 
754
613
  // src/workflows/workflows/get-leads.ts
755
614
  import { z as z3 } from "zod";
756
-
757
- // src/leads/contact-enrichment.ts
758
- var GENERIC_EMAIL_PREFIXES = /* @__PURE__ */ new Set([
759
- "admin",
760
- "contact",
761
- "customerservice",
762
- "hello",
763
- "help",
764
- "info",
765
- "inquiries",
766
- "mail",
767
- "office",
768
- "reception",
769
- "sales",
770
- "service",
771
- "support",
772
- "team"
773
- ]);
774
- var CONTACT_PATH2 = /(?:^|\/)(?:about|bio|contact|contact-us|meet|our-team|people|profile|staff|team)(?:\/|$)/i;
775
- var TEAM_CONTEXT = /\b(?:our team|meet the team|staff|agents?|brokers?|realtors?|people)\b/i;
776
- var OFFICE_CONTEXT = /\b(?:brokerage|company|corporate|front desk|main office|office|organization|reception)\b/i;
777
- function normalizeWords(value) {
778
- return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().split(/\s+/).filter(Boolean);
779
- }
780
- function safeHttpUrl2(value, base) {
781
- try {
782
- const parsed = new URL(value, base);
783
- if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
784
- return parsed;
785
- } catch {
786
- return null;
787
- }
788
- }
789
- function classifyEmailScope(evidence, entity) {
790
- const local = evidence.value.split("@")[0].toLowerCase().replace(/[^a-z0-9]+/g, " ");
791
- const nameWords = normalizeWords(entity.name).filter((word) => word.length > 1);
792
- const pagePath = safeHttpUrl2(evidence.sourceUrl)?.pathname.toLowerCase() ?? "";
793
- const combinedContext = `${evidence.context} ${pagePath}`.toLowerCase();
794
- const generic = GENERIC_EMAIL_PREFIXES.has(local.replace(/\s+/g, ""));
795
- const nameHits = nameWords.filter((word) => new RegExp(`(?:^|[^a-z0-9])${word}(?:$|[^a-z0-9])`, "i").test(`${local} ${combinedContext}`)).length;
796
- const hasExactPersonContext = entity.entityType === "person" && nameWords.length >= 2 && nameHits >= Math.min(2, nameWords.length) && (local.includes(nameWords[0]) || pagePath.includes(nameWords.join("-")) || pagePath.includes(nameWords.join("/")));
797
- if (!generic && hasExactPersonContext) return { scope: "person", reasons: ["named_person_page_context"] };
798
- if (generic || OFFICE_CONTEXT.test(combinedContext)) return { scope: "office", reasons: generic ? ["generic_mailbox_prefix"] : ["organization_page_context"] };
799
- if (TEAM_CONTEXT.test(combinedContext)) return { scope: "team", reasons: ["team_page_context"] };
800
- if (entity.entityType === "business") {
801
- const organizationWords = normalizeWords(entity.organization || entity.name).filter((word) => word.length > 2);
802
- if (organizationWords.some((word) => combinedContext.includes(word))) return { scope: "team", reasons: ["business_context"] };
803
- }
804
- return { scope: "unknown", reasons: ["public_occurrence_without_identity_evidence"] };
805
- }
806
- function selectPrimaryEmail(evidence, entity) {
807
- const scoped = evidence.map((item) => ({ ...item, ...classifyEmailScope(item, entity) }));
808
- const allowed = entity.entityType === "person" ? ["person", "team"] : ["person", "team", "office"];
809
- const sourcePriority = ["provided", "first_party_page", "structured_data", "serp_snippet"];
810
- const priority = ["person", "team", "office"];
811
- return scoped.filter((item) => allowed.includes(item.scope)).sort((a, b) => {
812
- const sourceDelta = sourcePriority.indexOf(a.sourceType) - sourcePriority.indexOf(b.sourceType);
813
- return sourceDelta || priority.indexOf(a.scope) - priority.indexOf(b.scope);
814
- })[0] ?? null;
815
- }
816
- function contactAttemptUrls(seedUrl, discoveredLinks, maxPages) {
817
- if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > 4) throw new Error("maxPages must be an integer from 1 to 4");
818
- const seed = safeHttpUrl2(seedUrl);
819
- if (!seed) return [];
820
- seed.hash = "";
821
- const urls = /* @__PURE__ */ new Set([seed.href]);
822
- const discovered = discoveredLinks.map((item) => typeof item === "string" ? item : item.url).map((item) => safeHttpUrl2(item, seed.href)).filter((item) => Boolean(item && item.origin === seed.origin && CONTACT_PATH2.test(item.pathname)));
823
- for (const item of discovered) {
824
- item.hash = "";
825
- urls.add(item.href);
826
- }
827
- for (const pathname of ["/contact", "/contact-us"]) urls.add(new URL(pathname, seed.origin).href);
828
- return [...urls].slice(0, maxPages);
829
- }
830
-
831
- // src/leads/serp-email-evidence.ts
832
- var EMAIL_CANDIDATE = /[A-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
833
- var DEVELOPER_CONTEXT2 = /\b(?:website|site|web)\s+(?:by|built|design(?:ed)?|develop(?:ed)?|powered)\b|\b(?:developer|webmaster)\b/i;
834
- var MAX_CONTEXT_CHARS2 = 280;
835
- var MAX_EMAILS2 = 10;
836
- function normalizeHostname(hostname) {
837
- return hostname.toLowerCase().replace(/^www\./, "").replace(/\.$/, "");
838
- }
839
- function isLocalHostname(hostname) {
840
- const host = normalizeHostname(hostname);
841
- return host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local") || !host.includes(".") && !host.includes(":");
842
- }
843
- function parsePublicLiteralUrl(websiteUrl) {
844
- try {
845
- const parsed = new URL(websiteUrl.trim());
846
- if (!["http:", "https:"].includes(parsed.protocol) || parsed.username || parsed.password) return null;
847
- if (isLocalHostname(parsed.hostname) || isPrivateIpAddress(parsed.hostname)) return null;
848
- return parsed;
849
- } catch {
850
- return null;
851
- }
852
- }
853
- function compactContext(title, snippet) {
854
- return `${title} ${snippet ?? ""}`.replace(/\s+/g, " ").trim().slice(0, MAX_CONTEXT_CHARS2);
855
- }
856
- function buildSerpEmailQuery(websiteUrl) {
857
- const parsed = parsePublicLiteralUrl(websiteUrl);
858
- if (!parsed) return null;
859
- parsed.hostname = normalizeHostname(parsed.hostname);
860
- return `email for ${parsed.protocol}//${parsed.host}/`;
861
- }
862
- function extractSerpEmailEvidence({
863
- websiteUrl,
864
- results
865
- }) {
866
- const parsedWebsite = parsePublicLiteralUrl(websiteUrl);
867
- if (!parsedWebsite) return [];
868
- const websiteDomain = normalizeHostname(parsedWebsite.hostname);
869
- const accepted = /* @__PURE__ */ new Map();
870
- const orderedResults = results.map((result, index) => ({ result, index })).sort((left, right) => {
871
- const leftPosition = Number.isFinite(left.result.position) ? left.result.position : Number.MAX_SAFE_INTEGER;
872
- const rightPosition = Number.isFinite(right.result.position) ? right.result.position : Number.MAX_SAFE_INTEGER;
873
- return leftPosition - rightPosition || left.index - right.index;
874
- });
875
- for (const { result } of orderedResults) {
876
- const searchableText = `${result.title} ${result.snippet ?? ""}`;
877
- const context = compactContext(result.title, result.snippet);
878
- for (const rawCandidate of searchableText.match(EMAIL_CANDIDATE) ?? []) {
879
- const value = normalizeEmail(rawCandidate);
880
- if (!value || DEVELOPER_CONTEXT2.test(searchableText) || normalizeHostname(value.slice(value.lastIndexOf("@") + 1)) !== websiteDomain) continue;
881
- if (accepted.has(value)) continue;
882
- accepted.set(value, {
883
- value,
884
- sourceUrl: result.url,
885
- sourceTitle: result.title,
886
- sourcePosition: result.position,
887
- context,
888
- sourceType: "serp_snippet"
889
- });
890
- }
891
- }
892
- return [...accepted.values()].slice(0, MAX_EMAILS2);
893
- }
894
-
895
- // src/workflows/workflows/get-leads.ts
896
615
  var GetLeadsInputSchema = z3.object({
897
616
  query: z3.string().min(1).describe('Business category, niche, or keyword to search on Google Maps, e.g. "roofers", "med spas", "dentists". Do not include the city here.'),
898
617
  location: z3.string().min(1).describe('City / market to search, e.g. "Houston, TX" or "Austin, Texas".'),
@@ -2182,12 +1901,6 @@ async function runWorkflowStep(id, rawInput, opts) {
2182
1901
  }
2183
1902
 
2184
1903
  export {
2185
- extractContactEvidence,
2186
- classifyEmailScope,
2187
- selectPrimaryEmail,
2188
- contactAttemptUrls,
2189
- buildSerpEmailQuery,
2190
- extractSerpEmailEvidence,
2191
1904
  slugify,
2192
1905
  workflowOutputBaseDir,
2193
1906
  listWorkflowReports,
@@ -2200,4 +1913,3 @@ export {
2200
1913
  runWorkflow,
2201
1914
  runWorkflowStep
2202
1915
  };
2203
- //# sourceMappingURL=chunk-MQQS7ONH.js.map
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  isVendorUnavailableError
3
- } from "./chunk-UN4LZVCB.js";
3
+ } from "./chunk-OZJMVCDK.js";
4
4
  import {
5
5
  CaptchaError,
6
6
  LocationMismatchError,
@@ -10,7 +10,7 @@ import {
10
10
  finishHarvestAttempt,
11
11
  publicErrorMessage,
12
12
  startHarvestAttempt
13
- } from "./chunk-4VG3NUUY.js";
13
+ } from "./chunk-GTGSD4YS.js";
14
14
 
15
15
  // src/api/webhook.ts
16
16
  async function deliverWebhook(url, payload, retries = 3) {
@@ -52,6 +52,9 @@ function looksLikeProxyUnavailable(message) {
52
52
  function looksLikeInternalServiceUnavailable(message) {
53
53
  return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
54
54
  }
55
+ function looksLikeBrowserSessionInterrupted(message) {
56
+ return /browser (?:has been )?closed|context (?:has been )?closed|target page, context or browser has been closed|session closed|page has been closed/i.test(message);
57
+ }
55
58
  function classifyHarvestProblem(err) {
56
59
  const message = errorMessage(err);
57
60
  if (err instanceof RequestAbortedError) {
@@ -114,6 +117,16 @@ function classifyHarvestProblem(err) {
114
117
  terminalStatus: "failed"
115
118
  };
116
119
  }
120
+ if (looksLikeBrowserSessionInterrupted(message)) {
121
+ return {
122
+ error_code: "browser_session_interrupted",
123
+ error_type: "extraction",
124
+ message: publicErrorMessage("browser_session_interrupted"),
125
+ retryable: true,
126
+ httpStatus: 503,
127
+ terminalStatus: "failed"
128
+ };
129
+ }
117
130
  if (looksLikeTimeout(err, message)) {
118
131
  return {
119
132
  error_code: "harvest_timeout",
@@ -222,4 +235,3 @@ export {
222
235
  publicErrorBoundary,
223
236
  createHarvestAttemptRecorder
224
237
  };
225
- //# sourceMappingURL=chunk-XPEN6OG5.js.map
@@ -8,10 +8,9 @@ import {
8
8
  } from "@onkernel/sdk";
9
9
  var BRIGHTDATA_NOT_CONFIGURED_MESSAGE = "SERP browser service is not configured";
10
10
  function isVendorUnavailableError(err, message) {
11
- return err instanceof KernelAuthenticationError || err instanceof KernelPermissionDeniedError || err instanceof KernelAPIConnectionError || err instanceof KernelAPIConnectionTimeoutError || err instanceof KernelInternalServerError || message === BRIGHTDATA_NOT_CONFIGURED_MESSAGE;
11
+ return err instanceof KernelAuthenticationError || err instanceof KernelPermissionDeniedError || err instanceof KernelAPIConnectionError || err instanceof KernelAPIConnectionTimeoutError || err instanceof KernelInternalServerError || /^401\s+Invalid or disabled API key\s*$/i.test(message.trim()) || message === BRIGHTDATA_NOT_CONFIGURED_MESSAGE;
12
12
  }
13
13
 
14
14
  export {
15
15
  isVendorUnavailableError
16
16
  };
17
- //# sourceMappingURL=chunk-UN4LZVCB.js.map
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  validatePublicHttpUrl
3
- } from "./chunk-UVFB24RZ.js";
3
+ } from "./chunk-GGZEC22A.js";
4
4
 
5
5
  // src/api/image-audit.ts
6
6
  var OVER_BYTES = 100 * 1024;
@@ -381,4 +381,3 @@ export {
381
381
  computeIssues,
382
382
  renderIssueReport
383
383
  };
384
- //# sourceMappingURL=chunk-342OUCSG.js.map
@@ -66,4 +66,3 @@ export {
66
66
  csvRecords,
67
67
  rowsToCsv
68
68
  };
69
- //# sourceMappingURL=chunk-G3P3ZDB4.js.map
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  getDb
3
- } from "./chunk-4VG3NUUY.js";
3
+ } from "./chunk-GTGSD4YS.js";
4
4
 
5
5
  // src/api/lead-list-enrichment-repository.ts
6
6
  import { createHash, randomUUID } from "crypto";
@@ -590,4 +590,3 @@ export {
590
590
  markLeadListEnrichmentOutboxDispatched,
591
591
  markLeadListEnrichmentOutboxFailed
592
592
  };
593
- //# sourceMappingURL=chunk-N54JS5WP.js.map
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  getDb
3
- } from "./chunk-4VG3NUUY.js";
3
+ } from "./chunk-GTGSD4YS.js";
4
4
 
5
5
  // src/lib/browser-service-env.ts
6
6
  function browserServiceApiKey() {
@@ -275,4 +275,3 @@ export {
275
275
  recordKernelSession,
276
276
  recordVendorUsage
277
277
  };
278
- //# sourceMappingURL=chunk-4LINSJNI.js.map
@@ -0,0 +1,6 @@
1
+ // src/version.ts
2
+ var PACKAGE_VERSION = "0.66.1";
3
+
4
+ export {
5
+ PACKAGE_VERSION
6
+ };