mcp-scraper 0.3.24 → 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +8220 -5991
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +320 -58
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +4 -3
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -3
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +186 -49
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -5
- package/dist/bin/paa-harvest.cjs +284 -10
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +5 -3
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-GZ4W72WF.js → chunk-2GHI2D6Q.js} +184 -48
- package/dist/chunk-2GHI2D6Q.js.map +1 -0
- package/dist/{chunk-ZAP7MYVO.js → chunk-3PRO376E.js} +2 -3
- package/dist/chunk-3PRO376E.js.map +1 -0
- package/dist/{chunk-LFATOGDF.js → chunk-BP27CZ5Q.js} +4 -3
- package/dist/chunk-BP27CZ5Q.js.map +1 -0
- package/dist/{chunk-DE7WP4UU.js → chunk-DOBQN3EY.js} +3 -3
- package/dist/chunk-DOBQN3EY.js.map +1 -0
- package/dist/{chunk-IGOEAKC7.js → chunk-G7PQ64LM.js} +274 -9
- package/dist/chunk-G7PQ64LM.js.map +1 -0
- package/dist/chunk-K443GQY5.js +24 -0
- package/dist/chunk-K443GQY5.js.map +1 -0
- package/dist/chunk-MS2UZ6HR.js +7 -0
- package/dist/chunk-MS2UZ6HR.js.map +1 -0
- package/dist/{chunk-H2R232HK.js → chunk-NNEIXK5L.js} +307 -62
- package/dist/chunk-NNEIXK5L.js.map +1 -0
- package/dist/{chunk-3KYRG7O7.js → chunk-TRNSXR46.js} +185 -2
- package/dist/chunk-TRNSXR46.js.map +1 -0
- package/dist/{chunk-JTWQGPKF.js → chunk-WYCER2HW.js} +33 -10
- package/dist/chunk-WYCER2HW.js.map +1 -0
- package/dist/{db-EG5ETPTY.js → db-FP2ABUU4.js} +32 -2
- package/dist/index.cjs +284 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.js +5 -3
- package/dist/index.js.map +1 -1
- package/dist/{server-2SRARAO5.js → server-HIGHKTEV.js} +1563 -275
- package/dist/server-HIGHKTEV.js.map +1 -0
- package/dist/{worker-UM5UM6O5.js → worker-UC6D2756.js} +7 -6
- package/dist/{worker-UM5UM6O5.js.map → worker-UC6D2756.js.map} +1 -1
- package/docs/kernel-proxy-future-enhancements.md +80 -0
- package/docs/spec-browser-routing.md +108 -0
- package/docs/spec-mcp-tools.md +146 -0
- package/docs/spec-server-instructions.md +94 -0
- package/docs/specs/audit-visual-demo.md +309 -0
- package/package.json +5 -2
- package/dist/chunk-3KYRG7O7.js.map +0 -1
- package/dist/chunk-DE7WP4UU.js.map +0 -1
- package/dist/chunk-GZ4W72WF.js.map +0 -1
- package/dist/chunk-H2R232HK.js.map +0 -1
- package/dist/chunk-H7R4PPMG.js +0 -7
- package/dist/chunk-H7R4PPMG.js.map +0 -1
- package/dist/chunk-IGOEAKC7.js.map +0 -1
- package/dist/chunk-JTWQGPKF.js.map +0 -1
- package/dist/chunk-LFATOGDF.js.map +0 -1
- package/dist/chunk-ZAP7MYVO.js.map +0 -1
- package/dist/server-2SRARAO5.js.map +0 -1
- /package/dist/{db-EG5ETPTY.js.map → db-FP2ABUU4.js.map} +0 -0
package/dist/bin/api-server.js
CHANGED
|
@@ -17,9 +17,9 @@ loadDotEnv();
|
|
|
17
17
|
async function main() {
|
|
18
18
|
const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
|
|
19
19
|
import("@hono/node-server"),
|
|
20
|
-
import("../server-
|
|
21
|
-
import("../worker-
|
|
22
|
-
import("../db-
|
|
20
|
+
import("../server-HIGHKTEV.js"),
|
|
21
|
+
import("../worker-UC6D2756.js"),
|
|
22
|
+
import("../db-FP2ABUU4.js")
|
|
23
23
|
]);
|
|
24
24
|
const PORT = parseInt(process.env.PORT ?? "3001");
|
|
25
25
|
try {
|
|
@@ -8,7 +8,7 @@ var import_promises4 = require("fs/promises");
|
|
|
8
8
|
var import_node_path3 = require("path");
|
|
9
9
|
|
|
10
10
|
// src/version.ts
|
|
11
|
-
var PACKAGE_VERSION = "0.3.
|
|
11
|
+
var PACKAGE_VERSION = "0.3.26";
|
|
12
12
|
|
|
13
13
|
// src/cli/agent-config.ts
|
|
14
14
|
function apiKeyValue(options) {
|
|
@@ -339,6 +339,26 @@ function slugify(s) {
|
|
|
339
339
|
return s.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
|
|
340
340
|
}
|
|
341
341
|
|
|
342
|
+
// src/output/memory-library-sink.ts
|
|
343
|
+
async function postToMemoryLibrary(args) {
|
|
344
|
+
const url = process.env.MCP_MEMORY_INGEST_URL;
|
|
345
|
+
const key = process.env.MCP_MEMORY_KEY;
|
|
346
|
+
if (!url || !key) return;
|
|
347
|
+
try {
|
|
348
|
+
const res = await fetch(url, {
|
|
349
|
+
method: "POST",
|
|
350
|
+
headers: { "content-type": "application/json" },
|
|
351
|
+
body: JSON.stringify({ data: { apiKey: key, title: args.title, content: args.content, source: args.source } })
|
|
352
|
+
});
|
|
353
|
+
const body = await res.json().catch(() => null);
|
|
354
|
+
if (!res.ok || body?.result?.error) {
|
|
355
|
+
console.warn("[memory-library-sink] ingest not accepted:", res.status, body?.result?.message ?? "");
|
|
356
|
+
}
|
|
357
|
+
} catch (err) {
|
|
358
|
+
console.warn("[memory-library-sink] ingest failed:", err?.message);
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
|
|
342
362
|
// src/workflows/artifact-writer.ts
|
|
343
363
|
function workflowOutputBaseDir(outputDir) {
|
|
344
364
|
return outputDir?.trim() || process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || (0, import_node_path2.join)((0, import_node_os2.homedir)(), "Downloads", "mcp-scraper");
|
|
@@ -379,6 +399,7 @@ var ArtifactWriter = class _ArtifactWriter {
|
|
|
379
399
|
async remember(kind, label, path, rows) {
|
|
380
400
|
const size = await (0, import_promises2.stat)(path).then((s) => s.size).catch(() => void 0);
|
|
381
401
|
this.artifacts.push({ kind, label, path, bytes: size, rows });
|
|
402
|
+
await postToMemoryLibrary({ title: `${this.title} ${label}`, content: `artifact: ${path}`, source: `mcp-scraper-workflow:${this.workflowId}:${this.runId}` });
|
|
382
403
|
return path;
|
|
383
404
|
}
|
|
384
405
|
async writeJson(label, relativePath, data) {
|
|
@@ -473,7 +494,7 @@ async function openWorkflowReport(id, outputDir) {
|
|
|
473
494
|
|
|
474
495
|
// src/workflows/registry.ts
|
|
475
496
|
var import_promises3 = require("fs/promises");
|
|
476
|
-
var
|
|
497
|
+
var import_zod7 = require("zod");
|
|
477
498
|
|
|
478
499
|
// src/workflows/http-client.ts
|
|
479
500
|
var WorkflowHttpClient = class {
|
|
@@ -796,7 +817,7 @@ var import_zod3 = require("zod");
|
|
|
796
817
|
|
|
797
818
|
// src/schemas.ts
|
|
798
819
|
var import_zod2 = require("zod");
|
|
799
|
-
var DEFAULT_PROXY_MODE = "
|
|
820
|
+
var DEFAULT_PROXY_MODE = "none";
|
|
800
821
|
var DEFAULT_MAPS_PROXY_MODE = "location";
|
|
801
822
|
var HarvestOptionsSchema = import_zod2.z.object({
|
|
802
823
|
query: import_zod2.z.string().min(1),
|
|
@@ -818,7 +839,8 @@ var HarvestOptionsSchema = import_zod2.z.object({
|
|
|
818
839
|
outputDir: import_zod2.z.string().default("./paa-output"),
|
|
819
840
|
format: import_zod2.z.enum(["json", "csv", "both"]).default("both"),
|
|
820
841
|
serpOnly: import_zod2.z.boolean().default(false),
|
|
821
|
-
pages: import_zod2.z.number().int().min(1).max(2).default(1)
|
|
842
|
+
pages: import_zod2.z.number().int().min(1).max(2).default(1),
|
|
843
|
+
softDeadlineMs: import_zod2.z.number().optional()
|
|
822
844
|
});
|
|
823
845
|
var MapsPlaceOptionsSchema = import_zod2.z.object({
|
|
824
846
|
businessName: import_zod2.z.string().min(1),
|
|
@@ -1024,20 +1046,41 @@ ${summary}
|
|
|
1024
1046
|
}
|
|
1025
1047
|
};
|
|
1026
1048
|
|
|
1027
|
-
// src/workflows/workflows/
|
|
1049
|
+
// src/workflows/workflows/get-leads.ts
|
|
1028
1050
|
var import_zod4 = require("zod");
|
|
1029
|
-
var
|
|
1030
|
-
query: import_zod4.z.string().min(1),
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
concurrency: import_zod4.z.number().int().min(1).max(5).default(5),
|
|
1038
|
-
proxyMode: import_zod4.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
1039
|
-
returnPartial: import_zod4.z.boolean().default(true)
|
|
1051
|
+
var GetLeadsInputSchema = import_zod4.z.object({
|
|
1052
|
+
query: import_zod4.z.string().min(1).describe('Business category, niche, or keyword to search on Google Maps, e.g. "roofers", "med spas", "dentists". Do not include the city here.'),
|
|
1053
|
+
location: import_zod4.z.string().min(1).describe('City / market to search, e.g. "Houston, TX" or "Austin, Texas".'),
|
|
1054
|
+
maxResults: import_zod4.z.number().int().min(1).max(50).default(25).describe("How many Maps businesses to collect for the market. Maximum 50."),
|
|
1055
|
+
enrichWebsites: import_zod4.z.boolean().default(true).describe("Visit each business website (home + contact pages) to harvest email and social links. Uses the proxy/browser-backed extractor so blocked sites still resolve."),
|
|
1056
|
+
hydrateReviewCounts: import_zod4.z.boolean().default(true).describe("Deep-dive each profile to confirm the review count and booking URL that the Maps search list omits."),
|
|
1057
|
+
concurrency: import_zod4.z.number().int().min(1).max(4).default(3).describe("How many businesses to enrich in parallel. Keep low to respect per-account concurrency limits."),
|
|
1058
|
+
proxyMode: import_zod4.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE).describe("Proxy targeting for the Maps search. location targets the market; none is for local debugging.")
|
|
1040
1059
|
});
|
|
1060
|
+
var LEADS_CSV_HEADERS = [
|
|
1061
|
+
"position",
|
|
1062
|
+
"business_name",
|
|
1063
|
+
"review_stars",
|
|
1064
|
+
"review_count",
|
|
1065
|
+
"category",
|
|
1066
|
+
"address",
|
|
1067
|
+
"phone",
|
|
1068
|
+
"website",
|
|
1069
|
+
"email",
|
|
1070
|
+
"email_domain_match",
|
|
1071
|
+
"email_status",
|
|
1072
|
+
"socials",
|
|
1073
|
+
"booking_url",
|
|
1074
|
+
"place_url",
|
|
1075
|
+
"cid",
|
|
1076
|
+
"source_location"
|
|
1077
|
+
];
|
|
1078
|
+
var EMAIL_RE = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g;
|
|
1079
|
+
var MAILTO_RE = /mailto:([^"'?>\s]+)/gi;
|
|
1080
|
+
var SOCIAL_RE = /https?:\/\/(?:www\.)?(?:facebook|instagram|linkedin|twitter|youtube|tiktok)\.com\/[^\s"'<>)]+/gi;
|
|
1081
|
+
var EMAIL_BAD = [".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico", "sentry", "wix.com", "wixpress", "example.", "godaddy", "squarespace", "schema.org", "@2x", "u003e", "core-js"];
|
|
1082
|
+
var EMAIL_PREFIX = ["info@", "office@", "contact@", "sales@", "hello@", "admin@", "service@", "support@"];
|
|
1083
|
+
var SOCIAL_SKIP = ["/sharer", "/share?", "/intent/", "plugins/", "/dialog/", "platform.", "badge", "developers."];
|
|
1041
1084
|
async function mapLimit(items, limit, fn) {
|
|
1042
1085
|
const out = new Array(items.length);
|
|
1043
1086
|
let next = 0;
|
|
@@ -1051,6 +1094,224 @@ async function mapLimit(items, limit, fn) {
|
|
|
1051
1094
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
1052
1095
|
return out;
|
|
1053
1096
|
}
|
|
1097
|
+
function hostRoot(url) {
|
|
1098
|
+
try {
|
|
1099
|
+
return new URL(url).hostname.replace(/^www\./, "").split(".")[0].toLowerCase();
|
|
1100
|
+
} catch {
|
|
1101
|
+
return "";
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
function cleanEmails(raw) {
|
|
1105
|
+
const out = /* @__PURE__ */ new Set();
|
|
1106
|
+
for (const value of raw) {
|
|
1107
|
+
const email = value.trim().toLowerCase().replace(/\.$/, "");
|
|
1108
|
+
if (email.split("@").length !== 2) continue;
|
|
1109
|
+
if (email.length > 60) continue;
|
|
1110
|
+
if (EMAIL_BAD.some((bad) => email.includes(bad))) continue;
|
|
1111
|
+
out.add(email);
|
|
1112
|
+
}
|
|
1113
|
+
return [...out];
|
|
1114
|
+
}
|
|
1115
|
+
function pickEmail(emails, host) {
|
|
1116
|
+
if (!emails.length) return "";
|
|
1117
|
+
const root = hostRoot(host);
|
|
1118
|
+
for (const prefix of EMAIL_PREFIX) {
|
|
1119
|
+
const hit = emails.find((e) => e.startsWith(prefix) && root && e.split("@")[1].includes(root));
|
|
1120
|
+
if (hit) return hit;
|
|
1121
|
+
}
|
|
1122
|
+
const domainHit = emails.find((e) => root && e.split("@")[1].includes(root));
|
|
1123
|
+
if (domainHit) return domainHit;
|
|
1124
|
+
for (const prefix of EMAIL_PREFIX) {
|
|
1125
|
+
const hit = emails.find((e) => e.startsWith(prefix));
|
|
1126
|
+
if (hit) return hit;
|
|
1127
|
+
}
|
|
1128
|
+
return emails[0];
|
|
1129
|
+
}
|
|
1130
|
+
function parseContacts(html) {
|
|
1131
|
+
const mailtos = [...html.matchAll(MAILTO_RE)].map((m) => decodeURIComponent(m[1]).split("?")[0]);
|
|
1132
|
+
const inline = html.match(EMAIL_RE) ?? [];
|
|
1133
|
+
const emails = cleanEmails([...mailtos, ...inline]);
|
|
1134
|
+
const socials = [...new Set((html.match(SOCIAL_RE) ?? []).map((s) => s.replace(/[)"'<>]+$/, "")).filter((s) => !SOCIAL_SKIP.some((skip) => s.toLowerCase().includes(skip))))].slice(0, 5);
|
|
1135
|
+
return { emails, socials };
|
|
1136
|
+
}
|
|
1137
|
+
function buildAttemptUrls(website) {
|
|
1138
|
+
const urls = /* @__PURE__ */ new Set();
|
|
1139
|
+
urls.add(website);
|
|
1140
|
+
try {
|
|
1141
|
+
const origin = new URL(website).origin;
|
|
1142
|
+
urls.add(`${origin}/contact`);
|
|
1143
|
+
urls.add(`${origin}/contact-us`);
|
|
1144
|
+
} catch {
|
|
1145
|
+
return [...urls];
|
|
1146
|
+
}
|
|
1147
|
+
return [...urls].slice(0, 3);
|
|
1148
|
+
}
|
|
1149
|
+
var getLeadsWorkflowDefinition = {
|
|
1150
|
+
id: "get-leads",
|
|
1151
|
+
title: "Get Leads",
|
|
1152
|
+
description: "Build an outreach-ready local lead list: Google Maps search for a niche in a market, confirm review counts + booking URLs, then visit each business website (home + contact) to harvest email and social links, with proxy/browser-backed extraction so blocked sites still resolve. Saves a leads CSV.",
|
|
1153
|
+
inputSchema: GetLeadsInputSchema,
|
|
1154
|
+
async run(input, ctx) {
|
|
1155
|
+
await ctx.artifacts.writeManifest("running", {}, [], []);
|
|
1156
|
+
const warnings = [];
|
|
1157
|
+
const errors = [];
|
|
1158
|
+
const search = await ctx.client.post("/maps/search", {
|
|
1159
|
+
query: input.query,
|
|
1160
|
+
location: input.location,
|
|
1161
|
+
maxResults: input.maxResults,
|
|
1162
|
+
proxyMode: input.proxyMode
|
|
1163
|
+
}, 18e4);
|
|
1164
|
+
await ctx.artifacts.writeJson("Maps search raw", "raw/maps-search.json", search);
|
|
1165
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1166
|
+
const deduped = search.results.filter((result) => {
|
|
1167
|
+
const key = result.cid ?? result.placeUrl ?? result.name.toLowerCase();
|
|
1168
|
+
if (seen.has(key)) return false;
|
|
1169
|
+
seen.add(key);
|
|
1170
|
+
return true;
|
|
1171
|
+
});
|
|
1172
|
+
const rows = await mapLimit(deduped, input.concurrency, async (result) => {
|
|
1173
|
+
let reviewCount = result.reviewCount ?? "";
|
|
1174
|
+
let bookingUrl = "";
|
|
1175
|
+
let website = result.websiteUrl ?? "";
|
|
1176
|
+
if (input.hydrateReviewCounts) {
|
|
1177
|
+
try {
|
|
1178
|
+
const place = await ctx.client.post("/maps/place", {
|
|
1179
|
+
businessName: result.name,
|
|
1180
|
+
location: input.location,
|
|
1181
|
+
includeReviews: false
|
|
1182
|
+
}, 18e4);
|
|
1183
|
+
reviewCount = place.reviewCount ?? reviewCount;
|
|
1184
|
+
bookingUrl = place.bookingUrl ?? "";
|
|
1185
|
+
website = website || (place.website ?? "");
|
|
1186
|
+
} catch (err) {
|
|
1187
|
+
warnings.push(`Review hydration failed for ${result.name}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
let email = "";
|
|
1191
|
+
let socials = [];
|
|
1192
|
+
let emailStatus = website ? "no email found" : "no website";
|
|
1193
|
+
let domainMatch = "";
|
|
1194
|
+
if (input.enrichWebsites && website) {
|
|
1195
|
+
const attempts = buildAttemptUrls(website);
|
|
1196
|
+
let reached = false;
|
|
1197
|
+
const allEmails = [];
|
|
1198
|
+
const allSocials = /* @__PURE__ */ new Set();
|
|
1199
|
+
for (const attemptUrl of attempts) {
|
|
1200
|
+
try {
|
|
1201
|
+
const page = await ctx.client.post("/extract-url", { url: attemptUrl }, 12e4);
|
|
1202
|
+
reached = true;
|
|
1203
|
+
const parsed = parseContacts(page.bodyHtml ?? page.bodyMarkdown ?? "");
|
|
1204
|
+
allEmails.push(...parsed.emails);
|
|
1205
|
+
parsed.socials.forEach((s) => allSocials.add(s));
|
|
1206
|
+
if (parsed.emails.length) break;
|
|
1207
|
+
} catch {
|
|
1208
|
+
continue;
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
socials = [...allSocials].slice(0, 5);
|
|
1212
|
+
email = pickEmail([...new Set(allEmails)], website);
|
|
1213
|
+
if (email) {
|
|
1214
|
+
domainMatch = hostRoot(website) && email.split("@")[1].includes(hostRoot(website)) ? "yes" : "NO-verify";
|
|
1215
|
+
emailStatus = domainMatch === "yes" ? "ok" : "ok (off-domain \u2014 verify)";
|
|
1216
|
+
} else if (!reached) {
|
|
1217
|
+
emailStatus = "site unreachable";
|
|
1218
|
+
} else if (socials.length) {
|
|
1219
|
+
emailStatus = "form/social only";
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
return {
|
|
1223
|
+
position: result.position,
|
|
1224
|
+
business_name: result.name,
|
|
1225
|
+
review_stars: result.rating ?? "",
|
|
1226
|
+
review_count: reviewCount,
|
|
1227
|
+
category: result.category ?? "",
|
|
1228
|
+
address: result.address ?? "",
|
|
1229
|
+
phone: result.phone ?? "",
|
|
1230
|
+
website,
|
|
1231
|
+
email,
|
|
1232
|
+
email_domain_match: domainMatch,
|
|
1233
|
+
email_status: emailStatus,
|
|
1234
|
+
socials: socials.join(" ; "),
|
|
1235
|
+
booking_url: bookingUrl,
|
|
1236
|
+
place_url: result.placeUrl,
|
|
1237
|
+
cid: result.cid ?? "",
|
|
1238
|
+
source_location: input.location
|
|
1239
|
+
};
|
|
1240
|
+
});
|
|
1241
|
+
await ctx.artifacts.writeCsv("Leads CSV", "exports/leads.csv", LEADS_CSV_HEADERS, rows);
|
|
1242
|
+
await ctx.artifacts.writeJson("Leads evidence", "evidence.json", { input, search: { query: search.searchQuery, resultCount: search.resultCount }, rows });
|
|
1243
|
+
const withEmail = rows.filter((r) => r.email).length;
|
|
1244
|
+
const domainMatched = rows.filter((r) => r.email_domain_match === "yes").length;
|
|
1245
|
+
const withReviewCount = rows.filter((r) => r.review_count).length;
|
|
1246
|
+
const withSocials = rows.filter((r) => r.socials).length;
|
|
1247
|
+
const withWebsite = rows.filter((r) => r.website).length;
|
|
1248
|
+
const summary = `${rows.length} ${input.query} in ${input.location}: ${withEmail} emails (${domainMatched} domain-matched), ${withReviewCount} review counts, ${withSocials} with socials.`;
|
|
1249
|
+
await ctx.artifacts.writeText("Summary", "summary.md", `# Get Leads
|
|
1250
|
+
|
|
1251
|
+
${summary}
|
|
1252
|
+
`);
|
|
1253
|
+
await ctx.artifacts.writeText("Agent tasks", "tasks.md", [
|
|
1254
|
+
"# Get Leads \u2014 next steps",
|
|
1255
|
+
"",
|
|
1256
|
+
"- [ ] Verify deliverability of the harvested emails before outreach (MX/SMTP check).",
|
|
1257
|
+
'- [ ] For "form/social only" rows, use the website contact form or the captured social profile.',
|
|
1258
|
+
'- [ ] Treat "off-domain \u2014 verify" emails with care; they may belong to a web developer or parent brand.',
|
|
1259
|
+
"- [ ] Respect CAN-SPAM / GDPR \u2014 scraped contact data is not consent to email."
|
|
1260
|
+
].join("\n"));
|
|
1261
|
+
const reportPath = await ctx.artifacts.writeHtml("HTML report", "report.html", renderWorkflowReport({
|
|
1262
|
+
title: "Get Leads",
|
|
1263
|
+
subtitle: `${input.query} \xB7 ${input.location}`,
|
|
1264
|
+
summary,
|
|
1265
|
+
warnings,
|
|
1266
|
+
tables: [
|
|
1267
|
+
{
|
|
1268
|
+
title: "Leads",
|
|
1269
|
+
columns: ["position", "business_name", "review_stars", "review_count", "phone", "website", "email", "email_status", "socials", "booking_url"],
|
|
1270
|
+
rows
|
|
1271
|
+
}
|
|
1272
|
+
]
|
|
1273
|
+
}));
|
|
1274
|
+
const status2 = errors.length ? "failed" : warnings.length ? "partial" : "succeeded";
|
|
1275
|
+
const counts = {
|
|
1276
|
+
businesses: rows.length,
|
|
1277
|
+
withWebsite,
|
|
1278
|
+
emails: withEmail,
|
|
1279
|
+
domainMatchedEmails: domainMatched,
|
|
1280
|
+
reviewCounts: withReviewCount,
|
|
1281
|
+
withSocials
|
|
1282
|
+
};
|
|
1283
|
+
await ctx.artifacts.writeManifest(status2, counts, warnings, errors);
|
|
1284
|
+
return { title: "Get Leads", summary, status: status2, counts, warnings, errors, reportPath };
|
|
1285
|
+
}
|
|
1286
|
+
};
|
|
1287
|
+
|
|
1288
|
+
// src/workflows/workflows/local-competitive-audit.ts
|
|
1289
|
+
var import_zod5 = require("zod");
|
|
1290
|
+
var LocalCompetitiveAuditInputSchema = import_zod5.z.object({
|
|
1291
|
+
query: import_zod5.z.string().min(1),
|
|
1292
|
+
state: import_zod5.z.string().min(2).default("TN"),
|
|
1293
|
+
minPopulation: import_zod5.z.number().int().min(0).default(1e5),
|
|
1294
|
+
maxCities: import_zod5.z.number().int().min(1).max(100).default(25),
|
|
1295
|
+
maxResultsPerCity: import_zod5.z.number().int().min(1).max(50).default(20),
|
|
1296
|
+
hydrateTop: import_zod5.z.number().int().min(0).max(10).default(5),
|
|
1297
|
+
maxReviews: import_zod5.z.number().int().min(0).max(500).default(50),
|
|
1298
|
+
concurrency: import_zod5.z.number().int().min(1).max(5).default(5),
|
|
1299
|
+
proxyMode: import_zod5.z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
|
|
1300
|
+
returnPartial: import_zod5.z.boolean().default(true)
|
|
1301
|
+
});
|
|
1302
|
+
async function mapLimit2(items, limit, fn) {
|
|
1303
|
+
const out = new Array(items.length);
|
|
1304
|
+
let next = 0;
|
|
1305
|
+
async function worker() {
|
|
1306
|
+
while (next < items.length) {
|
|
1307
|
+
const index = next;
|
|
1308
|
+
next += 1;
|
|
1309
|
+
out[index] = await fn(items[index], index);
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
1313
|
+
return out;
|
|
1314
|
+
}
|
|
1054
1315
|
function numberFrom(value) {
|
|
1055
1316
|
if (value === null || value === void 0 || value === "") return null;
|
|
1056
1317
|
const parsed = Number(String(value).replace(/[^\d.]/g, ""));
|
|
@@ -1104,7 +1365,7 @@ var localCompetitiveAuditWorkflowDefinition = {
|
|
|
1104
1365
|
seen.add(key);
|
|
1105
1366
|
return true;
|
|
1106
1367
|
});
|
|
1107
|
-
const hydrated = await
|
|
1368
|
+
const hydrated = await mapLimit2(deduped, 3, async ({ city, result }, index) => {
|
|
1108
1369
|
try {
|
|
1109
1370
|
const detail = await ctx.client.post("/maps/place", {
|
|
1110
1371
|
businessName: result.name,
|
|
@@ -1224,7 +1485,7 @@ ${summary}
|
|
|
1224
1485
|
};
|
|
1225
1486
|
|
|
1226
1487
|
// src/workflows/workflows/comparison-briefs.ts
|
|
1227
|
-
var
|
|
1488
|
+
var import_zod6 = require("zod");
|
|
1228
1489
|
|
|
1229
1490
|
// src/workflows/workflows/seo-workflow-utils.ts
|
|
1230
1491
|
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
@@ -1367,7 +1628,7 @@ function pageSummaryRow(page, source) {
|
|
|
1367
1628
|
schema_types: (page.schemaTypes ?? []).join("; ")
|
|
1368
1629
|
};
|
|
1369
1630
|
}
|
|
1370
|
-
async function
|
|
1631
|
+
async function mapLimit3(items, limit, fn) {
|
|
1371
1632
|
const out = new Array(items.length);
|
|
1372
1633
|
let next = 0;
|
|
1373
1634
|
async function worker() {
|
|
@@ -1382,49 +1643,49 @@ async function mapLimit2(items, limit, fn) {
|
|
|
1382
1643
|
}
|
|
1383
1644
|
|
|
1384
1645
|
// src/workflows/workflows/comparison-briefs.ts
|
|
1385
|
-
var ProxyModeSchema =
|
|
1386
|
-
var MapComparisonInputSchema =
|
|
1387
|
-
query:
|
|
1388
|
-
location:
|
|
1389
|
-
state:
|
|
1390
|
-
minPopulation:
|
|
1391
|
-
maxCities:
|
|
1392
|
-
maxResultsPerCity:
|
|
1393
|
-
hydrateTop:
|
|
1394
|
-
maxReviews:
|
|
1395
|
-
concurrency:
|
|
1646
|
+
var ProxyModeSchema = import_zod6.z.enum(["location", "configured", "none"]);
|
|
1647
|
+
var MapComparisonInputSchema = import_zod6.z.object({
|
|
1648
|
+
query: import_zod6.z.string().min(1),
|
|
1649
|
+
location: import_zod6.z.string().optional(),
|
|
1650
|
+
state: import_zod6.z.string().optional(),
|
|
1651
|
+
minPopulation: import_zod6.z.number().int().min(0).default(1e5),
|
|
1652
|
+
maxCities: import_zod6.z.number().int().min(1).max(100).default(5),
|
|
1653
|
+
maxResultsPerCity: import_zod6.z.number().int().min(1).max(50).default(20),
|
|
1654
|
+
hydrateTop: import_zod6.z.number().int().min(0).max(10).default(5),
|
|
1655
|
+
maxReviews: import_zod6.z.number().int().min(0).max(500).default(25),
|
|
1656
|
+
concurrency: import_zod6.z.number().int().min(1).max(5).default(5),
|
|
1396
1657
|
proxyMode: ProxyModeSchema.default(DEFAULT_MAPS_PROXY_MODE),
|
|
1397
|
-
returnPartial:
|
|
1658
|
+
returnPartial: import_zod6.z.boolean().default(true)
|
|
1398
1659
|
}).refine((input) => input.location || input.state, {
|
|
1399
1660
|
message: "Either location or state is required for map-comparison"
|
|
1400
1661
|
});
|
|
1401
|
-
var SerpComparisonInputSchema =
|
|
1402
|
-
keyword:
|
|
1403
|
-
domain:
|
|
1404
|
-
url:
|
|
1405
|
-
location:
|
|
1406
|
-
maxResults:
|
|
1407
|
-
maxQuestions:
|
|
1408
|
-
extractTop:
|
|
1409
|
-
includePaa:
|
|
1410
|
-
includeAiOverview:
|
|
1411
|
-
returnPartial:
|
|
1662
|
+
var SerpComparisonInputSchema = import_zod6.z.object({
|
|
1663
|
+
keyword: import_zod6.z.string().min(1),
|
|
1664
|
+
domain: import_zod6.z.string().optional(),
|
|
1665
|
+
url: import_zod6.z.string().url().optional(),
|
|
1666
|
+
location: import_zod6.z.string().optional(),
|
|
1667
|
+
maxResults: import_zod6.z.number().int().min(1).max(20).default(10),
|
|
1668
|
+
maxQuestions: import_zod6.z.number().int().min(1).max(200).default(40),
|
|
1669
|
+
extractTop: import_zod6.z.number().int().min(0).max(10).default(5),
|
|
1670
|
+
includePaa: import_zod6.z.boolean().default(true),
|
|
1671
|
+
includeAiOverview: import_zod6.z.boolean().default(true),
|
|
1672
|
+
returnPartial: import_zod6.z.boolean().default(true)
|
|
1412
1673
|
});
|
|
1413
|
-
var PaaExpansionBriefInputSchema =
|
|
1414
|
-
keyword:
|
|
1415
|
-
location:
|
|
1416
|
-
maxQuestions:
|
|
1417
|
-
depth:
|
|
1418
|
-
returnPartial:
|
|
1674
|
+
var PaaExpansionBriefInputSchema = import_zod6.z.object({
|
|
1675
|
+
keyword: import_zod6.z.string().min(1),
|
|
1676
|
+
location: import_zod6.z.string().optional(),
|
|
1677
|
+
maxQuestions: import_zod6.z.number().int().min(1).max(300).default(80),
|
|
1678
|
+
depth: import_zod6.z.number().int().min(1).max(6).default(3),
|
|
1679
|
+
returnPartial: import_zod6.z.boolean().default(true)
|
|
1419
1680
|
});
|
|
1420
|
-
var AiOverviewLanguageInputSchema =
|
|
1421
|
-
keyword:
|
|
1422
|
-
domain:
|
|
1423
|
-
url:
|
|
1424
|
-
location:
|
|
1425
|
-
maxQuestions:
|
|
1426
|
-
extractTop:
|
|
1427
|
-
returnPartial:
|
|
1681
|
+
var AiOverviewLanguageInputSchema = import_zod6.z.object({
|
|
1682
|
+
keyword: import_zod6.z.string().min(1),
|
|
1683
|
+
domain: import_zod6.z.string().optional(),
|
|
1684
|
+
url: import_zod6.z.string().url().optional(),
|
|
1685
|
+
location: import_zod6.z.string().optional(),
|
|
1686
|
+
maxQuestions: import_zod6.z.number().int().min(1).max(200).default(40),
|
|
1687
|
+
extractTop: import_zod6.z.number().int().min(0).max(8).default(3),
|
|
1688
|
+
returnPartial: import_zod6.z.boolean().default(true)
|
|
1428
1689
|
});
|
|
1429
1690
|
function businessRowsFromMaps(location, query, results) {
|
|
1430
1691
|
return results.map((result) => ({
|
|
@@ -1541,7 +1802,7 @@ function pageGapRows(targetPage, competitorPages) {
|
|
|
1541
1802
|
return rows.slice(0, 150);
|
|
1542
1803
|
}
|
|
1543
1804
|
async function extractPages(ctx, sources, warnings, labelPrefix) {
|
|
1544
|
-
return
|
|
1805
|
+
return mapLimit3(sources.filter((source) => source.url), 2, async (source, index) => {
|
|
1545
1806
|
try {
|
|
1546
1807
|
const page = await ctx.client.post("/extract-url", { url: source.url }, 18e4);
|
|
1547
1808
|
await ctx.artifacts.writeJson(`${labelPrefix} ${index + 1}`, `raw/extract-url/${labelPrefix.toLowerCase()}-${index + 1}.json`, page);
|
|
@@ -1590,7 +1851,7 @@ var mapComparisonWorkflowDefinition = {
|
|
|
1590
1851
|
}
|
|
1591
1852
|
const compareRows = comparisonRows(rows);
|
|
1592
1853
|
const selected = compareRows.slice(0, input.hydrateTop * Math.max(1, input.location ? 1 : input.maxCities));
|
|
1593
|
-
const hydrated = await
|
|
1854
|
+
const hydrated = await mapLimit3(selected, 3, async (row, index) => {
|
|
1594
1855
|
try {
|
|
1595
1856
|
const detail = await ctx.client.post("/maps/place", {
|
|
1596
1857
|
businessName: row.business_name,
|
|
@@ -1925,6 +2186,7 @@ var aiOverviewLanguageWorkflowDefinition = {
|
|
|
1925
2186
|
// src/workflows/registry.ts
|
|
1926
2187
|
var DEFINITIONS = [
|
|
1927
2188
|
directoryWorkflowDefinition,
|
|
2189
|
+
getLeadsWorkflowDefinition,
|
|
1928
2190
|
agentPacketWorkflowDefinition,
|
|
1929
2191
|
localCompetitiveAuditWorkflowDefinition,
|
|
1930
2192
|
mapComparisonWorkflowDefinition,
|