mcp-scraper 0.2.0 → 0.2.1

package/dist/bin/mcp-stdio-server.cjs

@@ -113,6 +113,12 @@ var HttpMcpToolExecutor = class {
   mapsSearch(input) {
     return this.call("/maps/search", input);
   }
+  directoryWorkflow(input) {
+    const cityCount = typeof input.maxCities === "number" ? input.maxCities : 25;
+    const concurrency = typeof input.concurrency === "number" && input.concurrency > 0 ? input.concurrency : 5;
+    const timeoutMs = this.httpTimeoutOverrideMs ?? Math.min(9e5, Math.max(18e4, Math.ceil(cityCount / concurrency) * 12e4));
+    return this.call("/directory/run", input, timeoutMs);
+  }
   creditsInfo(input) {
     return this.call("/billing/credits", input);
   }
@@ -130,7 +136,7 @@ var import_node_fs2 = require("fs");
 var import_node_path2 = require("path");
 
 // src/version.ts
-var PACKAGE_VERSION = "0.2.0";
+var PACKAGE_VERSION = "0.2.1";
 
 // src/mcp/mcp-response-formatter.ts
 var import_node_fs = require("fs");
@@ -800,6 +806,11 @@ function formatMapsSearch(raw, input) {
   if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
   const d = parsed.data;
   const results = d.results ?? [];
+  const normalizedResults = results.map((result) => ({
+    ...result,
+    phone: result.phone ?? null,
+    hoursStatus: result.hoursStatus ?? null
+  }));
   const searchQuery = d.searchQuery ?? [input.query, input.location].filter(Boolean).join(" ");
   const requestedMax = d.requestedMaxResults ?? input.maxResults ?? 10;
   const durationMs = d.durationMs;
@@ -839,7 +850,79 @@ ${rows}`,
       extractedAt: d.extractedAt,
       requestedMaxResults: requestedMax,
       resultCount: results.length,
-      results,
+      results: normalizedResults,
+      durationMs: durationMs ?? 0
+    }
+  };
+}
+function formatDirectoryWorkflow(raw, input) {
+  const parsed = parseData(raw);
+  if ("error" in parsed) return { content: [{ type: "text", text: parsed.error }], isError: true };
+  const d = parsed.data;
+  const cities = (d.cities ?? []).map((city) => ({
+    ...city,
+    results: city.results.map((result) => ({
+      ...result,
+      phone: result.phone ?? null,
+      hoursStatus: result.hoursStatus ?? null
+    }))
+  }));
+  const warnings = d.warnings ?? [];
+  const csvPath = d.csvPath ?? null;
+  const totalResultCount = d.totalResultCount ?? cities.reduce((sum, city) => sum + city.resultCount, 0);
+  const durationMs = d.durationMs;
+  const marketRows = cities.map((city) => {
+    const zips = city.zips?.length ? city.zips.slice(0, 8).join(" ") + (city.zips.length > 8 ? ` +${city.zips.length - 8}` : "") : "\u2014";
+    return `| ${cell(city.city)} | ${city.population.toLocaleString()} | ${city.zips?.length ?? 0} | ${city.resultCount} | ${city.status} | ${cell(zips)} |`;
+  }).join("\n");
+  const businessRows = cities.flatMap((city) => city.results.slice(0, 3).map((result) => ({ city, result }))).map(({ city, result }) => {
+    const rating = [result.rating, result.reviewCount ? `(${result.reviewCount})` : null].filter(Boolean).join(" ");
+    return `| ${cell(city.city)} | ${result.position} | ${cell(result.name)} | ${cell(result.category)} | ${cell(rating)} | ${result.websiteUrl ? `[site](${result.websiteUrl})` : "\u2014"} | [maps](${result.placeUrl}) |`;
+  }).join("\n");
+  const warningText = warnings.length ? `
+## Warnings
+${warnings.map((w) => `- ${w}`).join("\n")}` : "";
+  const csvText = csvPath ? `
+**CSV:** \`${csvPath}\`` : "";
+  const full = [
+    `# Directory Workflow: ${input.query}`,
+    `**Markets:** ${cities.length} \xB7 **Maps results:** ${totalResultCount} \xB7 **State:** ${d.state ?? input.state ?? "US"} \xB7 **Population threshold:** ${d.minPopulation ?? input.minPopulation ?? 1e5}`,
+    csvText,
+    `
+## Markets
+| City | Population | ZIPs | Maps Results | Status | ZIP Sample |
+|---|---:|---:|---:|---|---|
+${marketRows}`,
+    businessRows ? `
+## Top Candidates By City
+| City | # | Name | Category | Rating | Website | Maps |
+|---|---:|---|---|---|---|---|
+${businessRows}` : null,
+    warningText,
+    `
+## Sources
+- Population: ${d.censusSourceUrl ?? "Census Population Estimates Program"}
+- ZIP groups: ${d.usZipsSourcePath ?? "not configured"}`,
+    durationMs != null ? `
+*Completed in ${(durationMs / 1e3).toFixed(1)}s*` : null
+  ].filter(Boolean).join("\n");
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      query: d.query,
+      state: d.state,
+      minPopulation: d.minPopulation,
+      populationYear: d.populationYear,
+      maxResultsPerCity: d.maxResultsPerCity,
+      concurrency: d.concurrency,
+      censusSourceUrl: d.censusSourceUrl,
+      usZipsSourcePath: d.usZipsSourcePath ?? null,
+      warnings,
+      extractedAt: d.extractedAt,
+      selectedCityCount: d.selectedCityCount,
+      totalResultCount,
+      csvPath,
+      cities,
       durationMs: durationMs ?? 0
     }
   };
@@ -1005,8 +1088,8 @@ var HarvestPaaInputSchema = {
   gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from location or user language. Examples: United States us, United Kingdom gb, Japan jp, Canada ca, Australia au."),
   hl: import_zod.z.string().default("en").describe("Google interface/content language inferred from the user request. Use en unless the user asks for another language or locale."),
   device: import_zod.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
-  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default so city/state searches create or reuse a matching residential proxy. Use configured for the static configured proxy. Use none only for direct-network debugging."),
-  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only when the user gives a specific ZIP or city-center proxy targeting needs to be forced."),
+  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt and retries CAPTCHA, proxy tunnel failure, and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
+  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
   debug: import_zod.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior.")
 };
 var ExtractUrlInputSchema = {
@@ -1063,7 +1146,25 @@ var MapsSearchInputSchema = {
   location: import_zod.z.string().optional().describe('City, region, country, or service area for the Maps search, e.g. "Denver, CO". Infer from the user request when present.'),
   gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from location."),
   hl: import_zod.z.string().length(2).default("en").describe("Language inferred from user request."),
-  maxResults: import_zod.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more.")
+  maxResults: import_zod.z.number().int().min(1).max(50).default(10).describe("Number of Google Maps business/profile candidates to return. Default 10. Maximum 50. Use 10 unless the user asks for more."),
+  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state Maps searches; it creates a fresh residential proxy ID when the browser service is available. Use configured for the server proxy ID, and none only for local direct-network debugging."),
+  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or city-center ZIP."),
+  debug: import_zod.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics when debugging Maps localization, CAPTCHA, or proxy behavior.")
+};
+var DirectoryWorkflowInputSchema = {
+  query: import_zod.z.string().min(1).describe("Business category, niche, or keyword to search on Google Maps for every selected market, e.g. roofers, dentists, med spas. Do not include the city here."),
+  state: import_zod.z.string().min(2).default("TN").describe("US state abbreviation or state name used to select Census places, e.g. TN or Tennessee."),
+  minPopulation: import_zod.z.number().int().min(0).default(1e5).describe('Minimum Census place population for market selection. Use 100000 for "cities above 100k population".'),
+  populationYear: import_zod.z.number().int().min(2020).max(2025).default(2025).describe("Census population estimate year from the 2020-2025 Population Estimates Program city/place dataset."),
+  maxCities: import_zod.z.number().int().min(1).max(100).default(25).describe("Maximum number of markets to process after sorting by population descending."),
+  maxResultsPerCity: import_zod.z.number().int().min(1).max(50).default(50).describe("Google Maps business/profile candidates to collect for each city. Maximum 50."),
+  concurrency: import_zod.z.number().int().min(1).max(5).default(5).describe("How many city Maps searches to run in parallel. Use 5 for broad directory batches unless debugging."),
+  includeZipGroups: import_zod.z.boolean().default(true).describe("Attach ZIP groups from a configured US ZIPS CSV when available. Set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath in local/test mode."),
+  usZipsCsvPath: import_zod.z.string().optional().describe("Local/test-only path to a US ZIPS CSV with state_abbr, zipcode, county, city columns, such as Lead Magician tools/analytics/data/uszips.csv. Deployed APIs should use MCP_SCRAPER_USZIPS_CSV_PATH instead."),
+  saveCsv: import_zod.z.boolean().default(true).describe("Save a directory-ready CSV to the MCP Scraper output directory and return its path. CSV rows include source_location, result_position, business_name, review_stars, category, address, phone, hours_status, website_url, directions_url, place_url, CID fields, population, and ZIP groups."),
+  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode for every city Maps search. Use location by default for US city/state batches; it creates fresh residential proxy IDs when the browser service is available. Use configured for the server proxy ID, and none only for local direct-network debugging."),
+  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
+  debug: import_zod.z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
 };
 var NullableString = import_zod.z.string().nullable();
 var MapsSearchOutputSchema = {
@@ -1084,12 +1185,62 @@ var MapsSearchOutputSchema = {
     reviewCount: NullableString,
     category: NullableString,
     address: NullableString,
+    phone: NullableString,
+    hoursStatus: NullableString,
     websiteUrl: NullableString,
     directionsUrl: NullableString,
     metadata: import_zod.z.array(import_zod.z.string())
   })),
   durationMs: import_zod.z.number().int().min(0)
 };
+var DirectoryMapsBusinessOutput = import_zod.z.object({
+  position: import_zod.z.number().int().min(1),
+  name: import_zod.z.string(),
+  placeUrl: import_zod.z.string().url(),
+  cid: NullableString,
+  cidDecimal: NullableString,
+  rating: NullableString,
+  reviewCount: NullableString,
+  category: NullableString,
+  address: NullableString,
+  phone: NullableString,
+  hoursStatus: NullableString,
+  websiteUrl: NullableString,
+  directionsUrl: NullableString,
+  metadata: import_zod.z.array(import_zod.z.string())
+});
+var DirectoryWorkflowOutputSchema = {
+  query: import_zod.z.string(),
+  state: import_zod.z.string(),
+  minPopulation: import_zod.z.number().int().min(0),
+  populationYear: import_zod.z.number().int().min(2020).max(2025),
+  maxResultsPerCity: import_zod.z.number().int().min(1).max(50),
+  concurrency: import_zod.z.number().int().min(1).max(5),
+  censusSourceUrl: import_zod.z.string().url(),
+  usZipsSourcePath: NullableString,
+  warnings: import_zod.z.array(import_zod.z.string()),
+  extractedAt: import_zod.z.string(),
+  selectedCityCount: import_zod.z.number().int().min(0),
+  totalResultCount: import_zod.z.number().int().min(0),
+  csvPath: NullableString,
+  cities: import_zod.z.array(import_zod.z.object({
+    city: import_zod.z.string(),
+    state: import_zod.z.string(),
+    location: import_zod.z.string(),
+    cityKey: import_zod.z.string(),
+    censusName: import_zod.z.string(),
+    population: import_zod.z.number().int().min(0),
+    populationYear: import_zod.z.number().int().min(2020).max(2025),
+    zips: import_zod.z.array(import_zod.z.string()),
+    counties: import_zod.z.array(import_zod.z.string()),
+    status: import_zod.z.enum(["ok", "empty", "failed"]),
+    error: NullableString,
+    resultCount: import_zod.z.number().int().min(0),
+    durationMs: import_zod.z.number().int().min(0),
+    results: import_zod.z.array(DirectoryMapsBusinessOutput)
+  })),
+  durationMs: import_zod.z.number().int().min(0)
+};
 var OrganicResultOutput = import_zod.z.object({
   position: import_zod.z.number().int(),
   title: import_zod.z.string(),
@@ -1269,8 +1420,8 @@ var SearchSerpInputSchema = {
   gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from location or user language."),
   hl: import_zod.z.string().default("en").describe("Google interface/content language inferred from user request."),
   device: import_zod.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use desktop by default; use mobile only when the user asks for mobile rankings."),
-  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default so city/state searches create or reuse a matching residential proxy. Use configured for the static configured proxy. Use none only for direct-network debugging."),
-  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use only when the user gives a specific ZIP or city-center proxy targeting needs to be forced."),
+  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy targeting mode. Use location by default for US city/state SERPs; it creates a fresh residential proxy ID per attempt and retries CAPTCHA, proxy tunnel failure, and wrong-location evidence before returning. Use configured only for the static configured proxy. Use none only for direct-network debugging."),
+  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting. Use when the user gives a specific ZIP or when city-center targeting needs to be forced. With proxyMode location this ZIP is used for each fresh proxy attempt."),
   debug: import_zod.z.boolean().default(false).describe("Include sanitized browser/session/location diagnostics in the response. Use true when debugging localization, CAPTCHA, or proxy behavior."),
   pages: import_zod.z.number().int().min(1).max(2).default(1).describe("Number of result pages to fetch (1\u20132)")
 };
@@ -1280,8 +1431,8 @@ var CaptureSerpSnapshotInputSchema = {
   gl: import_zod.z.string().length(2).default("us").describe("Google country code inferred from the requested market, e.g. us, gb, ca, au."),
   hl: import_zod.z.string().default("en").describe("Google interface/content language inferred from the user request."),
   device: import_zod.z.enum(["desktop", "mobile"]).default("desktop").describe("SERP device context. Use mobile only when the user asks for mobile rankings or mobile SERP evidence."),
-  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized residential proxy targeting, configured for the static residential proxy, and none only for direct-network debugging."),
-  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed."),
+  proxyMode: import_zod.z.enum(["location", "configured", "none"]).default("location").describe("Proxy behavior for capture. Use location for localized US residential evidence; it creates a fresh proxy ID per attempt and retries CAPTCHA, proxy tunnel failure, and wrong-location evidence before returning. Use configured only for the static residential proxy, and none only for direct-network debugging."),
+  proxyZip: import_zod.z.string().regex(/^\d{5}$/).optional().describe("Optional US ZIP override for residential location proxy targeting when a precise city-center or ZIP proxy is needed. With proxyMode location this ZIP is used for each fresh proxy attempt."),
   pages: import_zod.z.number().int().min(1).max(2).default(1).describe("Number of Google result pages to capture. Use 1 normally and 2 only when the user needs deeper ranking evidence."),
   debug: import_zod.z.boolean().default(false).describe("Include sanitized browser, proxy, and location diagnostics. Use true when debugging localization, CAPTCHA, proxy selection, or capture reliability."),
   includePageSnapshots: import_zod.z.boolean().default(false).describe("Also capture ranking-page snapshots for selected SERP URLs through the same product capture path."),
@@ -1356,14 +1507,14 @@ function buildPaaExtractorMcpServer(executor2, options = {}) {
   if (savesReports) registerSavedReportResources(server2);
   server2.registerTool("harvest_paa", {
     title: "Google PAA + SERP Harvest",
-    description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
+    description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). For US local SERPs, leave proxyMode as location so the service uses fresh residential proxy IDs across retries and rejects wrong-location evidence instead of returning a bad market. Use maxQuestions 30 normally, 100-200 for "full", "deep", "all", or comprehensive research. Deep harvests above 100 questions can run for several minutes with no interim progress \u2014 warn the user before starting one and keep maxQuestions at or below 100 unless they explicitly want a deep harvest. Credits are charged by extracted question; unused request hold is refunded.'),
     inputSchema: HarvestPaaInputSchema,
     outputSchema: HarvestPaaOutputSchema,
     annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
   }, async (input) => formatHarvestPaa(await executor2.harvestPaa(input), input));
   server2.registerTool("search_serp", {
     title: "Google SERP Lookup",
-    description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request."),
+    description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request. For US city/state rankings, keep proxyMode as location and pass proxyZip when a city-center ZIP is known; location mode uses fresh residential proxy IDs and retries CAPTCHA, proxy tunnel failures, and wrong-location evidence before returning."),
     inputSchema: SearchSerpInputSchema,
     outputSchema: SearchSerpOutputSchema,
     annotations: liveWebToolAnnotations("Google SERP Lookup")
@@ -1431,11 +1582,18 @@ function buildPaaExtractorMcpServer(executor2, options = {}) {
   }, async (input) => formatMapsPlaceIntel(await executor2.mapsPlaceIntel(input), input));
   server2.registerTool("maps_search", {
     title: "Google Maps Business Search",
-    description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, and profile metadata. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
+    description: withReportNote('Search Google Maps for multiple businesses/profiles by category, niche, keyword, or local market. Use this when the user asks for several Google Business Profiles, GMBs, GBPs, leads, prospects, competitors, or "more than the 3-pack." For US city/state Maps searches, keep proxyMode as location so the browser service can create a fresh residential proxy ID for that market; pass proxyZip only when a specific ZIP or city-center ZIP is known. Returns up to 50 candidates with names, place URLs, CIDs when available, ratings, review counts, and profile metadata. Default maxResults is 10; maximum is 50. Use maps_place_intel afterward only when a selected business needs full details and reviews.'),
     inputSchema: MapsSearchInputSchema,
     outputSchema: MapsSearchOutputSchema,
     annotations: liveWebToolAnnotations("Google Maps Business Search")
   }, async (input) => formatMapsSearch(await executor2.mapsSearch(input), input));
+  server2.registerTool("directory_workflow", {
+    title: "Directory Workflow: Markets + Maps",
+    description: withReportNote('Build directory/prospecting datasets by selecting US city markets from the free Census Population Estimates city/place dataset, optionally joining configured US ZIPS/Lead Magician ZIP groups, then running Google Maps business searches for each city in parallel. Use this when the user wants "all cities over 100k population in a state", "build a directory CSV", "find markets then get Maps data", or similar location-database + Maps workflows. Set minPopulation, state, query, maxResultsPerCity, and concurrency. Use concurrency up to 5 for parallel city sessions. Keep proxyMode as location so each city can use a fresh residential proxy ID when the browser service is available; retryable city failures use fresh proxies across attempts. Saved CSV rows include source_location, result_position, business_name, review_stars, category, address, phone, hours_status, website_url, directions_url, place_url, cid, cid_decimal, city population, and ZIP groups. This workflow captures star ratings from Maps list cards, not profile review counts; use maps_place_intel only when a selected profile needs deeper review details. For local Lead Magician ZIP enrichment, set MCP_SCRAPER_USZIPS_CSV_PATH on the API server or pass usZipsCsvPath only in local/test mode.'),
+    inputSchema: DirectoryWorkflowInputSchema,
+    outputSchema: DirectoryWorkflowOutputSchema,
+    annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
+  }, async (input) => formatDirectoryWorkflow(await executor2.directoryWorkflow(input), input));
   server2.registerTool("credits_info", {
     title: "MCP Scraper Credits & Costs",
     description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and optionally recent credit ledger entries. Does not expose payment methods or credit card information.",