mcp-scraper 0.2.5 → 0.2.6

package/dist/bin/mcp-stdio-server.js

@@ -2,9 +2,9 @@
 import {
   HttpMcpToolExecutor,
   buildPaaExtractorMcpServer
-} from "../chunk-ENI4DM3S.js";
+} from "../chunk-7WB2W6FO.js";
 import "../chunk-M2S27J6Z.js";
-import "../chunk-JDX57SEC.js";
+import "../chunk-T6SIPPOQ.js";
 
 // bin/mcp-stdio-server.ts
 import { readFileSync } from "fs";

package/dist/{chunk-ENI4DM3S.js → chunk-7WB2W6FO.js}

@@ -3,7 +3,7 @@ import {
 } from "./chunk-M2S27J6Z.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-JDX57SEC.js";
+} from "./chunk-T6SIPPOQ.js";
 
 // src/harvest-timeout.ts
 var VERCEL_FUNCTION_MAX_MS = 3e5;
@@ -1048,6 +1048,24 @@ var DirectoryWorkflowInputSchema = {
   proxyZip: z.string().regex(/^\d{5}$/).optional().describe("Optional ZIP override for proxy targeting. Normally omit it so each city can use its Lead Magician ZIP group or city/state location."),
   debug: z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics in each Maps browser session when supported.")
 };
+var RankTrackerModeSchema = z.enum(["maps", "organic", "ai_overview", "paa"]);
+var RankTrackerBlueprintInputSchema = {
+  projectName: z.string().min(1).optional().describe("Optional name for the rank tracker project, client, or campaign."),
+  targetDomain: z.string().min(1).optional().describe("Primary domain to track in organic results, AI Overview citations, and PAA sources, e.g. example.com."),
+  targetBusinessName: z.string().min(1).optional().describe("Primary Google Business Profile or brand/business name to match in Maps results. Required for reliable Maps rank tracking."),
+  trackingModes: z.array(RankTrackerModeSchema).min(1).max(4).default(["maps", "organic", "ai_overview", "paa"]).describe("Rank tracker surfaces to build. maps uses directory_workflow/maps_search. organic uses search_serp. ai_overview uses search_serp/harvest_paa. paa uses harvest_paa source presence."),
+  keywords: z.array(z.string().min(1)).max(200).default([]).describe("Seed keywords or service queries to track. Leave empty when the downstream AI should create the keyword table from user input."),
+  locations: z.array(z.string().min(1)).max(100).default([]).describe("Markets, cities, ZIPs, or service areas to track. Use city/state strings like Denver, CO for localized SERP and Maps checks."),
+  competitors: z.array(z.string().min(1)).max(100).default([]).describe("Optional competitor domains or business names to persist as comparison targets."),
+  database: z.enum(["postgres", "neon", "supabase", "sqlite", "mysql"]).default("postgres").describe("Database family the downstream AI should target when generating migrations."),
+  scheduleCadence: z.enum(["daily", "weekly", "monthly", "custom"]).default("weekly").describe("Default recurring rank check cadence for the generated cron/heartbeat plan."),
+  customCron: z.string().min(1).optional().describe("Cron expression to use when scheduleCadence is custom."),
+  timezone: z.string().min(1).default("UTC").describe("IANA timezone for scheduled rank checks, e.g. America/Denver."),
+  includeCron: z.boolean().default(true).describe("Include a cron or heartbeat worker plan. Keep true for production rank trackers."),
+  includeDashboard: z.boolean().default(true).describe("Include dashboard/reporting requirements in the generated prompt."),
+  includeAlerts: z.boolean().default(true).describe("Include alert rules for rank movement and SERP feature gains/losses."),
+  notes: z.string().max(4e3).optional().describe("Extra product, client, stack, or hosting requirements to include in the implementation prompt.")
+};
 var NullableString = z.string().nullable();
 var MapsSearchOutputSchema = {
   query: z.string(),
@@ -1123,6 +1141,39 @@ var DirectoryWorkflowOutputSchema = {
   })),
   durationMs: z.number().int().min(0)
 };
+var RankTrackerToolPlanOutput = z.object({
+  tool: z.string(),
+  purpose: z.string()
+});
+var RankTrackerTableOutput = z.object({
+  name: z.string(),
+  purpose: z.string(),
+  keyColumns: z.array(z.string())
+});
+var RankTrackerCronJobOutput = z.object({
+  name: z.string(),
+  purpose: z.string(),
+  modes: z.array(RankTrackerModeSchema),
+  recommendedTools: z.array(z.string())
+});
+var RankTrackerBlueprintOutputSchema = {
+  projectName: z.string(),
+  targetDomain: NullableString,
+  targetBusinessName: NullableString,
+  trackingModes: z.array(RankTrackerModeSchema),
+  database: z.string(),
+  recommendedTools: z.array(RankTrackerToolPlanOutput),
+  tables: z.array(RankTrackerTableOutput),
+  cron: z.object({
+    enabled: z.boolean(),
+    cadence: z.string(),
+    expression: z.string(),
+    timezone: z.string(),
+    jobs: z.array(RankTrackerCronJobOutput)
+  }),
+  metrics: z.array(z.string()),
+  implementationPrompt: z.string()
+};
 var OrganicResultOutput = z.object({
   position: z.number().int(),
   title: z.string(),
@@ -1337,6 +1388,329 @@ var CaptureSerpPageSnapshotsInputSchema = {
   debug: z.boolean().default(false).describe("Include sanitized browser/proxy diagnostics for page snapshot debugging. Use true for capture, network, or proxy troubleshooting.")
 };
 
+// src/mcp/rank-tracker-blueprint.ts
+var DEFAULT_MODES = ["maps", "organic", "ai_overview", "paa"];
+function normalizeDomain(value) {
+  const trimmed = value?.trim();
+  if (!trimmed) return null;
+  try {
+    const url = new URL(trimmed.includes("://") ? trimmed : `https://${trimmed}`);
+    return url.hostname.replace(/^www\./, "").toLowerCase();
+  } catch {
+    return trimmed.replace(/^https?:\/\//, "").replace(/^www\./, "").split("/")[0]?.toLowerCase() || trimmed;
+  }
+}
+function uniqueModes(input) {
+  const modes = input?.length ? input : DEFAULT_MODES;
+  return [...new Set(modes)];
+}
+function cronExpression(cadence, customCron) {
+  if (cadence === "daily") return "15 6 * * *";
+  if (cadence === "monthly") return "15 6 1 * *";
+  if (cadence === "custom") return customCron?.trim() || "CUSTOM_CRON_EXPRESSION_REQUIRED";
+  return "15 6 * * 1";
+}
+function includes(modes, mode) {
+  return modes.includes(mode);
+}
+function buildToolPlan(modes) {
+  const plans = [];
+  const push = (tool, purpose) => {
+    if (!plans.some((plan) => plan.tool === tool)) plans.push({ tool, purpose });
+  };
+  push("credits_info", "Check account balance and rates before scheduling recurring rank checks.");
+  if (includes(modes, "maps")) {
+    push("directory_workflow", "Seed or refresh city-by-city Maps datasets when building local market coverage.");
+    push("maps_search", "Run recurring Google Maps category checks for each keyword and location.");
+    push("maps_place_intel", "Hydrate selected target or competitor GBP details when profile-level evidence is needed.");
+  }
+  if (includes(modes, "organic")) {
+    push("search_serp", "Capture localized organic rankings, local pack data, and AI Overview status without PAA expansion.");
+  }
+  if (includes(modes, "ai_overview")) {
+    push("search_serp", "Capture quick AI Overview detection and organic context for each tracked query.");
+    push("harvest_paa", "Use when AI Overview text, PAA context, or source-level evidence needs deeper capture.");
+  }
+  if (includes(modes, "paa")) {
+    push("harvest_paa", "Capture People Also Ask questions, answers, and source sites for PAA presence tracking.");
+  }
+  return plans;
+}
+function buildTables(modes, includeDashboard, includeAlerts, hasCompetitors) {
+  const tables = [
+    {
+      name: "rank_tracker_projects",
+      purpose: "One row per client, campaign, or tracked property.",
+      keyColumns: ["id", "name", "target_domain", "target_business_name", "database_mode", "timezone", "created_at"]
+    },
+    {
+      name: "rank_tracker_targets",
+      purpose: "Normalized target identities and match rules for domains, URLs, business names, CIDs, and competitors.",
+      keyColumns: ["id", "project_id", "kind", "value", "display_name", "match_rules_json", "active"]
+    },
+    {
+      name: "rank_tracker_keywords",
+      purpose: "Tracked keywords or service queries, separated from location so calls can localize correctly.",
+      keyColumns: ["id", "project_id", "keyword", "intent", "tags_json", "active"]
+    },
+    {
+      name: "rank_tracker_locations",
+      purpose: "Markets, cities, ZIPs, country/language settings, and proxy targeting hints.",
+      keyColumns: ["id", "project_id", "label", "gl", "hl", "proxy_zip", "device", "active"]
+    },
+    {
+      name: "rank_tracker_runs",
+      purpose: "Heartbeat table for every scheduled or manual rank check batch.",
+      keyColumns: ["id", "project_id", "scheduled_for", "started_at", "completed_at", "status", "idempotency_key", "error_message"]
+    }
+  ];
+  if (includes(modes, "organic")) {
+    tables.push({
+      name: "rank_tracker_organic_results",
+      purpose: "SERP organic result rows from search_serp, including target-domain matches and competitor rows.",
+      keyColumns: ["id", "run_id", "keyword_id", "location_id", "device", "position", "title", "url", "domain", "is_target"]
+    });
+  }
+  if (includes(modes, "maps")) {
+    tables.push({
+      name: "rank_tracker_maps_results",
+      purpose: "Google Maps result rows from maps_search and directory_workflow, including GBP identity fields.",
+      keyColumns: ["id", "run_id", "keyword_id", "location_id", "position", "business_name", "place_url", "cid", "website_url", "rating", "review_count", "is_target"]
+    });
+    tables.push({
+      name: "rank_tracker_market_seeds",
+      purpose: "Directory workflow city, population, ZIP-group, and market seed records used to build local tracking coverage.",
+      keyColumns: ["id", "project_id", "query", "city", "state", "population", "zip_group_json", "last_seeded_run_id"]
+    });
+  }
+  if (includes(modes, "ai_overview")) {
+    tables.push({
+      name: "rank_tracker_ai_overviews",
+      purpose: "AI Overview detection, text snapshot, citation domains, and target citation status per query/location.",
+      keyColumns: ["id", "run_id", "keyword_id", "location_id", "detected", "target_cited", "cited_domains_json", "overview_text"]
+    });
+  }
+  if (includes(modes, "paa")) {
+    tables.push({
+      name: "rank_tracker_paa_sources",
+      purpose: "People Also Ask questions, answers, and source sites, with target source presence flags.",
+      keyColumns: ["id", "run_id", "keyword_id", "location_id", "question", "answer", "source_title", "source_site", "target_present"]
+    });
+  }
+  if (hasCompetitors) {
+    tables.push({
+      name: "rank_tracker_competitor_snapshots",
+      purpose: "Per-run competitor visibility summaries across organic, Maps, AI Overview, and PAA surfaces.",
+      keyColumns: ["id", "run_id", "target_id", "surface", "best_position", "presence_count", "share_of_surface"]
+    });
+  }
+  if (includeDashboard) {
+    tables.push({
+      name: "rank_tracker_daily_metrics",
+      purpose: "Rollup table for dashboard charts, trend lines, and share-of-visibility summaries.",
+      keyColumns: ["id", "project_id", "metric_date", "keyword_id", "location_id", "surface", "metric_name", "metric_value"]
+    });
+  }
+  if (includeAlerts) {
+    tables.push({
+      name: "rank_tracker_alert_events",
+      purpose: "Deduplicated gain/loss events for rank movement, Maps top 3, AI Overview citations, and PAA source presence.",
+      keyColumns: ["id", "project_id", "run_id", "surface", "severity", "event_key", "message", "created_at", "acknowledged_at"]
+    });
+  }
+  return tables;
+}
+function buildCronJobs(modes, includeDashboard, includeAlerts) {
+  const jobs = [
+    {
+      name: "rank-tracker-dispatch",
+      purpose: "Find due keyword/location/mode combinations, create a rank_tracker_runs row, and enqueue idempotent work items.",
+      modes,
+      recommendedTools: ["credits_info"]
+    }
+  ];
+  if (includes(modes, "maps")) {
+    jobs.push({
+      name: "maps-rank-check",
+      purpose: "Run maps_search per keyword/location and optionally directory_workflow for market seed refreshes.",
+      modes: ["maps"],
+      recommendedTools: ["maps_search", "directory_workflow", "maps_place_intel"]
+    });
+  }
+  if (includes(modes, "organic") || includes(modes, "ai_overview")) {
+    jobs.push({
+      name: "serp-rank-check",
+      purpose: "Run search_serp per keyword/location/device and persist organic ranks, local pack, and AI Overview status.",
+      modes: modes.filter((mode) => mode === "organic" || mode === "ai_overview"),
+      recommendedTools: ["search_serp"]
+    });
+  }
+  if (includes(modes, "paa") || includes(modes, "ai_overview")) {
+    jobs.push({
+      name: "paa-ai-evidence-check",
+      purpose: "Run harvest_paa for PAA source presence and deeper AI Overview/PAA evidence where needed.",
+      modes: modes.filter((mode) => mode === "paa" || mode === "ai_overview"),
+      recommendedTools: ["harvest_paa"]
+    });
+  }
+  if (includeDashboard) {
+    jobs.push({
+      name: "rank-tracker-rollups",
+      purpose: "Aggregate raw rows into daily metrics for charts, comparison tables, and trend lines.",
+      modes,
+      recommendedTools: []
+    });
+  }
+  if (includeAlerts) {
+    jobs.push({
+      name: "rank-tracker-alerts",
+      purpose: "Compare latest run against prior snapshots and emit deduplicated visibility gain/loss events.",
+      modes,
+      recommendedTools: []
+    });
+  }
+  return jobs;
+}
+function buildMetrics(modes, includeDashboard, includeAlerts) {
+  const metrics = ["run_success_rate", "last_successful_run_at", "stale_keyword_location_count"];
+  if (includes(modes, "maps")) {
+    metrics.push("maps_best_position", "maps_top_3_presence", "maps_target_found", "maps_competitor_count_above_target");
+  }
+  if (includes(modes, "organic")) {
+    metrics.push("organic_best_position", "organic_top_3_presence", "organic_top_10_presence", "organic_best_url");
+  }
+  if (includes(modes, "ai_overview")) {
+    metrics.push("ai_overview_detected", "target_ai_overview_cited", "ai_overview_citation_domain_count");
+  }
+  if (includes(modes, "paa")) {
+    metrics.push("paa_question_count", "target_paa_source_count", "paa_presence_rate", "paa_source_domains");
+  }
+  if (includeDashboard) metrics.push("share_of_visibility", "position_delta_7d", "position_delta_30d");
+  if (includeAlerts) metrics.push("visibility_gain_events", "visibility_loss_events");
+  return metrics;
+}
+function listOrPlaceholder(values, fallback) {
+  const clean = values?.map((value) => value.trim()).filter(Boolean) ?? [];
+  if (!clean.length) return fallback;
+  return clean.map((value) => `- ${value}`).join("\n");
+}
+function buildPrompt(input, modes, tools, tables, jobs, metrics, expression, targetDomain, targetBusinessName) {
+  const keywords = listOrPlaceholder(input.keywords, "- TODO: add tracked keywords");
+  const locations = listOrPlaceholder(input.locations, "- TODO: add tracked markets/locations");
+  const competitors = listOrPlaceholder(input.competitors, "- Optional: add competitor domains or GBP names");
+  const notes = input.notes?.trim() || "No extra implementation notes.";
+  return [
+    "You are building a production rank tracker powered by MCP Scraper. Create the database migrations, scheduled worker, ingestion functions, and dashboard/query layer described below.",
+    "",
+    `Project: ${input.projectName || "MCP Scraper Rank Tracker"}`,
+    `Target domain: ${targetDomain || "TODO_TARGET_DOMAIN"}`,
+    `Target business/GBP name: ${targetBusinessName || "TODO_TARGET_BUSINESS_NAME"}`,
+    `Database target: ${input.database || "postgres"}`,
+    `Tracking modes: ${modes.join(", ")}`,
+    "",
+    "Seed keywords:",
+    keywords,
+    "",
+    "Seed locations:",
+    locations,
+    "",
+    "Competitors:",
+    competitors,
+    "",
+    "MCP Scraper calls to use:",
+    tools.map((tool) => `- ${tool.tool}: ${tool.purpose}`).join("\n"),
+    "",
+    "Database tables to create:",
+    tables.map((table) => `- ${table.name}: ${table.keyColumns.join(", ")}`).join("\n"),
+    "",
+    "Scheduler and heartbeat requirements:",
+    `- Use cron expression "${expression}" in timezone "${input.timezone || "UTC"}" unless the user changes cadence.`,
+    "- Every scheduled batch must insert or update rank_tracker_runs before calling MCP tools.",
+    "- Use idempotency_key = project_id + keyword_id + location_id + mode + device + scheduled_date.",
+    "- Mark runs as running, succeeded, failed, or skipped; persist error_message on failures.",
+    "- Run MCP calls sequentially by default because accounts have one concurrency slot unless the user bought more.",
+    "- Retry retryable upstream failures with backoff, but do not duplicate rows for the same idempotency key.",
+    "",
+    "Mode-specific ingestion rules:",
+    "- maps: Use directory_workflow to seed/refresh market coverage and maps_search for recurring keyword/location rank checks. Match the target by targetBusinessName, website domain, CID, and place URL when available.",
+    "- organic: Use search_serp and persist every organic result row. Compute the best targetDomain position and best ranking URL per keyword/location/device.",
+    "- ai_overview: Use search_serp for quick detection and harvest_paa when deeper text/source evidence is needed. Track detected, overview_text, cited domains, and whether targetDomain is cited.",
+    "- paa: Use harvest_paa and persist each question/source pair. Track whether sourceSite or answer/source evidence matches targetDomain, then compute PAA presence rate.",
+    "",
+    "Metrics to expose:",
+    metrics.map((metric) => `- ${metric}`).join("\n"),
+    "",
+    "Alert requirements:",
+    input.includeAlerts ? "- Emit deduplicated alert_events when target gains/loses Maps top 3, organic top 10, AI Overview citation, or PAA source presence." : "- Alerts are out of scope for this build unless the user asks for them.",
+    "",
+    "Dashboard requirements:",
+    input.includeDashboard ? "- Build views for latest position by keyword/location, trend deltas, competitors above target, AI Overview citations, and PAA source wins/losses." : "- Dashboard is out of scope for this build unless the user asks for it.",
+    "",
+    "Extra notes:",
+    notes
+  ].join("\n");
+}
+function buildRankTrackerBlueprint(input) {
+  const trackingModes = uniqueModes(input.trackingModes);
+  const targetDomain = normalizeDomain(input.targetDomain);
+  const targetBusinessName = input.targetBusinessName?.trim() || null;
+  const projectName = input.projectName?.trim() || "MCP Scraper Rank Tracker";
+  const database = input.database || "postgres";
+  const expression = cronExpression(input.scheduleCadence || "weekly", input.customCron);
+  const tools = buildToolPlan(trackingModes);
+  const tables = buildTables(trackingModes, input.includeDashboard !== false, input.includeAlerts !== false, Boolean(input.competitors?.length));
+  const jobs = input.includeCron === false ? [] : buildCronJobs(trackingModes, input.includeDashboard !== false, input.includeAlerts !== false);
+  const metrics = buildMetrics(trackingModes, input.includeDashboard !== false, input.includeAlerts !== false);
+  const implementationPrompt = buildPrompt(input, trackingModes, tools, tables, jobs, metrics, expression, targetDomain, targetBusinessName);
+  const blueprint = {
+    projectName,
+    targetDomain,
+    targetBusinessName,
+    trackingModes,
+    database,
+    recommendedTools: tools,
+    tables,
+    cron: {
+      enabled: input.includeCron !== false,
+      cadence: input.scheduleCadence || "weekly",
+      expression: input.includeCron === false ? "disabled" : expression,
+      timezone: input.timezone || "UTC",
+      jobs
+    },
+    metrics,
+    implementationPrompt
+  };
+  const text = [
+    `# ${projectName}`,
+    "",
+    "Rank tracker build blueprint generated from MCP Scraper tool capabilities.",
+    "",
+    "## Modes",
+    trackingModes.map((mode) => `- ${mode}`).join("\n"),
+    "",
+    "## Recommended MCP Tools",
+    tools.map((tool) => `- \`${tool.tool}\` - ${tool.purpose}`).join("\n"),
+    "",
+    "## Database Tables",
+    tables.map((table) => `- \`${table.name}\` - ${table.purpose}`).join("\n"),
+    "",
+    "## Cron / Heartbeat",
+    blueprint.cron.enabled ? `- Cadence: ${blueprint.cron.cadence}
+- Cron: \`${blueprint.cron.expression}\`
+- Timezone: ${blueprint.cron.timezone}
+- Jobs: ${jobs.map((job) => job.name).join(", ")}` : "- Disabled by input. Still create rank_tracker_runs for manual runs.",
+    "",
+    "## Implementation Prompt",
+    "```text",
+    implementationPrompt,
+    "```"
+  ].join("\n");
+  return {
+    content: [{ type: "text", text }],
+    structuredContent: blueprint
+  };
+}
+
 // src/mcp/paa-mcp-server.ts
 function liveWebToolAnnotations(title) {
   return {
@@ -1347,6 +1721,15 @@ function liveWebToolAnnotations(title) {
     openWorldHint: true
   };
 }
+function localPlanningToolAnnotations(title) {
+  return {
+    title,
+    readOnlyHint: true,
+    destructiveHint: false,
+    idempotentHint: true,
+    openWorldHint: false
+  };
+}
 function listSavedReports() {
   try {
     const dir = outputBaseDir();
@@ -1480,6 +1863,13 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
     outputSchema: DirectoryWorkflowOutputSchema,
     annotations: liveWebToolAnnotations("Directory Workflow: Markets + Maps")
   }, async (input) => formatDirectoryWorkflow(await executor.directoryWorkflow(input), input));
+  server.registerTool("rank_tracker_blueprint", {
+    title: "Rank Tracker Blueprint Builder",
+    description: "Generate a build-ready database, cron/heartbeat, ingestion, metrics, and AI implementation prompt for a rank tracker powered by MCP Scraper. Supports Maps rankings through directory_workflow/maps_search, organic rankings through search_serp, AI Overview citation tracking, and People Also Ask source presence tracking. This tool is local planning only; it does not call the web or spend credits.",
+    inputSchema: RankTrackerBlueprintInputSchema,
+    outputSchema: RankTrackerBlueprintOutputSchema,
+    annotations: localPlanningToolAnnotations("Rank Tracker Blueprint Builder")
+  }, async (input) => buildRankTrackerBlueprint(input));
   server.registerTool("credits_info", {
     title: "MCP Scraper Credits & Costs",
     description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and optionally recent credit ledger entries. Does not expose payment methods or credit card information.",
@@ -1615,4 +2005,4 @@ export {
   registerPaaExtractorMcpTools,
   HttpMcpToolExecutor
 };
-//# sourceMappingURL=chunk-ENI4DM3S.js.map
+//# sourceMappingURL=chunk-7WB2W6FO.js.map