mcp-scraper 0.1.7 → 0.1.8

package/dist/bin/api-server.js

@@ -17,8 +17,8 @@ loadDotEnv();
 async function main() {
   const [{ serve }, { app }, { startWorker }, { migrate }] = await Promise.all([
     import("@hono/node-server"),
-    import("../server-YNJHP5PU.js"),
-    import("../worker-PBG6LGET.js"),
+    import("../server-QXVVTKJP.js"),
+    import("../worker-AUCXFHEL.js"),
     import("../db-YWCNHBLH.js")
   ]);
   const PORT = parseInt(process.env.PORT ?? "3001");

package/dist/bin/mcp-stdio-server.cjs

@@ -128,7 +128,7 @@ var HttpMcpToolExecutor = class {
 var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
 
 // src/version.ts
-var PACKAGE_VERSION = "0.1.7";
+var PACKAGE_VERSION = "0.1.8";
 
 // src/mcp/mcp-tool-schemas.ts
 var import_zod = require("zod");
@@ -224,6 +224,120 @@ var MapsSearchOutputSchema = {
   })),
   durationMs: import_zod.z.number().int().min(0)
 };
+var OrganicResultOutput = import_zod.z.object({
+  position: import_zod.z.number().int(),
+  title: import_zod.z.string(),
+  url: import_zod.z.string(),
+  domain: import_zod.z.string(),
+  snippet: NullableString
+});
+var AiOverviewOutput = import_zod.z.object({
+  detected: import_zod.z.boolean(),
+  text: NullableString
+}).nullable();
+var EntityIdsOutput = import_zod.z.object({
+  kgIds: import_zod.z.array(import_zod.z.string()),
+  cids: import_zod.z.array(import_zod.z.string()),
+  gcids: import_zod.z.array(import_zod.z.string())
+}).nullable();
+var HarvestPaaOutputSchema = {
+  query: import_zod.z.string(),
+  location: NullableString,
+  questionCount: import_zod.z.number().int().min(0),
+  completionStatus: NullableString,
+  questions: import_zod.z.array(import_zod.z.object({
+    question: import_zod.z.string(),
+    answer: NullableString,
+    sourceTitle: NullableString,
+    sourceSite: NullableString
+  })),
+  organicResults: import_zod.z.array(OrganicResultOutput),
+  aiOverview: AiOverviewOutput,
+  entityIds: EntityIdsOutput,
+  durationMs: import_zod.z.number().min(0).nullable()
+};
+var SearchSerpOutputSchema = {
+  query: import_zod.z.string(),
+  location: NullableString,
+  organicResults: import_zod.z.array(OrganicResultOutput),
+  localPack: import_zod.z.array(import_zod.z.object({
+    position: import_zod.z.number().int(),
+    name: import_zod.z.string(),
+    rating: NullableString,
+    reviewCount: NullableString,
+    websiteUrl: NullableString
+  })),
+  aiOverview: AiOverviewOutput,
+  entityIds: EntityIdsOutput
+};
+var ExtractUrlOutputSchema = {
+  url: import_zod.z.string(),
+  title: NullableString,
+  headings: import_zod.z.array(import_zod.z.object({
+    level: import_zod.z.number().int(),
+    text: import_zod.z.string()
+  })),
+  schemaBlockCount: import_zod.z.number().int().min(0),
+  entityName: NullableString,
+  entityTypes: import_zod.z.array(import_zod.z.string()),
+  napScore: import_zod.z.number().nullable(),
+  missingSchemaFields: import_zod.z.array(import_zod.z.string()),
+  screenshotSaved: NullableString
+};
+var ExtractSiteOutputSchema = {
+  url: import_zod.z.string(),
+  pageCount: import_zod.z.number().int().min(0),
+  pages: import_zod.z.array(import_zod.z.object({
+    url: import_zod.z.string(),
+    title: NullableString,
+    schemaTypes: import_zod.z.array(import_zod.z.string())
+  })),
+  durationMs: import_zod.z.number().min(0)
+};
+var MapsPlaceIntelOutputSchema = {
+  name: import_zod.z.string(),
+  rating: NullableString,
+  reviewCount: NullableString,
+  category: NullableString,
+  address: NullableString,
+  phone: NullableString,
+  website: NullableString,
+  hoursSummary: NullableString,
+  bookingUrl: NullableString,
+  kgmid: NullableString,
+  cidDecimal: NullableString,
+  cidUrl: NullableString,
+  lat: import_zod.z.number().nullable(),
+  lng: import_zod.z.number().nullable(),
+  reviewsStatus: import_zod.z.string(),
+  reviewsCollected: import_zod.z.number().int().min(0),
+  reviewTopics: import_zod.z.array(import_zod.z.object({
+    label: import_zod.z.string(),
+    count: import_zod.z.string()
+  }))
+};
+var CreditsInfoOutputSchema = {
+  balanceCredits: import_zod.z.number().nullable(),
+  matchedCost: import_zod.z.object({
+    label: import_zod.z.string(),
+    credits: import_zod.z.number(),
+    unit: import_zod.z.string(),
+    notes: NullableString
+  }).nullable(),
+  costs: import_zod.z.array(import_zod.z.object({
+    key: import_zod.z.string(),
+    label: import_zod.z.string(),
+    credits: import_zod.z.number(),
+    unit: import_zod.z.string(),
+    notes: NullableString
+  })),
+  ledger: import_zod.z.array(import_zod.z.object({
+    createdAt: import_zod.z.string(),
+    operation: import_zod.z.string(),
+    credits: import_zod.z.number(),
+    description: NullableString
+  }))
+};
 var MapSiteUrlsOutputSchema = {
   startUrl: import_zod.z.string(),
   totalFound: import_zod.z.number().int().min(0),
@@ -434,7 +548,7 @@ function debugSection(debug) {
   if (!debug || typeof debug !== "object") return "";
   const request = debug.request ?? {};
   const browser = debug.browser ?? {};
-  const kernel = browser.kernel ?? {};
+  const kernel = browser.browserRuntime ?? browser.kernel ?? {};
   const network = browser.networkLocation ?? {};
   const nav = browser.serpNavigation ?? {};
   const proxyResolution = kernel.proxyResolution ?? {};
@@ -460,12 +574,14 @@ function errorAttemptsSection(body) {
   const lines = attempts.slice(0, 5).map((attempt) => {
     const debug = attempt.debug ?? {};
     const browser = debug.browser ?? {};
-    const kernel = browser.kernel ?? {};
+    const kernel = browser.browserRuntime ?? browser.kernel ?? {};
     const proxyResolution = kernel.proxyResolution ?? {};
     const network = browser.networkLocation ?? {};
     const nav = browser.serpNavigation ?? {};
     const geo = [network.ip, network.city, network.region].filter(Boolean).join(" / ") || "geo unknown";
-    return `- Attempt ${attempt.attempt_number ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 session ${attempt.kernel_session_id ?? kernel.sessionId ?? "unknown"} \xB7 proxy ${debug.request?.proxyMode ?? kernel.proxyMode ?? "unknown"}${proxyResolution.source ? `/${proxyResolution.source}` : ""} \xB7 ${geo} \xB7 CAPTCHA ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 deleted ${attempt.kernel_delete_succeeded === true ? "yes" : attempt.kernel_delete_succeeded === false ? "no" : "unknown"}`;
+    const sessionId = attempt.browser_session_id ?? attempt.kernel_session_id ?? kernel.sessionId ?? "unknown";
+    const cleanupSucceeded = attempt.session_cleanup_succeeded ?? attempt.kernel_delete_succeeded;
+    return `- Attempt ${attempt.attempt_number ?? "?"}: ${attempt.outcome ?? attempt.status ?? "unknown"} \xB7 session ${sessionId} \xB7 proxy ${debug.request?.proxyMode ?? kernel.proxyMode ?? "unknown"}${proxyResolution.source ? `/${proxyResolution.source}` : ""} \xB7 ${geo} \xB7 CAPTCHA ${nav.captchaDetected === true ? "yes" : nav.captchaDetected === false ? "no" : "unknown"} \xB7 cleanup ${cleanupSucceeded === true ? "yes" : cleanupSucceeded === false ? "no" : "unknown"}`;
   });
   return `
 
@@ -512,7 +628,31 @@ ${serpRows}` : "";
   const full = `# PAA Report: "${input.query}"${input.location ? ` \xB7 ${input.location}` : ""}
 
 ${paaTable}${serpTable}${entityIdsSection(entityIds)}${aiSection}${statsLine}${debugSection(diagnostics?.debug)}${tips}`;
-  return oneBlock(full);
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      query: input.query,
+      location: input.location ?? null,
+      questionCount: flat.length,
+      completionStatus: diagnostics?.completionStatus ?? null,
+      questions: flat.map((r) => ({
+        question: String(r.question ?? ""),
+        answer: r.answer ?? null,
+        sourceTitle: r.source_title ?? null,
+        sourceSite: r.source_site ?? null
+      })),
+      organicResults: organic.map((r) => ({
+        position: Number(r.position) || 0,
+        title: String(r.title ?? ""),
+        url: String(r.url ?? ""),
+        domain: String(r.domain ?? ""),
+        snippet: r.snippet ?? null
+      })),
+      aiOverview: aiOvw ? { detected: aiOvw.detected === true, text: aiOvw.text ?? null } : null,
+      entityIds: entityIds ? { kgIds: entityIds.kgIds ?? [], cids: entityIds.cids ?? [], gcids: entityIds.gcids ?? [] } : null,
+      durationMs: durationMs ?? null
+    }
+  };
 }
 function formatSearchSerp(raw, input) {
   const parsed = parseData(raw);
@@ -550,7 +690,29 @@ ${localRows}` : "";
   const full = `# SERP Report: "${input.query}"${input.location ? ` \xB7 ${input.location}` : ""}
 
 ${serpTable}${localSection}${entityIdsSection(entityIds)}${aiSection}${debugSection(diagnostics?.debug)}${tips}`;
-  return oneBlock(full);
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      query: input.query,
+      location: input.location ?? null,
+      organicResults: organic.map((r) => ({
+        position: Number(r.position) || 0,
+        title: String(r.title ?? ""),
+        url: String(r.url ?? ""),
+        domain: String(r.domain ?? ""),
+        snippet: r.snippet ?? null
+      })),
+      localPack: localPack.map((b) => ({
+        position: Number(b.position) || 0,
+        name: String(b.name ?? ""),
+        rating: b.rating ?? null,
+        reviewCount: b.reviewCount ?? null,
+        websiteUrl: b.websiteUrl ?? null
+      })),
+      aiOverview: aiOvw ? { detected: aiOvw.detected === true, text: aiOvw.text ?? null } : null,
+      entityIds: entityIds ? { kgIds: entityIds.kgIds ?? [], cids: entityIds.cids ?? [], gcids: entityIds.gcids ?? [] } : null
+    }
+  };
 }
 function formatExtractUrl(raw, input) {
   const parsed = parseData(raw);
@@ -619,15 +781,27 @@ ${bodyMd.slice(0, 3e3)}${bodyMd.length > 3e3 ? "\n\n*(truncated)*" : ""}` : "";
 **${title}**
 ${headingSection}${kpoSection}${brandingSection}${bodySection}${screenshotSection}${mediaSection}${tips}`;
   const textResult = oneBlock(full);
+  const structuredContent = {
+    url,
+    title: d.title ?? null,
+    headings: headings.map((h) => ({ level: Number(h.level) || 0, text: String(h.text ?? "") })),
+    schemaBlockCount: schemaCount,
+    entityName: kpo?.entityName ?? null,
+    entityTypes: kpo?.type ?? [],
+    napScore: kpo?.napScore ?? null,
+    missingSchemaFields: kpo?.missingFields ?? [],
+    screenshotSaved: screenshotPath ?? null
+  };
   if (screenshotMeta?.base64) {
     return {
       content: [
         ...textResult.content,
         { type: "image", data: screenshotMeta.base64, mimeType: "image/png" }
-      ]
+      ],
+      structuredContent
     };
   }
-  return textResult;
+  return { ...textResult, structuredContent };
 }
 function formatMapSiteUrls(raw, input) {
   const parsed = parseData(raw);
@@ -697,7 +871,19 @@ ${pageRows}`,
 - Map URLs first: use \`map_site_urls\`
 - Inspect a single page: use \`extract_url\``
   ].join("\n");
-  return oneBlock(full);
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      url: input.url,
+      pageCount: pages.length,
+      pages: pages.map((p) => ({
+        url: String(p.url ?? ""),
+        title: p.title ?? null,
+        schemaTypes: p.kpo?.type ?? []
+      })),
+      durationMs: d.durationMs ?? 0
+    }
+  };
 }
 function formatYoutubeHarvest(raw, input) {
   const parsed = parseData(raw);
@@ -894,7 +1080,26 @@ ${costRows}` : "",
 |------|-----------|---------|-------------|
 ${ledgerRows}` : ""
   ].filter(Boolean).join("\n");
-  return oneBlock(full);
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      balanceCredits: typeof balance === "number" ? balance : null,
+      matchedCost: matched ? { label: matched.label, credits: matched.credits, unit: matched.unit, notes: matched.notes ?? null } : null,
+      costs: costs.map((c) => ({
+        key: c.key,
+        label: c.label,
+        credits: c.credits,
+        unit: c.unit,
+        notes: c.notes ?? null
+      })),
+      ledger: ledger.map((row) => ({
+        createdAt: String(row.created_at ?? ""),
+        operation: String(row.operation ?? ""),
+        credits: row.amount_mc / 1e3,
+        description: row.description ?? null
+      }))
+    }
+  };
 }
 function formatMapsSearch(raw, input) {
   const parsed = parseData(raw);
@@ -1043,7 +1248,28 @@ ${entitySection}` : null,
 ---
 *Extracted in ${(durationMs / 1e3).toFixed(1)}s*` : null
   ].filter(Boolean).join("\n");
-  return oneBlock(full);
+  return {
+    ...oneBlock(full),
+    structuredContent: {
+      name,
+      rating: rating ?? null,
+      reviewCount: reviewCount ?? null,
+      category: category ?? null,
+      address: address ?? null,
+      phone: phone ?? null,
+      website: website ?? null,
+      hoursSummary: hoursSummary ?? null,
+      bookingUrl: bookingUrl ?? null,
+      kgmid: kgmid ?? null,
+      cidDecimal: cidDecimal ?? null,
+      cidUrl: cidUrl ?? null,
+      lat: lat ?? null,
+      lng: lng ?? null,
+      reviewsStatus,
+      reviewsCollected: reviews.length,
+      reviewTopics: topics.map((t) => ({ label: String(t.label ?? ""), count: String(t.count ?? "") }))
+    }
+  };
 }
 function formatFacebookAdTranscribe(raw, input) {
   const parsed = parseData(raw);
@@ -1095,18 +1321,21 @@ function buildPaaExtractorMcpServer(executor2, options = {}) {
     title: "Google PAA + SERP Harvest",
     description: withReportNote('Best default tool for Google search research. Extracts People Also Ask questions plus answers/source URLs, organic SERP, local pack when present, entity IDs (CID/GCID/KG MID), and AI Overview. Infer the user language: split topic from location (e.g. "best hvac company in Denver CO" => query "best hvac company", location "Denver, CO", gl "us", hl "en"). Use maxQuestions 30 normally, 100-150 for "full", "deep", "all", or comprehensive research. Credits are charged by extracted question; unused request hold is refunded.'),
     inputSchema: HarvestPaaInputSchema,
+    outputSchema: HarvestPaaOutputSchema,
     annotations: liveWebToolAnnotations("Google PAA + SERP Harvest")
   }, async (input) => formatHarvestPaa(await executor2.harvestPaa(input), input));
   server2.registerTool("search_serp", {
     title: "Google SERP Lookup",
     description: withReportNote("Fast Google SERP lookup without PAA expansion. Use when the user asks for rankings, organic results, local pack, quick SERP, or positions. Split topic from location and infer gl/hl from the user request."),
     inputSchema: SearchSerpInputSchema,
+    outputSchema: SearchSerpOutputSchema,
     annotations: liveWebToolAnnotations("Google SERP Lookup")
   }, async (input) => formatSearchSerp(await executor2.searchSerp(input), input));
   server2.registerTool("extract_url", {
     title: "Single URL Extract",
     description: withReportNote("Extract structured data from one public URL: page content as Markdown, heading structure, JSON-LD schema, entity details, NAP score, metadata, and missing schema fields. Use when the user provides a single URL or asks to inspect/scrape one page."),
     inputSchema: ExtractUrlInputSchema,
+    outputSchema: ExtractUrlOutputSchema,
     annotations: liveWebToolAnnotations("Single URL Extract")
   }, async (input) => formatExtractUrl(await executor2.extractUrl(input), input));
   server2.registerTool("map_site_urls", {
@@ -1120,6 +1349,7 @@ function buildPaaExtractorMcpServer(executor2, options = {}) {
     title: "Multi-Page Site Extract",
     description: withReportNote("Run multi-page extraction across a public website. Returns per-page titles, H1s, metadata, headings, schema/entity data, canonical URLs, and content. Use for website audits, competitor audits, and full-site extraction."),
     inputSchema: ExtractSiteInputSchema,
+    outputSchema: ExtractSiteOutputSchema,
     annotations: liveWebToolAnnotations("Multi-Page Site Extract")
   }, async (input) => formatExtractSite(await executor2.extractSite(input), input));
   server2.registerTool("youtube_harvest", {
@@ -1159,6 +1389,7 @@ function buildPaaExtractorMcpServer(executor2, options = {}) {
     title: "Google Maps Business Profile Details",
     description: withReportNote('Extract Google Maps business intelligence for one known/named business: rating, review count, category, address, phone, website, hours, booking URL, review histogram, review topics, about attributes, entity IDs, and optional review cards. Do not use this for category searches, local market prospect lists, or requests for multiple GMB/GBP profiles; use maps_search first for those. Split business name from location (e.g. "Elite Roofing Denver CO" => businessName "Elite Roofing", location "Denver, CO"). Pass includeReviews true when the user asks for reviews/customer pain.'),
     inputSchema: MapsPlaceIntelInputSchema,
+    outputSchema: MapsPlaceIntelOutputSchema,
     annotations: liveWebToolAnnotations("Google Maps Business Profile Details")
   }, async (input) => formatMapsPlaceIntel(await executor2.mapsPlaceIntel(input), input));
   server2.registerTool("maps_search", {
@@ -1172,6 +1403,7 @@ function buildPaaExtractorMcpServer(executor2, options = {}) {
     title: "MCP Scraper Credits & Costs",
     description: "Answer questions about MCP Scraper credits: current credit balance, what a specific tool/action costs, the full cost table, and optionally recent credit ledger entries. Does not expose payment methods or credit card information.",
     inputSchema: CreditsInfoInputSchema,
+    outputSchema: CreditsInfoOutputSchema,
     annotations: {
       title: "MCP Scraper Credits & Costs",
       readOnlyHint: true,