mcp-scraper 0.3.15 → 0.3.16

package/dist/{chunk-7Y6JDBML.js → chunk-ACIUCZ27.js}

@@ -50,9 +50,9 @@ function renderInstallTerminal(options) {
     "1/1 install surfaces ready",
     colorize("Newest: saved hosted browser profiles for AI visibility. Share the watch_url, user signs in, then verify with browser_profile_status. Browser sessions are direct/no-proxy by default.", "lime", color),
     "",
-    `${colorize("Tools", "cyan", color)}  ${colorize("(44 MCP tools)", "muted", color)}`,
+    `${colorize("Tools", "cyan", color)}  ${colorize("(45 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
-    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "directory_workflow"], color),
+    toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
     toolRow("build", ["rank_tracker_workflow", "cron plan", "database prompt"], color),
     toolRow("media", ["youtube_harvest", "youtube_transcribe", "facebook_ad_search", "facebook_page_intel", "facebook_ad_transcribe", "facebook_video_transcribe", "instagram_profile_content", "instagram_media_download"], color),
     toolRow("browser", ["browser_open", "browser_profile_onboard", "browser_profile_status", "browser_close", "browser_screenshot", "browser_read", "browser_locate", "browser_replay_mark", "browser_replay_annotate"], color),
@@ -100,4 +100,4 @@ function renderInstallTerminal(options) {
 export {
   renderInstallTerminal
 };
-//# sourceMappingURL=chunk-7Y6JDBML.js.map
+//# sourceMappingURL=chunk-ACIUCZ27.js.map

package/dist/chunk-ACIUCZ27.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n  color?: boolean\n  version: string\n  apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n  reset: '\\x1b[0m',\n  cyan: '\\x1b[36m',\n  lime: '\\x1b[32m',\n  amber: '\\x1b[33m',\n  red: '\\x1b[31m',\n  muted: '\\x1b[90m',\n  bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n  if (!enabled) return value\n  return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n  const padded = label.padEnd(9, ' ')\n  return `  ${colorize(padded, 'muted', enabled)} ${tools.join(colorize('  .  ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n  const color = options.color ?? true\n  const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n  const ascii = String.raw`\n __  __  ____ ____ \n|  \\/  |/ ___|  _ \\\n| |\\/| | |   | |_) |\n| |  | | |___|  __/\n|_|  |_|\\____|_|\n\n ____   ____ ____      _    ____  _____ ____\n/ ___| / ___|  _ \\    / \\  |  _ \\| ____|  _ \\\n\\___ \\| |   | |_) |  / _ \\ | |_) |  _| | |_) |\n ___) | |___|  _ <  / ___ \\|  __/| |___|  _ <\n|____/ \\____|_| \\_\\/_/   \\_\\_|   |_____|_| \\_\\\n`\n\n  const claudeCommand = [\n    `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n    '  mcp-scraper-cli agent install claude --apply',\n  ].join('\\n')\n\n  const codexConfig = [\n    '[mcp_servers.mcp-scraper]',\n    'command = \"npx\"',\n    'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper-combined\"]',\n    `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n  ].join('\\n')\n\n  return [\n    colorize(`mcp-scraper v${options.version}`, 'bold', color),\n    colorize('> mcp-scraper-install', 'muted', color),\n    colorize(ascii, 'amber', color),\n    `${colorize('MCP Scraper Agent', 'cyan', color)}  .  v${options.version}  .  mcpscraper.dev`,\n    '1/1 install surfaces ready',\n    colorize('Newest: saved hosted browser profiles for AI visibility. Share the watch_url, user signs in, then verify with browser_profile_status. Browser sessions are direct/no-proxy by default.', 'lime', color),\n    '',\n    `${colorize('Tools', 'cyan', color)}  ${colorize('(45 MCP tools)', 'muted', color)}`,\n    toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n    toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n    toolRow('build', ['rank_tracker_workflow', 'cron plan', 'database prompt'], color),\n    toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download'], color),\n    toolRow('browser', ['browser_open', 'browser_profile_onboard', 'browser_profile_status', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n    toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n    `${colorize('Workflows', 'cyan', color)}  ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n    toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n    toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n    '',\n    colorize('Usage tips:', 'amber', color),\n    'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n    'Run mcp-scraper-combined in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n    'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n    'Customer auth setup: run browser_profile_onboard, send the watch_url, let the user sign in, then call browser_profile_status until AUTHENTICATED.',\n    'Reconnect setup: if a saved profile shows login controls later, run browser_profile_onboard again with the same profile name.',\n    'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n    'For MCP clients, use mcp-scraper-combined so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n    'If you hit the concurrency limit, add an extra slot for $5/month with mcp-scraper-cli billing concurrency checkout.',\n    '',\n    `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n    '',\n    colorize('Setup doctor', 'amber', color),\n    'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n    '',\n    colorize('Hosted profile setup', 'amber', color),\n    'In your MCP client, call browser_profile_onboard with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n    'Give the returned watch_url to the user. After they sign in, call browser_profile_status, then browser_open with the returned profile.',\n    '',\n    colorize('Claude Code one-command setup', 'amber', color),\n    claudeCommand,\n    'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n    '',\n    colorize('Codex config', 'amber', color),\n    codexConfig,\n    '',\n    colorize('Claude Desktop Extension', 'amber', color),\n    'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n    '',\n    colorize('Safety note:', 'muted', color),\n    'mcp-scraper-combined prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n    'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n    '',\n  ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,0LAA0L,QAAQ,KAAK;AAAA,IAChN;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,kBAAkB,SAAS,KAAK,CAAC;AAAA,IAClF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,yBAAyB,aAAa,iBAAiB,GAAG,KAAK;AAAA,IACjF,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,0BAA0B,GAAG,KAAK;AAAA,IAC9N,QAAQ,WAAW,CAAC,gBAAgB,2BAA2B,0BAA0B,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IAC1N,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}

package/dist/{chunk-WJ2LWHPA.js → chunk-ISZWGIWL.js}

@@ -7,7 +7,7 @@ import {
 } from "./chunk-WN7PBKMV.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-2MX5WOII.js";
+} from "./chunk-SOMBZK3V.js";
 
 // src/mcp/browser-agent-mcp-server.ts
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
@@ -1329,4 +1329,4 @@ export {
   buildBrowserAgentMcpServer,
   registerBrowserAgentMcpTools
 };
-//# sourceMappingURL=chunk-WJ2LWHPA.js.map
+//# sourceMappingURL=chunk-ISZWGIWL.js.map

package/dist/chunk-SOMBZK3V.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.3.16";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-SOMBZK3V.js.map

package/dist/chunk-SOMBZK3V.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.3.16'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{server-UOQ2JIUQ.js → server-LPVBSE2E.js}

@@ -92,6 +92,7 @@ import {
   CaptureSerpSnapshotInputSchema,
   CaptureSerpSnapshotOutputSchema,
   HttpMcpToolExecutor,
+  auditImages,
   buildLinkGraph,
   buildPaaExtractorMcpServer,
   computeIssues,
@@ -101,8 +102,9 @@ import {
   harvestTimeoutBudget,
   liveWebToolAnnotations,
   outputBaseDir,
+  renderImageSection,
   renderIssueReport
-} from "./chunk-WQAIOY2D.js";
+} from "./chunk-7XARJHS6.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -138,7 +140,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-LFATOGDF.js";
-import "./chunk-2MX5WOII.js";
+import "./chunk-SOMBZK3V.js";
 
 // src/api/outbound-sanitize.ts
 var KEY_RENAMES = {
@@ -7587,10 +7589,13 @@ function chunk(items, size) {
   for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
   return out;
 }
-function buildArtifacts(job, pages, branding) {
+function buildArtifacts(job, pages, branding, imageAudit) {
   const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
   const issues = computeIssues(pages, metrics);
-  const reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
+  let reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
+  if (imageAudit) reportMd += `
+
+${renderImageSection(imageAudit)}`;
   const pageRows = pages.map((p) => {
     const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
     const m = metrics.get(p.url);
@@ -7604,6 +7609,10 @@ function buildArtifacts(job, pages, branding) {
     { name: "links.jsonl", body: jsonl(edges), type: "application/x-ndjson" },
     { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" }
   ];
+  if (imageAudit) {
+    files.push({ name: "images.jsonl", body: jsonl(imageAudit.rows), type: "application/x-ndjson" });
+    files.push({ name: "images-summary.json", body: JSON.stringify(imageAudit.summary, null, 2), type: "application/json" });
+  }
   if (branding) files.push({ name: "branding.json", body: JSON.stringify(branding, null, 2), type: "application/json" });
   return files;
 }
@@ -7648,10 +7657,14 @@ var siteExtractFn = inngest.createFunction(
       });
     }
     const branding = Array.isArray(job.options.formats) && job.options.formats.includes("branding") ? await step.run("branding", () => extractBranding(job.startUrl, key).catch(() => null)) : null;
+    const imageAudit = Array.isArray(job.options.formats) && job.options.formats.includes("images") ? await step.run("image-audit", async () => {
+      const pages = await getExtractedPages(jobId);
+      return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
+    }) : null;
     const artifacts = await step.run("finalize", async () => {
       const pages = await getExtractedPages(jobId);
       const store = getBlobStore();
-      const files = buildArtifacts(job, pages, branding);
+      const files = buildArtifacts(job, pages, branding, imageAudit);
       const stored = [];
       for (const f of files) stored.push(await store.put(`extract/${jobId}/${f.name}`, f.body, f.type));
       await completeExtractJob(jobId, stored);
@@ -7920,7 +7933,7 @@ var CATALOG = {
           kind: "simple",
           endpoint: "/extract-site",
           mcpName: "extract_site",
-          tagline: "Map + bulk scrape + SEO report",
+          tagline: "Map + bulk scrape (content)",
           params: [
             URL_PARAM,
             { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
@@ -7928,7 +7941,21 @@ var CATALOG = {
             { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true },
             { key: "background", label: "Run in background", type: "boolean", default: true, advanced: true }
           ],
-          examples: ["https://steenshoney.com/ (full-site SEO crawl)"]
+          examples: ["https://steenshoney.com/ (full-site content crawl)"]
+        },
+        {
+          key: "audit_site",
+          label: "Audit",
+          kind: "simple",
+          endpoint: "/extract-site",
+          mcpName: "audit_site",
+          tagline: "Technical SEO audit (Screaming-Frog style)",
+          params: [
+            URL_PARAM,
+            { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
+            { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true }
+          ],
+          examples: ["https://steenshoney.com/ (issues + link graph + image audit)"]
         }
       ]
     },
@@ -17475,4 +17502,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-UOQ2JIUQ.js.map
+//# sourceMappingURL=server-LPVBSE2E.js.map