mcp-scraper 0.3.15 → 0.3.17

package/dist/chunk-PVWWY5NV.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.3.17";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-PVWWY5NV.js.map

package/dist/chunk-PVWWY5NV.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.3.17'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{chunk-WJ2LWHPA.js → chunk-UFD7N36R.js}

@@ -7,7 +7,7 @@ import {
 } from "./chunk-WN7PBKMV.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-2MX5WOII.js";
+} from "./chunk-PVWWY5NV.js";
 
 // src/mcp/browser-agent-mcp-server.ts
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
@@ -1329,4 +1329,4 @@ export {
   buildBrowserAgentMcpServer,
   registerBrowserAgentMcpTools
 };
-//# sourceMappingURL=chunk-WJ2LWHPA.js.map
+//# sourceMappingURL=chunk-UFD7N36R.js.map

package/dist/{server-UOQ2JIUQ.js → server-TKWY47JA.js}

@@ -92,7 +92,9 @@ import {
   CaptureSerpSnapshotInputSchema,
   CaptureSerpSnapshotOutputSchema,
   HttpMcpToolExecutor,
+  auditImages,
   buildLinkGraph,
+  buildLinkReport,
   buildPaaExtractorMcpServer,
   computeIssues,
   configureReportSaving,
@@ -101,8 +103,10 @@ import {
   harvestTimeoutBudget,
   liveWebToolAnnotations,
   outputBaseDir,
-  renderIssueReport
-} from "./chunk-WQAIOY2D.js";
+  renderImageSection,
+  renderIssueReport,
+  renderLinkReport
+} from "./chunk-N33YEY6W.js";
 import {
   CaptchaError,
   RECAPTCHA_INSTRUCTIONS,
@@ -138,7 +142,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-LFATOGDF.js";
-import "./chunk-2MX5WOII.js";
+import "./chunk-PVWWY5NV.js";
 
 // src/api/outbound-sanitize.ts
 var KEY_RENAMES = {
@@ -7587,10 +7591,14 @@ function chunk(items, size) {
   for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
   return out;
 }
-function buildArtifacts(job, pages, branding) {
+function buildArtifacts(job, pages, branding, imageAudit) {
   const { edges, metrics } = buildLinkGraph(pages, job.startUrl);
   const issues = computeIssues(pages, metrics);
-  const reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
+  const linkReport = buildLinkReport(edges, [...metrics.values()], job.startUrl);
+  let reportMd = renderIssueReport(job.startUrl, pages, issues, metrics);
+  if (imageAudit) reportMd += `
+
+${renderImageSection(imageAudit)}`;
   const pageRows = pages.map((p) => {
     const { bodyMarkdown: _b, schema: _s, outlinks: _o, ...rest } = p;
     const m = metrics.get(p.url);
@@ -7602,8 +7610,15 @@ function buildArtifacts(job, pages, branding) {
     { name: "issues.json", body: JSON.stringify(issues, null, 2), type: "application/json" },
     { name: "pages.jsonl", body: jsonl(pageRows), type: "application/x-ndjson" },
     { name: "links.jsonl", body: jsonl(edges), type: "application/x-ndjson" },
-    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" }
+    { name: "link-metrics.jsonl", body: jsonl([...metrics.values()]), type: "application/x-ndjson" },
+    { name: "link-report.md", body: renderLinkReport(linkReport), type: "text/markdown" },
+    { name: "links-summary.json", body: JSON.stringify(linkReport.summary, null, 2), type: "application/json" },
+    { name: "external-domains.json", body: JSON.stringify(linkReport.externalDomains, null, 2), type: "application/json" }
   ];
+  if (imageAudit) {
+    files.push({ name: "images.jsonl", body: jsonl(imageAudit.rows), type: "application/x-ndjson" });
+    files.push({ name: "images-summary.json", body: JSON.stringify(imageAudit.summary, null, 2), type: "application/json" });
+  }
   if (branding) files.push({ name: "branding.json", body: JSON.stringify(branding, null, 2), type: "application/json" });
   return files;
 }
@@ -7648,10 +7663,14 @@ var siteExtractFn = inngest.createFunction(
       });
     }
     const branding = Array.isArray(job.options.formats) && job.options.formats.includes("branding") ? await step.run("branding", () => extractBranding(job.startUrl, key).catch(() => null)) : null;
+    const imageAudit = Array.isArray(job.options.formats) && job.options.formats.includes("images") ? await step.run("image-audit", async () => {
+      const pages = await getExtractedPages(jobId);
+      return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
+    }) : null;
     const artifacts = await step.run("finalize", async () => {
       const pages = await getExtractedPages(jobId);
       const store = getBlobStore();
-      const files = buildArtifacts(job, pages, branding);
+      const files = buildArtifacts(job, pages, branding, imageAudit);
       const stored = [];
       for (const f of files) stored.push(await store.put(`extract/${jobId}/${f.name}`, f.body, f.type));
       await completeExtractJob(jobId, stored);
@@ -7920,7 +7939,7 @@ var CATALOG = {
           kind: "simple",
           endpoint: "/extract-site",
           mcpName: "extract_site",
-          tagline: "Map + bulk scrape + SEO report",
+          tagline: "Map + bulk scrape (content)",
           params: [
             URL_PARAM,
             { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
@@ -7928,7 +7947,21 @@ var CATALOG = {
             { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true },
             { key: "background", label: "Run in background", type: "boolean", default: true, advanced: true }
           ],
-          examples: ["https://steenshoney.com/ (full-site SEO crawl)"]
+          examples: ["https://steenshoney.com/ (full-site content crawl)"]
+        },
+        {
+          key: "audit_site",
+          label: "Audit",
+          kind: "simple",
+          endpoint: "/extract-site",
+          mcpName: "audit_site",
+          tagline: "Technical SEO audit (Screaming-Frog style)",
+          params: [
+            URL_PARAM,
+            { key: "maxPages", label: "Max pages", type: "number", min: 1, max: 1e4, default: 100 },
+            { key: "rotateProxies", label: "Rotate proxies (blocked sites)", type: "boolean", default: true }
+          ],
+          examples: ["https://steenshoney.com/ (issues + link graph + image audit)"]
         }
       ]
     },
@@ -17475,4 +17508,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-UOQ2JIUQ.js.map
+//# sourceMappingURL=server-TKWY47JA.js.map