mcp-scraper 0.7.0 → 0.8.0

package/dist/{server-JNY4XPZE.js → server-WBDVVHR6.js}

@@ -22,6 +22,7 @@ import {
 import {
   HttpMcpToolExecutor,
   MemoryMcpToolExecutor,
+  buildLinkGraph,
   buildPaaExtractorMcpServer,
   configureReportSaving,
   exportFanout,
@@ -33,11 +34,13 @@ import {
   registerSerpIntelligenceCaptureTools,
   sanitizeAttempts,
   sanitizeHarvestResult
-} from "./chunk-YGRZU7IR.js";
+} from "./chunk-ITJC3NN4.js";
 import {
   auditImages,
+  buildLinkReport,
+  computeIssues,
   getBlobStore
-} from "./chunk-GWRIO6JT.js";
+} from "./chunk-R7EETU7Z.js";
 import {
   browserServiceApiKey,
   browserServiceProfileName,
@@ -67,7 +70,7 @@ import {
   RawMapsOverviewSchema,
   RawMapsReviewStatsSchema
 } from "./chunk-XGIPATLV.js";
-import "./chunk-EGWJ74EX.js";
+import "./chunk-3MKSXDJ7.js";
 import {
   completeExtractJob,
   countSuccessfulPages,
@@ -4780,7 +4783,8 @@ async function spiderSite(opts) {
     totalFound: results.length,
     durationMs: Date.now() - startMs,
     truncated: queue.length > 0 || results.length >= maxUrls,
-    browserRetries
+    browserRetries,
+    sitemapUrls: sitemapUrls.slice(0, maxUrls)
   };
 }
 
@@ -4904,6 +4908,23 @@ async function crawlWithRotation(urls, opts) {
   return urls.map((u) => byUrl.get(u) ?? { url: u, html: null, status: 0, via: "browser" });
 }
 
+// src/api/seo-audit.ts
+function buildSeoAudit(pages, startUrl) {
+  const { edges, metrics } = buildLinkGraph(pages, startUrl);
+  const issues = computeIssues(pages, metrics);
+  const { summary } = buildLinkReport(edges, [...metrics.values()], startUrl);
+  return {
+    issues,
+    linkSummary: summary,
+    pageMetrics: [...metrics.values()].map((m) => ({
+      url: m.url,
+      inboundInternal: m.inlinks,
+      outboundInternal: m.outlinksInternal,
+      orphan: m.orphan
+    }))
+  };
+}
+
 // src/api/site-extractor.ts
 var PIXEL_WIDTHS = { i: 4, l: 4, j: 4, ".": 4, ",": 4, "'": 4, t: 6, f: 6, r: 6, " ": 4, m: 14, w: 13, W: 16, M: 16 };
 function estimatePixels(s) {
@@ -4947,6 +4968,7 @@ function emptyPageData(url, status, via) {
     imageLinks: [],
     indexable: status === 200,
     indexabilityReason: status === 200 ? null : "non-200",
+    inSitemap: null,
     schemaTypes: [],
     canonicalUrl: null,
     internalLinks: 0,
@@ -4963,7 +4985,8 @@ function makeSeedSpider(startUrl, urls) {
     totalFound: urls.length,
     durationMs: 0,
     truncated: false,
-    browserRetries: 0
+    browserRetries: 0,
+    sitemapUrls: []
   };
 }
 var UA2 = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
@@ -5134,6 +5157,7 @@ function parsePageData(url, html, status, via, resp) {
     imageLinks,
     indexable,
     indexabilityReason,
+    inSitemap: null,
     schemaTypes,
     canonicalUrl,
     internalLinks,
@@ -5147,11 +5171,11 @@ async function extractPagesRotating(urls, opts) {
   const fetched = await crawlWithRotation(urls, { apiKey: opts.kernelApiKey, concurrency: opts.concurrency, urlsPerBrowser: opts.urlsPerBrowser });
   return fetched.map((r) => r.html ? parsePageData(r.url, r.html, r.status || 200, "browser", { headers: r.headers, responseTimeMs: r.responseTimeMs, redirectUrl: r.redirectUrl }) : emptyPageData(r.url, r.status || null, "browser"));
 }
-async function discoverUrls(startUrl, maxPages, kernelApiKey) {
+async function discoverUrlsWithSitemap(startUrl, maxPages, kernelApiKey) {
   const sitemapUrls = await discoverSitemapUrls2(startUrl, maxPages);
-  if (sitemapUrls.length > 0) return sitemapUrls;
+  if (sitemapUrls.length > 0) return { urls: sitemapUrls, sitemapUrls };
   const spider = await spiderSite({ startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey });
-  return spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
+  return { urls: spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url), sitemapUrls: spider.sitemapUrls };
 }
 async function fetchAndParse(url, kernelApiKey) {
   const startedAt = Date.now();
@@ -5210,6 +5234,7 @@ async function extractSite(opts) {
   let browserRetries = 0;
   let truncated = false;
   let spider;
+  let sitemapUrls = [];
   if (rotating) {
     const browserConcurrency = Math.max(1, Math.min(opts.parallelism ?? 1, 8));
     const urlsPerBrowser = opts.rotateProxyEvery ?? 10;
@@ -5227,7 +5252,8 @@ async function extractSite(opts) {
       for (const u of opts.seedUrls) enqueue(u);
     } else {
       enqueue(opts.startUrl);
-      for (const u of await discoverSitemapUrls2(opts.startUrl, maxPages)) enqueue(u);
+      sitemapUrls = await discoverSitemapUrls2(opts.startUrl, maxPages);
+      for (const u of sitemapUrls) enqueue(u);
     }
     const waveCap = Math.max(browserConcurrency * urlsPerBrowser, 30);
     let done = 0;
@@ -5256,6 +5282,7 @@ async function extractSite(opts) {
     spider = makeSeedSpider(opts.startUrl, [...seen]);
   } else {
     spider = seeded ? makeSeedSpider(opts.startUrl, opts.seedUrls) : await spiderSite({ startUrl: opts.startUrl, maxUrls: maxPages, concurrency: 12, kernelApiKey: opts.kernelApiKey });
+    sitemapUrls = spider.sitemapUrls;
     const urlsToExtract = spider.urls.filter((u) => u.status === 200 || u.status === null).slice(0, maxPages).map((u) => u.url);
     browserRetries = spider.browserRetries;
     await runWithConcurrency(urlsToExtract, concurrency, async (url) => {
@@ -5265,10 +5292,17 @@ async function extractSite(opts) {
     });
     truncated = spider.truncated || pages.length >= maxPages;
   }
+  const sitemapSet = sitemapUrls.length > 0 ? new Set(sitemapUrls.map((u) => normalizeUrl(u, opts.startUrl) ?? u)) : null;
+  for (const p of pages) p.inSitemap = sitemapSet ? sitemapSet.has(normalizeUrl(p.url, opts.startUrl) ?? p.url) : null;
   let branding = null;
   if (opts.formats?.includes("branding") && opts.kernelApiKey) {
     branding = await extractBranding(opts.startUrl, opts.kernelApiKey).catch(() => null);
   }
+  const seoAudit = opts.formats?.includes("issues") ? buildSeoAudit(pages, opts.startUrl) : null;
+  let imageAudit = null;
+  if (opts.formats?.includes("images")) {
+    imageAudit = await auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
+  }
   return {
     startUrl: opts.startUrl,
     pages,
@@ -5277,7 +5311,9 @@ async function extractSite(opts) {
     durationMs: Date.now() - startMs,
     truncated,
     browserRetries,
-    branding
+    branding,
+    seoAudit,
+    imageAudit
   };
 }
 
@@ -7519,12 +7555,13 @@ var siteExtractFn = inngest.createFunction(
     const maxPages = Number(job.options.maxPages ?? 1e4);
     const concurrency = Number(job.options.concurrency ?? 1);
     const urlsPerBrowser = Number(job.options.urlsPerBrowser ?? job.options.rotateProxyEvery ?? 10);
-    const seed = await step.run("discover", async () => {
-      const discovered = await discoverUrls(job.startUrl, maxPages, key);
-      const out = discovered.length ? discovered : [job.startUrl];
-      await setExtractJobTotal(jobId, out.length);
-      return out;
+    const seedRaw = await step.run("discover", async () => {
+      const discovered = await discoverUrlsWithSitemap(job.startUrl, maxPages, key);
+      const urls = discovered.urls.length ? discovered.urls : [job.startUrl];
+      await setExtractJobTotal(jobId, urls.length);
+      return { urls, sitemapUrls: discovered.sitemapUrls };
     });
+    const seed = Array.isArray(seedRaw) ? { urls: seedRaw, sitemapUrls: [] } : seedRaw;
     const siteHost = (() => {
       try {
         return new URL(job.startUrl).hostname.replace(/^www\./, "");
@@ -7548,8 +7585,9 @@ var siteExtractFn = inngest.createFunction(
         return false;
       }
     };
-    const queue = [...seed];
-    const seen = new Set(seed.map(norm));
+    const sitemapSet = seed.sitemapUrls.length > 0 ? new Set(seed.sitemapUrls.map(norm)) : null;
+    const queue = [...seed.urls];
+    const seen = new Set(seed.urls.map(norm));
     let crawled = 0;
     const MAX_BATCHES = Math.ceil(maxPages / BATCH) + 30;
     for (let i = 0; i < MAX_BATCHES && crawled < maxPages && crawled < queue.length; i++) {
@@ -7557,7 +7595,7 @@ var siteExtractFn = inngest.createFunction(
       if (!batch.length) break;
       const newLinks = await step.run(`crawl-batch-${i}`, async () => {
         const pages = await extractPagesRotating(batch, { kernelApiKey: key, concurrency, urlsPerBrowser });
-        await saveExtractPages(jobId, pages);
+        await saveExtractPages(jobId, pages.map((p) => ({ ...p, inSitemap: sitemapSet ? sitemapSet.has(norm(p.url)) : null })));
         const found = [];
         for (const p of pages) for (const l of p.outlinks ?? []) {
           if (l.internal && l.href && sameSite(l.href)) found.push(l.href);
@@ -7581,9 +7619,13 @@ var siteExtractFn = inngest.createFunction(
       const pages = await getExtractedPages(jobId);
       return auditImages(pages, { concurrency: 12, timeoutMs: 12e3 });
     }) : null;
+    const seoAudit = Array.isArray(job.options.formats) && job.options.formats.includes("issues") ? await step.run("seo-audit", async () => {
+      const pages = await getExtractedPages(jobId);
+      return buildSeoAudit(pages, job.startUrl);
+    }) : null;
     const artifacts = await step.run("finalize", async () => {
-      const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
-      const stored = await assembleExtractArtifacts(job, { branding, imageAudit });
+      const { assembleExtractArtifacts } = await import("./extract-bundle-U4D5LW5W.js");
+      const stored = await assembleExtractArtifacts(job, { branding, imageAudit, seoAudit });
       await completeExtractJob(jobId, stored);
       return stored;
     });
@@ -9089,7 +9131,7 @@ var ExtractSiteBodySchema = z11.object({
   rotateProxies: z11.boolean().optional(),
   rotateProxyEvery: z11.number().int().min(1).max(100).optional(),
   background: z11.boolean().optional(),
-  formats: z11.array(z11.enum(["markdown", "links", "json", "images", "branding"])).optional()
+  formats: z11.array(z11.enum(["markdown", "links", "json", "images", "branding", "issues"])).optional()
 });
 var YoutubeHarvestBodySchema = z11.object({
   mode: z11.enum(["search", "channel"]),
@@ -20458,6 +20500,7 @@ app.use("*", async (c, next) => {
 });
 function opForCostPath(p) {
   if (p === "/harvest" || p === "/harvest/sync") return "harvest";
+  if (p === "/diff-page") return "diff_page";
   if (p === "/extract-url") return "page_scrape";
   if (p === "/extract-site") return "extract_site";
   if (p === "/map-urls") return "url_map";
@@ -20630,6 +20673,13 @@ app.get("/catalog", (c) => {
   c.header("Cache-Control", "public, max-age=60");
   return c.json(CATALOG);
 });
+app.get("/rates", (c) => {
+  const costs = CREDIT_COST_CATALOG.map(({ aliases, ...cost }) => ({
+    ...cost,
+    ...cost.notes ? { notes: cost.notes.replace(/ via fal\.ai/g, "") } : {}
+  }));
+  return c.json({ creditsPerDollarStarter: Math.round(1 / 15e-5), costs });
+});
 app.get("/me", async (c) => {
   const token = getCookie2(c, "session");
   if (!token) return c.json({ authenticated: false });
@@ -21905,7 +21955,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
   }
   const jobId = c.req.param("id");
   const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-NVSZH35Y.js");
-  const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
+  const { assembleExtractArtifacts } = await import("./extract-bundle-U4D5LW5W.js");
   const job = await getExtractJob2(jobId);
   if (!job) return c.json({ error: "job not found" }, 404);
   const stored = await assembleExtractArtifacts(job, {});
@@ -22076,4 +22126,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-JNY4XPZE.js.map
+//# sourceMappingURL=server-WBDVVHR6.js.map