mcp-scraper 0.1.1 → 0.1.2

package/dist/bin/api-server.cjs

@@ -4628,9 +4628,12 @@ function parsePageData(url, html, status, via) {
   const bodyText = html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
   const wordCount = bodyText.split(" ").filter((w) => w.length > 2).length;
   const schemaTypes = [];
+  const schema = [];
   for (const m of html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
     try {
       const parsed = JSON.parse(m[1]);
+      if (Array.isArray(parsed)) schema.push(...parsed);
+      else schema.push(parsed);
       const collect = (obj) => {
         if (!obj || typeof obj !== "object") return;
         const o = obj;
@@ -4642,6 +4645,13 @@ function parsePageData(url, html, status, via) {
     } catch {
     }
   }
+  const mainHtml = (html.match(/<main[^>]*>([\s\S]*?)<\/main>/i)?.[1] ?? html.match(/<article[^>]*>([\s\S]*?)<\/article>/i)?.[1] ?? html.match(/<body[^>]*>([\s\S]*?)<\/body>/i)?.[1] ?? html).replace(/<(script|style|nav|header|footer|aside|noscript|svg|iframe)[^>]*>[\s\S]*?<\/\1>/gi, "");
+  let bodyMarkdown = "";
+  try {
+    bodyMarkdown = turndown.turndown(mainHtml).replace(/\n{3,}/g, "\n\n").trim().slice(0, MAX_PAGE_MARKDOWN);
+  } catch {
+    bodyMarkdown = bodyText.slice(0, MAX_PAGE_MARKDOWN);
+  }
   let internalLinks = 0;
   let externalLinks = 0;
   for (const m of html.matchAll(/href\s*=\s*["']([^"'\s>]+)/gi)) {
@@ -4652,7 +4662,7 @@ function parsePageData(url, html, status, via) {
     } catch {
     }
   }
-  return { url, status, via, title, metaDescription, h1, headings, wordCount, schemaTypes, canonicalUrl, internalLinks, externalLinks };
+  return { url, status, via, title, metaDescription, h1, headings, wordCount, schemaTypes, canonicalUrl, internalLinks, externalLinks, bodyMarkdown, schema };
 }
 async function fetchAndParse(url, kernelApiKey) {
   try {
@@ -4667,13 +4677,13 @@ async function fetchAndParse(url, kernelApiKey) {
         const html = await res.text();
         return parsePageData(url, html, res.status, "fetch");
       }
-      return { url, status: res.status, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0 };
+      return { url, status: res.status, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
     }
     if ((res.status === 403 || res.status === 429) && kernelApiKey) {
       const html = await fetchWithKernel(url);
       return parsePageData(url, html, 200, "browser");
     }
-    return { url, status: res.status, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0 };
+    return { url, status: res.status, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
   } catch {
     if (kernelApiKey) {
       try {
@@ -4682,7 +4692,7 @@ async function fetchAndParse(url, kernelApiKey) {
       } catch {
       }
     }
-    return { url, status: null, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0 };
+    return { url, status: null, via: "fetch", title: null, metaDescription: null, h1: null, headings: [], wordCount: 0, schemaTypes: [], canonicalUrl: null, internalLinks: 0, externalLinks: 0, bodyMarkdown: "", schema: [] };
   }
 }
 async function runWithConcurrency(items, concurrency, fn) {
@@ -4723,14 +4733,17 @@ async function extractSite(opts) {
     browserRetries
   };
 }
-var UA2, EXTRACT_CONCURRENCY;
+var import_turndown2, UA2, EXTRACT_CONCURRENCY, MAX_PAGE_MARKDOWN, turndown;
 var init_site_extractor = __esm({
   "src/api/site-extractor.ts"() {
     "use strict";
+    import_turndown2 = __toESM(require("turndown"), 1);
     init_site_mapper();
     init_kernel_fetch();
     UA2 = "Mozilla/5.0 (compatible; ThorbitBot/1.0; +https://thorbit.ai)";
     EXTRACT_CONCURRENCY = 6;
+    MAX_PAGE_MARKDOWN = 4e4;
+    turndown = new import_turndown2.default({ headingStyle: "atx", bulletListMarker: "-", codeBlockStyle: "fenced" });
   }
 });
 
@@ -14937,6 +14950,9 @@ function truncate(s, max) {
   if (!s) return "";
   return s.length > max ? s.slice(0, max) + "\u2026" : s;
 }
+function cell(s) {
+  return String(s ?? "").replace(/\r?\n+/g, " ").replace(/\|/g, "\\|").replace(/\s+/g, " ").trim();
+}
 function debugSection(debug) {
   if (!debug || typeof debug !== "object") return "";
   const request = debug.request ?? {};
@@ -14990,14 +15006,14 @@ function formatHarvestPaa(raw, input) {
   const diagnostics = d.diagnostics;
   const durationMs = d.stats?.durationMs;
   const paaRows = flat.map(
-    (r, i) => `| ${i + 1} | ${r.question} | ${truncate(r.answer, 120)} | ${r.source_site ?? ""} |`
+    (r, i) => `| ${i + 1} | ${cell(r.question)} | ${cell(truncate(r.answer, 120))} | ${cell(r.source_title || r.source_site || "")} |`
   ).join("\n");
   const paaTable = flat.length ? `## People Also Ask (${flat.length} questions)
 | # | Question | Answer | Source |
 |---|----------|--------|--------|
 ${paaRows}` : "## People Also Ask\n*Google did not return a People Also Ask block for this query/location. SERP data was extracted successfully when available.*";
   const serpRows = organic.map(
-    (r) => `| ${r.position} | ${r.title} | [${r.domain}](${r.url}) | ${truncate(r.snippet, 100)} |`
+    (r) => `| ${r.position} | ${cell(r.title)} | [${cell(r.domain)}](${r.url}) | ${cell(truncate(r.snippet, 100))} |`
   ).join("\n");
   const serpTable = organic.length ? `
 ## Organic Results (${organic.length})
@@ -15045,14 +15061,14 @@ function formatSearchSerp(raw, input) {
   const aiOvw = d.aiOverview;
   const diagnostics = d.diagnostics;
   const serpRows = organic.map(
-    (r) => `| ${r.position} | ${r.title} | [${r.domain}](${r.url}) | ${truncate(r.snippet, 100)} |`
+    (r) => `| ${r.position} | ${cell(r.title)} | [${cell(r.domain)}](${r.url}) | ${cell(truncate(r.snippet, 100))} |`
   ).join("\n");
   const serpTable = organic.length ? `## Organic Results (${organic.length})
 | # | Title | URL | Snippet |
 |---|-------|-----|----------|
 ${serpRows}` : "## Organic Results\n*None found*";
   const localRows = localPack.map(
-    (b) => `| ${b.position} | ${b.name} | ${b.rating ?? "\u2014"} (${b.reviewCount ?? "0"}) | ${b.websiteUrl ? `[link](${b.websiteUrl})` : "\u2014"} |`
+    (b) => `| ${b.position} | ${cell(b.name)} | ${b.rating ?? "\u2014"} (${b.reviewCount ?? "0"}) | ${b.websiteUrl ? `[link](${b.websiteUrl})` : "\u2014"} |`
   ).join("\n");
   const localSection = localPack.length ? `
 ## Local Pack (${localPack.length})
@@ -15213,7 +15229,7 @@ function formatExtractSite(raw, input) {
   const pages = d.pages ?? [];
   const pageRows = pages.map((p, i) => {
     const schemaInfo = p.kpo?.type?.join(", ") ?? (Array.isArray(p.schema) && p.schema.length ? `${p.schema.length} block(s)` : "\u2014");
-    return `| ${i + 1} | ${p.title ?? "Untitled"} | ${p.url} | ${schemaInfo} |`;
+    return `| ${i + 1} | ${cell(p.title ?? "Untitled")} | ${p.url} | ${schemaInfo} |`;
   }).join("\n");
   const full = [
     `# Site Extract: ${input.url}`,
@@ -15245,7 +15261,7 @@ function formatYoutubeHarvest(raw, input) {
   const videos = d.videos ?? [];
   const label = input.mode === "channel" ? input.channelHandle ?? "channel" : `"${input.query ?? ""}"`;
   const videoRows = videos.map(
-    (v, i) => `| ${i + 1} | ${truncate(v.title, 70)} | ${v.channelName} | ${v.views ?? "\u2014"} | ${v.duration ?? "\u2014"} | \`${v.videoId}\` |`
+    (v, i) => `| ${i + 1} | ${cell(truncate(v.title, 70))} | ${cell(v.channelName)} | ${v.views ?? "\u2014"} | ${v.duration ?? "\u2014"} | \`${v.videoId}\` |`
   ).join("\n");
   const channelSection = d.channelMeta ? `
 ## Channel
@@ -15288,7 +15304,7 @@ function formatYoutubeTranscribe(raw, input) {
     const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
     const mm = String(Math.floor(sec / 60)).padStart(2, "0");
     const ss = String(sec % 60).padStart(2, "0");
-    return `| ${mm}:${ss} | ${truncate(c.text, 120)} |`;
+    return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
   }).join("\n");
   const full = [
     `# YouTube Transcript: \`${input.videoId}\``,
@@ -15361,7 +15377,7 @@ function formatFacebookAdSearch(raw, input) {
   const d = parsed.data;
   const advertisers = d.results ?? d.advertisers ?? [];
   const rows = advertisers.map(
-    (a, i) => `| ${i + 1} | ${a.name} | ${a.adCount ?? "\u2014"} | \`${a.libraryId ?? "\u2014"}\` |`
+    (a, i) => `| ${i + 1} | ${cell(a.name)} | ${a.adCount ?? "\u2014"} | \`${a.libraryId ?? "\u2014"}\` |`
   ).join("\n");
   const full = [
     `# Facebook Ad Library Search: "${input.query}"`,
@@ -15558,7 +15574,7 @@ function formatFacebookAdTranscribe(raw, input) {
     const sec = Number.isFinite(c.timestamp[0]) ? Math.floor(c.timestamp[0]) : 0;
     const mm = String(Math.floor(sec / 60)).padStart(2, "0");
     const ss = String(sec % 60).padStart(2, "0");
-    return `| ${mm}:${ss} | ${truncate(c.text, 120)} |`;
+    return `| ${mm}:${ss} | ${cell(truncate(c.text, 120))} |`;
   }).join("\n");
   const full = [
     `# Facebook Ad Transcript`,