mcp-scraper 0.88.2 → 0.89.0

package/dist/chunk-GGZEC22A.js

@@ -1,215 +0,0 @@
-// src/api/url-utils.ts
-import { isIP } from "net";
-import { lookup } from "dns/promises";
-import ipaddr from "ipaddr.js";
-function unbracketIpLiteral(value) {
-  return value.startsWith("[") && value.endsWith("]") ? value.slice(1, -1) : value;
-}
-function isPrivateIpAddress(address) {
-  const normalized = unbracketIpLiteral(address);
-  if (!isIP(normalized)) return false;
-  try {
-    const parsed = ipaddr.parse(normalized);
-    const isExtraNonGlobalIpv4 = (ipv4) => {
-      const extraNonGlobal = [
-        ["192.0.0.0", 24],
-        ["192.0.2.0", 24],
-        ["198.18.0.0", 15],
-        ["198.51.100.0", 24],
-        ["203.0.113.0", 24]
-      ];
-      return extraNonGlobal.some(([network, bits]) => ipv4.match(ipaddr.parse(network), bits));
-    };
-    if (parsed instanceof ipaddr.IPv6 && parsed.isIPv4MappedAddress()) {
-      const mapped = parsed.toIPv4Address();
-      return mapped.range() !== "unicast" || isExtraNonGlobalIpv4(mapped);
-    }
-    if (parsed instanceof ipaddr.IPv4) {
-      if (isExtraNonGlobalIpv4(parsed)) return true;
-    }
-    return parsed.range() !== "unicast";
-  } catch {
-    return true;
-  }
-}
-async function resolvesToPrivateAddress(hostname) {
-  const host = unbracketIpLiteral(hostname.toLowerCase());
-  if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) return true;
-  if (isPrivateIpAddress(host)) return true;
-  try {
-    const addresses = await lookup(host, { all: true, verbatim: true });
-    return addresses.length === 0 || addresses.some((entry) => isPrivateIpAddress(entry.address));
-  } catch {
-    return true;
-  }
-}
-async function validatePublicHttpUrl(raw, opts) {
-  let parsed;
-  try {
-    parsed = new URL(raw.trim());
-  } catch {
-    return { error: `Invalid ${opts.field}` };
-  }
-  const allowedProtocols = opts.requireHttps ? ["https:"] : ["http:", "https:"];
-  if (!allowedProtocols.includes(parsed.protocol)) {
-    return { error: opts.requireHttps ? `${opts.field} must use https` : `${opts.field} must use http or https` };
-  }
-  if (await resolvesToPrivateAddress(parsed.hostname)) {
-    return { error: `${opts.field} must resolve to a public internet host` };
-  }
-  return { parsed };
-}
-
-// src/lib/html-document.ts
-import { load } from "cheerio";
-var REMOVED_CONTENT = "script,style,noscript,svg,iframe,canvas,nav,header,footer,aside";
-var MAX_CONTENT_URL_ATTRIBUTE_CHARS = 4096;
-function normalizedText(value) {
-  return (value ?? "").replace(/\s+/g, " ").trim();
-}
-function parseJsonLd(raw) {
-  const clean = raw.replace(/[\x00-\x09\x0b\x0c\x0e-\x1f]/g, " ").trim();
-  if (!clean) return null;
-  try {
-    return JSON.parse(clean);
-  } catch (error) {
-    const position = /at position (\d+)/.exec(error instanceof Error ? error.message : "")?.[1];
-    if (!position) return null;
-    try {
-      return JSON.parse(clean.slice(0, Number(position)));
-    } catch {
-      return null;
-    }
-  }
-}
-function collectSchemaTypes(value, output, remaining = { value: 2e3 }) {
-  if (remaining.value-- <= 0 || value == null) return;
-  if (Array.isArray(value)) {
-    for (const item of value) collectSchemaTypes(item, output, remaining);
-    return;
-  }
-  if (typeof value !== "object") return;
-  const record = value;
-  const type = record["@type"];
-  if (typeof type === "string" && type) output.add(type);
-  if (Array.isArray(type)) {
-    for (const item of type) if (typeof item === "string" && item) output.add(item);
-  }
-  if (Array.isArray(record["@graph"])) collectSchemaTypes(record["@graph"], output, remaining);
-}
-function contentRoot($) {
-  const main = $("main").first();
-  if (main.length) return main;
-  const article = $("article").first();
-  if (article.length) return article;
-  const body = $("body").first();
-  return body.length ? body : $.root();
-}
-function numericAttribute(element, name) {
-  const value = Number(element.attr(name) ?? 0);
-  return Number.isFinite(value) && value > 0 ? value : null;
-}
-function loadHtmlDocument(sourceHtml) {
-  const $ = load(sourceHtml);
-  const meta = {};
-  $("meta").each((_index, element) => {
-    const node = $(element);
-    const key = normalizedText(node.attr("name") ?? node.attr("property")).toLowerCase();
-    const content = normalizedText(node.attr("content"));
-    if (key && content) meta[key] = content;
-  });
-  const headings = [];
-  $("h1,h2,h3,h4,h5,h6").each((_index, element) => {
-    const node = $(element);
-    const text = normalizedText(node.text());
-    if (text) headings.push({ level: Number(element.tagName.slice(1)), text });
-  });
-  const schema = [];
-  const schemaTypeSet = /* @__PURE__ */ new Set();
-  $('script[type="application/ld+json"]').each((_index, element) => {
-    const parsed = parseJsonLd($(element).text());
-    if (parsed == null) return;
-    if (Array.isArray(parsed)) schema.push(...parsed);
-    else schema.push(parsed);
-    collectSchemaTypes(parsed, schemaTypeSet);
-  });
-  const microdataTypes = /* @__PURE__ */ new Set();
-  $("[itemtype]").each((_index, element) => {
-    const itemtype = $(element).attr("itemtype") ?? "";
-    for (const value of itemtype.split(/\s+/)) {
-      try {
-        const url = new URL(value);
-        if (url.hostname === "schema.org" || url.hostname === "www.schema.org") {
-          const type = url.pathname.replace(/^\/+/, "");
-          if (type) microdataTypes.add(type);
-        }
-      } catch {
-      }
-    }
-  });
-  const root = contentRoot($).clone();
-  root.find(REMOVED_CONTENT).remove();
-  root.find("[href],[src],[srcset],[data-src],[data-srcset]").each((_index, element) => {
-    const node = $(element);
-    for (const attribute of ["href", "src", "srcset", "data-src", "data-srcset"]) {
-      const value = node.attr(attribute);
-      if (value && value.length > MAX_CONTENT_URL_ATTRIBUTE_CHARS) node.removeAttr(attribute);
-    }
-  });
-  root.find("img").each((_index, element) => {
-    const image = $(element);
-    const src = image.attr("src");
-    if (!src || src.startsWith("data:")) {
-      const lazy = image.attr("data-src") ?? image.attr("data-lazy-src") ?? image.attr("data-original");
-      if (lazy) image.attr("src", lazy);
-    }
-  });
-  const mainHtml = root.html() ?? "";
-  const mainText = normalizedText(root.text());
-  const links = [];
-  $("a[href]").each((_index, element) => {
-    const node = $(element);
-    const href = normalizedText(node.attr("href"));
-    if (!href) return;
-    links.push({
-      href,
-      text: normalizedText(node.text()),
-      rel: normalizedText(node.attr("rel")).toLowerCase() || null
-    });
-  });
-  const images = [];
-  $("img").each((_index, element) => {
-    const node = $(element);
-    const attributes = Object.fromEntries(Object.entries(element.attribs ?? {}).map(([key, value]) => [key.toLowerCase(), value]));
-    images.push({
-      src: node.attr("src") ?? node.attr("data-src") ?? node.attr("data-lazy-src") ?? node.attr("data-original") ?? null,
-      srcset: node.attr("srcset") ?? node.attr("data-srcset") ?? null,
-      alt: node.attr("alt") ?? null,
-      width: numericAttribute(node, "width"),
-      height: numericAttribute(node, "height"),
-      attributes
-    });
-  });
-  return {
-    $,
-    sourceHtml,
-    title: normalizedText($("title").first().text()) || null,
-    meta,
-    headings,
-    schema,
-    schemaTypes: [...schemaTypeSet],
-    microdataTypes: [...microdataTypes],
-    canonicalUrl: normalizedText($('link[rel~="canonical"]').first().attr("href")) || null,
-    mainHtml,
-    mainText,
-    bodyClass: $("body").first().attr("class") ?? "",
-    links,
-    images
-  };
-}
-
-export {
-  isPrivateIpAddress,
-  validatePublicHttpUrl,
-  loadHtmlDocument
-};

package/dist/chunk-GXBZXWXB.js

@@ -1,184 +0,0 @@
-// src/schemas.ts
-import { z } from "zod";
-var DEFAULT_PROXY_MODE = "none";
-var DEFAULT_MAPS_PROXY_MODE = "none";
-var HarvestOptionsSchema = z.object({
-  query: z.string().min(1),
-  location: z.string().optional(),
-  gl: z.string().length(2).default("us"),
-  hl: z.string().length(2).default("en"),
-  device: z.enum(["desktop", "mobile"]).default("desktop"),
-  proxyMode: z.enum(["location", "configured", "none"]).default(DEFAULT_PROXY_MODE),
-  proxyZip: z.string().regex(/^\d{5}$/).optional(),
-  keepDefaultProxy: z.boolean().optional(),
-  requireUsEgress: z.boolean().optional(),
-  maxAttempts: z.number().int().min(1).max(12).optional(),
-  debug: z.boolean().default(false),
-  depth: z.number().int().min(1).max(30).default(3),
-  maxQuestions: z.number().int().min(1).max(1e3).default(100),
-  headless: z.boolean().default(false),
-  profileDir: z.string().optional(),
-  proxy: z.string().url().optional(),
-  kernelApiKey: z.string().optional(),
-  kernelProxyId: z.string().optional(),
-  kernelProfileName: z.string().optional(),
-  kernelProfileSaveChanges: z.boolean().optional(),
-  kernelStealth: z.boolean().optional(),
-  serpIdentity: z.string().regex(/^[a-z0-9][a-z0-9_-]{0,63}$/).optional(),
-  kernelProxyResolution: z.unknown().optional(),
-  outputDir: z.string().default("./paa-output"),
-  format: z.enum(["json", "csv", "both"]).default("both"),
-  serpOnly: z.boolean().default(false),
-  questionsOnly: z.boolean().default(false),
-  questionGrowthRecoveryRounds: z.number().int().min(0).max(4).default(2),
-  pages: z.number().int().min(1).max(2).default(1),
-  includeAllSerpFeatures: z.boolean().default(false),
-  includeLocalPack: z.boolean().default(false),
-  includeForums: z.boolean().default(false),
-  includeVideos: z.boolean().default(false),
-  includeAiOverview: z.boolean().default(false),
-  includeWhatPeopleSaying: z.boolean().default(false),
-  includeShortVideos: z.boolean().default(false),
-  recency: z.enum(["day", "week", "month", "year"]).optional(),
-  softDeadlineMs: z.number().optional()
-});
-function recencyToTbs(recency) {
-  switch (recency) {
-    case "day":
-      return "qdr:d";
-    case "week":
-      return "qdr:w";
-    case "month":
-      return "qdr:m";
-    case "year":
-      return "qdr:y";
-    default:
-      return void 0;
-  }
-}
-var MapsPlaceOptionsSchema = z.object({
-  businessName: z.string().min(1),
-  location: z.string().min(1),
-  gl: z.string().length(2).default("us"),
-  hl: z.string().length(2).default("en"),
-  includeReviews: z.boolean().default(false),
-  maxReviews: z.number().int().min(1).max(500).default(50),
-  includeServices: z.boolean().default(false),
-  includeImages: z.boolean().default(false),
-  imageScope: z.enum(["owner", "all"]).default("all"),
-  maxImages: z.number().int().min(1).max(250).default(100),
-  maxInlineImages: z.number().int().min(0).max(5).default(3),
-  kernelApiKey: z.string().optional(),
-  kernelProxyId: z.string().optional(),
-  headless: z.boolean().default(true)
-});
-var MapsSearchOptionsSchema = z.object({
-  query: z.string().min(1),
-  location: z.string().optional(),
-  gl: z.string().length(2).default("us"),
-  hl: z.string().length(2).default("en"),
-  maxResults: z.number().int().min(1).max(50).default(10),
-  includeServices: z.boolean().default(false),
-  proxyMode: z.enum(["location", "configured", "none"]).default(DEFAULT_MAPS_PROXY_MODE),
-  proxyZip: z.string().regex(/^\d{5}$/).optional(),
-  serpRedirect: z.boolean().optional(),
-  forceDirectEgress: z.boolean().optional(),
-  debug: z.boolean().default(false),
-  kernelApiKey: z.string().optional(),
-  kernelProxyId: z.string().optional(),
-  kernelProxyResolution: z.unknown().optional(),
-  headless: z.boolean().default(true)
-});
-var RawPAAItemSchema = z.object({
-  questionId: z.string().min(1),
-  googleLinkId: z.string().nullable(),
-  googleEvidenceId: z.string().nullable(),
-  question: z.string().min(1),
-  answer: z.string().optional(),
-  sourceTitle: z.string().optional(),
-  sourceSite: z.string().optional(),
-  sourceCite: z.string().optional(),
-  sources: z.array(z.object({
-    title: z.string().nullable(),
-    site: z.string().nullable(),
-    url: z.string().min(1),
-    rawUrl: z.string().min(1),
-    resolvedUrl: z.string().nullable(),
-    linkType: z.enum(["plain", "google_url_redirect", "google_goto_redirect"]),
-    resolutionStatus: z.enum(["not_needed", "resolved", "unresolved", "rejected"])
-  })).default([])
-});
-var RawMapsOverviewSchema = z.object({
-  name: z.string().nullable(),
-  rating: z.string().nullable(),
-  reviewCount: z.string().nullable(),
-  category: z.string().nullable(),
-  address: z.string().nullable(),
-  hoursSummary: z.string().nullable(),
-  phone: z.string().nullable(),
-  phoneDisplay: z.string().nullable(),
-  website: z.string().nullable(),
-  plusCode: z.string().nullable(),
-  bookingUrl: z.string().nullable()
-});
-var RawMapsHoursRowSchema = z.object({
-  day: z.string(),
-  hours: z.string()
-});
-var RawMapsReviewStatsSchema = z.object({
-  reviewHistogram: z.array(z.object({
-    stars: z.number(),
-    count: z.string()
-  })),
-  reviewTopics: z.array(z.object({
-    label: z.string(),
-    count: z.string()
-  }))
-});
-var RawMapsReviewCardSchema = z.object({
-  reviewId: z.string(),
-  author: z.string().nullable(),
-  stars: z.string().nullable(),
-  date: z.string().nullable(),
-  text: z.string().nullable(),
-  ownerResponse: z.string().nullable()
-});
-var RawMapsAboutAttributeSchema = z.object({
-  section: z.string(),
-  attribute: z.string()
-});
-
-// src/output/memory-library-sink.ts
-async function postToMemoryLibrary(args) {
-  const key = process.env.MCP_SCRAPER_API_KEY?.trim();
-  const baseUrl = (process.env.MCP_SCRAPER_BASE_URL ?? "https://mcpscraper.dev").replace(/\/$/, "");
-  if (!key) return;
-  try {
-    const res = await fetch(`${baseUrl}/memory/mcp-call`, {
-      method: "POST",
-      headers: { "content-type": "application/json", "x-api-key": key },
-      body: JSON.stringify({ toolName: "libraryIngestTool", args })
-    });
-    const body = await res.json().catch(() => null);
-    if (!res.ok || body?.ok === false) {
-      console.warn("[memory-library-sink] ingest not accepted:", res.status, body?.error ?? "");
-    }
-  } catch (err) {
-    console.warn("[memory-library-sink] ingest failed:", err?.message);
-  }
-}
-
-export {
-  DEFAULT_PROXY_MODE,
-  DEFAULT_MAPS_PROXY_MODE,
-  HarvestOptionsSchema,
-  recencyToTbs,
-  MapsPlaceOptionsSchema,
-  MapsSearchOptionsSchema,
-  RawPAAItemSchema,
-  RawMapsOverviewSchema,
-  RawMapsHoursRowSchema,
-  RawMapsReviewStatsSchema,
-  RawMapsAboutAttributeSchema,
-  postToMemoryLibrary
-};