omnifetch-lib 1.0.0

package/dist/index.js

@@ -0,0 +1,728 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  DEFAULT_CONFIG: () => DEFAULT_CONFIG,
+  default: () => index_default,
+  fetchJson: () => fetchJson,
+  fetchText: () => fetchText,
+  omniFetch: () => omniFetch
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/utils/detector.ts
+var cheerio = __toESM(require("cheerio"));
+var SKELETON_INDICATORS = [
+  "loading",
+  "skeleton",
+  "placeholder",
+  "spinner",
+  "shimmer",
+  "lazy-load",
+  "js-loading"
+];
+var PAYWALL_INDICATORS = [
+  "paywall",
+  "subscribe",
+  "subscription",
+  "sign-in",
+  "sign in",
+  "log-in",
+  "log in",
+  "login",
+  "create account",
+  "register to read",
+  "premium content",
+  "members only",
+  "subscriber only",
+  "paid content",
+  "unlock this",
+  "continue reading",
+  "read more for",
+  "free trial"
+];
+var MIN_CONTENT_LENGTH = 500;
+function isSkeletonPage(html) {
+  const $ = cheerio.load(html);
+  const body = $("body");
+  const bodyHtml = body.html() || "";
+  const lowerHtml = bodyHtml.toLowerCase();
+  for (const indicator of SKELETON_INDICATORS) {
+    if (lowerHtml.includes(`class="${indicator}`) || lowerHtml.includes(`class='${indicator}`) || lowerHtml.includes(`${indicator}-`)) {
+      return true;
+    }
+  }
+  const textContent = body.text().replace(/\s+/g, " ").trim();
+  if (textContent.length < MIN_CONTENT_LENGTH) {
+    const emptyDivs = $("div:empty").length;
+    const totalDivs = $("div").length;
+    if (totalDivs > 0 && emptyDivs / totalDivs > 0.5) {
+      return true;
+    }
+  }
+  const noscript = $("noscript").text().trim();
+  if (noscript.length > 100 && textContent.length < MIN_CONTENT_LENGTH) {
+    return true;
+  }
+  return false;
+}
+function isPaywalled(html) {
+  const $ = cheerio.load(html);
+  const body = $("body");
+  const lowerHtml = (body.html() || "").toLowerCase();
+  let paywallScore = 0;
+  for (const indicator of PAYWALL_INDICATORS) {
+    if (lowerHtml.includes(indicator)) {
+      paywallScore++;
+    }
+  }
+  if (paywallScore >= 2) {
+    return true;
+  }
+  const modalSelectors = [
+    '[class*="paywall"]',
+    '[class*="subscribe-wall"]',
+    '[class*="registration-wall"]',
+    '[id*="paywall"]',
+    "[data-paywall]"
+  ];
+  for (const selector of modalSelectors) {
+    if ($(selector).length > 0) {
+      return true;
+    }
+  }
+  const article = $('article, [class*="article"], [class*="content"]').first();
+  if (article.length > 0) {
+    const articleText = article.text().trim();
+    if (articleText.length < 1e3 && paywallScore >= 1) {
+      return true;
+    }
+  }
+  return false;
+}
+function extractTitle(html) {
+  const $ = cheerio.load(html);
+  const sources = [
+    () => $('meta[property="og:title"]').attr("content"),
+    () => $('meta[name="twitter:title"]').attr("content"),
+    () => $("title").text(),
+    () => $("h1").first().text()
+  ];
+  for (const getTitle of sources) {
+    const title = getTitle()?.trim();
+    if (title && title.length > 0) {
+      return title.replace(/\s+/g, " ").replace(/\|.*$/, "").replace(/-\s*[^-]+$/, "").trim();
+    }
+  }
+  return "";
+}
+function hasValidContent(html) {
+  const $ = cheerio.load(html);
+  const contentSelectors = [
+    "article",
+    '[class*="article"]',
+    '[class*="content"]',
+    '[class*="post"]',
+    "main",
+    '[role="main"]'
+  ];
+  for (const selector of contentSelectors) {
+    const element = $(selector).first();
+    if (element.length > 0) {
+      const text = element.text().replace(/\s+/g, " ").trim();
+      if (text.length >= MIN_CONTENT_LENGTH) {
+        return true;
+      }
+    }
+  }
+  const bodyText = $("body").text().replace(/\s+/g, " ").trim();
+  return bodyText.length >= MIN_CONTENT_LENGTH;
+}
+
+// src/tiers/light.ts
+var BROWSER_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.9",
+  "Accept-Encoding": "gzip, deflate, br",
+  "Cache-Control": "no-cache",
+  "Pragma": "no-cache",
+  "Sec-Ch-Ua": '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+  "Sec-Ch-Ua-Mobile": "?0",
+  "Sec-Ch-Ua-Platform": '"Windows"',
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "none",
+  "Sec-Fetch-User": "?1",
+  "Upgrade-Insecure-Requests": "1"
+};
+async function lightFetch(url, timeout = 3e4, customHeaders = {}) {
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        ...BROWSER_HEADERS,
+        ...customHeaders
+      },
+      redirect: "follow",
+      signal: controller.signal
+    });
+    clearTimeout(timeoutId);
+    if (!response.ok) {
+      if (response.status === 403 || response.status === 401) {
+        return {
+          success: false,
+          shouldFallback: true,
+          fallbackReason: "paywall",
+          error: `HTTP ${response.status}: Access denied`
+        };
+      }
+      return {
+        success: false,
+        shouldFallback: true,
+        fallbackReason: "error",
+        error: `HTTP ${response.status}: ${response.statusText}`
+      };
+    }
+    const html = await response.text();
+    const finalUrl = response.url;
+    if (!html || html.trim().length === 0) {
+      return {
+        success: false,
+        shouldFallback: true,
+        fallbackReason: "empty",
+        error: "Empty response received"
+      };
+    }
+    if (isSkeletonPage(html)) {
+      return {
+        success: false,
+        html,
+        title: extractTitle(html),
+        shouldFallback: true,
+        fallbackReason: "skeleton",
+        finalUrl
+      };
+    }
+    if (!hasValidContent(html)) {
+      return {
+        success: false,
+        html,
+        title: extractTitle(html),
+        shouldFallback: true,
+        fallbackReason: "skeleton",
+        finalUrl
+      };
+    }
+    return {
+      success: true,
+      html,
+      title: extractTitle(html),
+      shouldFallback: false,
+      finalUrl
+    };
+  } catch (error) {
+    clearTimeout(timeoutId);
+    const errorMessage = error instanceof Error ? error.message : "Unknown error";
+    if (errorMessage.includes("abort")) {
+      return {
+        success: false,
+        shouldFallback: true,
+        fallbackReason: "error",
+        error: "Request timeout"
+      };
+    }
+    return {
+      success: false,
+      shouldFallback: true,
+      fallbackReason: "error",
+      error: errorMessage
+    };
+  }
+}
+
+// src/tiers/headless.ts
+async function headlessFetch(url, netlifyEndpoint, timeout = 3e4) {
+  if (!netlifyEndpoint) {
+    return {
+      success: false,
+      shouldFallback: true,
+      fallbackReason: "error",
+      error: "No Netlify endpoint configured. Set netlifyEndpoint in config."
+    };
+  }
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(netlifyEndpoint, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({ url }),
+      signal: controller.signal
+    });
+    clearTimeout(timeoutId);
+    if (!response.ok) {
+      const errorText = await response.text();
+      return {
+        success: false,
+        shouldFallback: true,
+        fallbackReason: "error",
+        error: `Headless fetch failed: ${response.status} - ${errorText}`
+      };
+    }
+    const result = await response.json();
+    if (!result.success || !result.html) {
+      return {
+        success: false,
+        shouldFallback: true,
+        fallbackReason: "error",
+        error: result.error || "No HTML content returned from headless fetch"
+      };
+    }
+    const html = result.html;
+    const title = extractTitle(html);
+    if (isPaywalled(html)) {
+      return {
+        success: false,
+        html,
+        title,
+        shouldFallback: true,
+        fallbackReason: "paywall",
+        finalUrl: result.finalUrl
+      };
+    }
+    return {
+      success: true,
+      html,
+      title,
+      shouldFallback: false,
+      finalUrl: result.finalUrl
+    };
+  } catch (error) {
+    clearTimeout(timeoutId);
+    const errorMessage = error instanceof Error ? error.message : "Unknown error";
+    if (errorMessage.includes("abort")) {
+      return {
+        success: false,
+        shouldFallback: true,
+        fallbackReason: "error",
+        error: "Headless fetch timeout"
+      };
+    }
+    return {
+      success: false,
+      shouldFallback: true,
+      fallbackReason: "error",
+      error: `Headless fetch error: ${errorMessage}`
+    };
+  }
+}
+
+// src/tiers/search.ts
+var DDG_SEARCH_URL = "https://html.duckduckgo.com/html/";
+function calculateSimilarity(str1, str2) {
+  const s1 = str1.toLowerCase().trim();
+  const s2 = str2.toLowerCase().trim();
+  if (s1 === s2) return 100;
+  if (s1.length === 0 || s2.length === 0) return 0;
+  const words1 = new Set(s1.split(/\s+/).filter((w) => w.length > 2));
+  const words2 = new Set(s2.split(/\s+/).filter((w) => w.length > 2));
+  if (words1.size === 0 || words2.size === 0) return 0;
+  let matches = 0;
+  for (const word of words1) {
+    if (words2.has(word)) matches++;
+  }
+  const similarity = matches * 2 / (words1.size + words2.size) * 100;
+  return Math.round(similarity);
+}
+function parseSearchResults(html) {
+  const results = [];
+  const resultRegex = /<a[^>]*class="result__a"[^>]*href="([^"]*)"[^>]*>([^<]*)<\/a>/gi;
+  const snippetRegex = /<a[^>]*class="result__snippet"[^>]*>([^<]*)<\/a>/gi;
+  const links = [];
+  let match;
+  while ((match = resultRegex.exec(html)) !== null) {
+    const url = decodeURIComponent(match[1].replace(/.*uddg=/, "").split("&")[0]);
+    const title = match[2].trim();
+    if (url.startsWith("http")) {
+      links.push({ url, title });
+    }
+  }
+  const snippets = [];
+  while ((match = snippetRegex.exec(html)) !== null) {
+    snippets.push(match[1].trim());
+  }
+  for (let i = 0; i < links.length; i++) {
+    results.push({
+      url: links[i].url,
+      title: links[i].title,
+      snippet: snippets[i] || "",
+      matchScore: 0
+      // Will be calculated later
+    });
+  }
+  return results;
+}
+async function searchFallback(title, originalUrl, timeout = 3e4) {
+  if (!title || title.trim().length === 0) {
+    return {
+      success: false,
+      shouldFallback: false,
+      error: "No title available for search fallback",
+      suggestions: []
+    };
+  }
+  try {
+    const searchUrl = `${DDG_SEARCH_URL}?q=${encodeURIComponent(title)}`;
+    const response = await fetch(searchUrl, {
+      method: "GET",
+      headers: {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+        "Accept": "text/html"
+      }
+    });
+    if (!response.ok) {
+      return {
+        success: false,
+        shouldFallback: false,
+        error: `Search failed: ${response.status}`,
+        suggestions: []
+      };
+    }
+    const searchHtml = await response.text();
+    const results = parseSearchResults(searchHtml);
+    const originalDomain = new URL(originalUrl).hostname;
+    const scoredResults = results.filter((r) => !r.url.includes(originalDomain)).map((r) => ({
+      ...r,
+      matchScore: calculateSimilarity(title, r.title)
+    })).sort((a, b) => b.matchScore - a.matchScore);
+    const perfectMatch = scoredResults.find((r) => r.matchScore === 100);
+    if (perfectMatch) {
+      const mirrorResult = await lightFetch(perfectMatch.url, timeout);
+      if (mirrorResult.success && mirrorResult.html) {
+        return {
+          success: true,
+          html: mirrorResult.html,
+          title: mirrorResult.title || title,
+          shouldFallback: false,
+          finalUrl: perfectMatch.url
+        };
+      }
+    }
+    const suggestions = scoredResults.slice(0, 5).map((r) => r.url);
+    return {
+      success: false,
+      shouldFallback: false,
+      error: "No exact match found",
+      suggestions
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : "Unknown error";
+    return {
+      success: false,
+      shouldFallback: false,
+      error: `Search fallback error: ${errorMessage}`,
+      suggestions: []
+    };
+  }
+}
+
+// src/parsers/json.ts
+var cheerio2 = __toESM(require("cheerio"));
+function extractJson(html) {
+  const $ = cheerio2.load(html);
+  const result = {};
+  const jsonLdScripts = $('script[type="application/ld+json"]');
+  const jsonLdData = [];
+  jsonLdScripts.each((_, el) => {
+    try {
+      const content = $(el).html();
+      if (content) {
+        const parsed = JSON.parse(content);
+        jsonLdData.push(parsed);
+      }
+    } catch {
+    }
+  });
+  if (jsonLdData.length > 0) {
+    result.jsonLd = jsonLdData.length === 1 ? jsonLdData[0] : jsonLdData;
+  }
+  const ogData = {};
+  $('meta[property^="og:"]').each((_, el) => {
+    const property = $(el).attr("property")?.replace("og:", "");
+    const content = $(el).attr("content");
+    if (property && content) {
+      ogData[property] = content;
+    }
+  });
+  if (Object.keys(ogData).length > 0) {
+    result.openGraph = ogData;
+  }
+  const twitterData = {};
+  $('meta[name^="twitter:"]').each((_, el) => {
+    const name = $(el).attr("name")?.replace("twitter:", "");
+    const content = $(el).attr("content");
+    if (name && content) {
+      twitterData[name] = content;
+    }
+  });
+  if (Object.keys(twitterData).length > 0) {
+    result.twitter = twitterData;
+  }
+  const meta = {};
+  const metaSelectors = [
+    { name: "description", selector: 'meta[name="description"]' },
+    { name: "keywords", selector: 'meta[name="keywords"]' },
+    { name: "author", selector: 'meta[name="author"]' },
+    { name: "robots", selector: 'meta[name="robots"]' }
+  ];
+  for (const { name, selector } of metaSelectors) {
+    const content = $(selector).attr("content");
+    if (content) {
+      meta[name] = content;
+    }
+  }
+  if (Object.keys(meta).length > 0) {
+    result.meta = meta;
+  }
+  result.title = $("title").text().trim() || $('meta[property="og:title"]').attr("content") || $("h1").first().text().trim() || "";
+  const article = $("article").first();
+  if (article.length > 0) {
+    result.article = {
+      headline: article.find("h1, h2").first().text().trim(),
+      content: article.text().replace(/\s+/g, " ").trim().substring(0, 1e3)
+    };
+  }
+  const links = [];
+  $("a[href]").each((_, el) => {
+    const href = $(el).attr("href");
+    const text = $(el).text().trim();
+    if (href && text && href.startsWith("http") && links.length < 20) {
+      links.push({ text, href });
+    }
+  });
+  if (links.length > 0) {
+    result.links = links;
+  }
+  return Object.keys(result).length > 0 ? result : null;
+}
+
+// src/parsers/text.ts
+var cheerio3 = __toESM(require("cheerio"));
+var import_turndown = __toESM(require("turndown"));
+var turndown = new import_turndown.default({
+  headingStyle: "atx",
+  codeBlockStyle: "fenced",
+  bulletListMarker: "-"
+});
+turndown.remove(["script", "style", "nav", "footer", "header", "aside", "iframe", "noscript"]);
+function extractText(html, asMarkdown = true) {
+  const $ = cheerio3.load(html);
+  $('script, style, nav, footer, header, aside, iframe, noscript, [role="navigation"], [role="banner"], [role="contentinfo"]').remove();
+  $('[class*="sidebar"], [class*="navigation"], [class*="menu"], [class*="footer"], [class*="header"], [class*="ad-"], [class*="advertisement"], [id*="sidebar"], [id*="navigation"], [id*="menu"], [id*="footer"], [id*="header"], [id*="ad-"]').remove();
+  let contentElement = $("article").first();
+  if (contentElement.length === 0) {
+    contentElement = $("main").first();
+  }
+  if (contentElement.length === 0) {
+    contentElement = $('[role="main"]').first();
+  }
+  if (contentElement.length === 0) {
+    const contentSelectors = [
+      '[class*="article-content"]',
+      '[class*="post-content"]',
+      '[class*="entry-content"]',
+      '[class*="content-body"]',
+      '[class*="story-body"]',
+      ".content",
+      "#content"
+    ];
+    for (const selector of contentSelectors) {
+      const el = $(selector).first();
+      if (el.length > 0 && el.text().trim().length > 200) {
+        contentElement = el;
+        break;
+      }
+    }
+  }
+  if (contentElement.length === 0) {
+    contentElement = $("body");
+  }
+  const contentHtml = contentElement.html() || "";
+  if (asMarkdown) {
+    let markdown = turndown.turndown(contentHtml);
+    markdown = markdown.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+/g, " ").trim();
+    return markdown;
+  } else {
+    let text = contentElement.text();
+    text = text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").trim();
+    return text;
+  }
+}
+
+// src/fetcher.ts
+async function fetchContent(url, config) {
+  const timeout = config.timeout ?? 3e4;
+  const headers = config.headers ?? {};
+  const tier1Result = await lightFetch(url, timeout, headers);
+  if (tier1Result.success && tier1Result.html) {
+    return formatResult(tier1Result.html, config.mode, 1, tier1Result.finalUrl, tier1Result.title);
+  }
+  if (!config.skipHeadless && tier1Result.shouldFallback) {
+    const netlifyEndpoint = config.netlifyEndpoint;
+    if (netlifyEndpoint) {
+      const tier2Result = await headlessFetch(url, netlifyEndpoint, timeout);
+      if (tier2Result.success && tier2Result.html) {
+        return formatResult(tier2Result.html, config.mode, 2, tier2Result.finalUrl, tier2Result.title);
+      }
+      if (tier2Result.fallbackReason !== "paywall" && !config.skipSearch) {
+        if (tier1Result.html) {
+          return formatResult(tier1Result.html, config.mode, 1, tier1Result.finalUrl, tier1Result.title);
+        }
+      }
+      if (tier2Result.fallbackReason === "paywall" || isPaywalled(tier1Result.html || "")) {
+        if (!config.skipSearch) {
+          const title = tier2Result.title || tier1Result.title || "";
+          return await executeSearchFallback(title, url, config.mode, timeout);
+        }
+      }
+    } else if (tier1Result.fallbackReason === "skeleton") {
+      if (tier1Result.html) {
+        const result = formatResult(tier1Result.html, config.mode, 1, tier1Result.finalUrl, tier1Result.title);
+        result.error = "Content may be incomplete (no headless endpoint configured)";
+        return result;
+      }
+    }
+  }
+  if (!config.skipSearch && (tier1Result.fallbackReason === "paywall" || isPaywalled(tier1Result.html || ""))) {
+    const title = tier1Result.title || "";
+    return await executeSearchFallback(title, url, config.mode, timeout);
+  }
+  if (tier1Result.html) {
+    const result = formatResult(tier1Result.html, config.mode, 1, tier1Result.finalUrl, tier1Result.title);
+    if (tier1Result.error) {
+      result.error = tier1Result.error;
+    }
+    return result;
+  }
+  return {
+    success: false,
+    error: tier1Result.error || "Failed to fetch content"
+  };
+}
+async function executeSearchFallback(title, originalUrl, mode, timeout) {
+  const searchResult = await searchFallback(title, originalUrl, timeout);
+  if (searchResult.success && searchResult.html) {
+    return formatResult(searchResult.html, mode, 3, searchResult.finalUrl, searchResult.title);
+  }
+  return {
+    success: false,
+    error: "Content not found",
+    suggestions: searchResult.suggestions || [],
+    title
+  };
+}
+function formatResult(html, mode, tier, finalUrl, title) {
+  if (mode === "JSON") {
+    const jsonContent = extractJson(html);
+    return {
+      success: true,
+      content: jsonContent || { rawHtml: html },
+      tier,
+      finalUrl,
+      title
+    };
+  } else {
+    const textContent = extractText(html, true);
+    return {
+      success: true,
+      content: textContent,
+      tier,
+      finalUrl,
+      title
+    };
+  }
+}
+
+// src/types.ts
+var DEFAULT_CONFIG = {
+  mode: "TEXT",
+  timeout: 3e4,
+  netlifyEndpoint: "",
+  headers: {},
+  skipHeadless: false,
+  skipSearch: false
+};
+
+// src/index.ts
+async function omniFetch(url, config = {}) {
+  if (!url || typeof url !== "string") {
+    return {
+      success: false,
+      error: "Invalid URL provided"
+    };
+  }
+  try {
+    new URL(url);
+  } catch {
+    return {
+      success: false,
+      error: "Invalid URL format"
+    };
+  }
+  const fullConfig = {
+    mode: config.mode ?? DEFAULT_CONFIG.mode,
+    timeout: config.timeout ?? DEFAULT_CONFIG.timeout,
+    netlifyEndpoint: config.netlifyEndpoint ?? DEFAULT_CONFIG.netlifyEndpoint,
+    headers: config.headers ?? DEFAULT_CONFIG.headers,
+    skipHeadless: config.skipHeadless ?? DEFAULT_CONFIG.skipHeadless,
+    skipSearch: config.skipSearch ?? DEFAULT_CONFIG.skipSearch
+  };
+  return await fetchContent(url, fullConfig);
+}
+async function fetchText(url, options) {
+  return omniFetch(url, { ...options, mode: "TEXT" });
+}
+async function fetchJson(url, options) {
+  return omniFetch(url, { ...options, mode: "JSON" });
+}
+var index_default = omniFetch;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DEFAULT_CONFIG,
+  fetchJson,
+  fetchText,
+  omniFetch
+});