@detiq/crawler 1.1.9 → 1.1.11

package/dist/extractors/advanced.js

@@ -1,282 +1,265 @@
-import { parseHTML } from 'linkedom';
-export function extractByCssSelector(html, selector, attr) {
-    try {
-        const { document } = parseHTML(html);
-        const nodes = Array.from(document.querySelectorAll(selector));
-        const matches = nodes.map((n) => {
-            if (attr)
-                return n.getAttribute(attr) ?? '';
-            return n.textContent?.trim() ?? '';
-        }).filter(Boolean);
-        return { matches, count: matches.length, method: 'css-selector' };
-    }
-    catch (e) {
-        return { matches: [], count: 0, method: 'css-selector-error' };
-    }
+import { parseHTML } from "linkedom";
+function extractByCssSelector(html, selector, attr) {
+  try {
+    const { document } = parseHTML(html);
+    const nodes = Array.from(document.querySelectorAll(selector));
+    const matches = nodes.map((n) => {
+      if (attr) return n.getAttribute(attr) ?? "";
+      return n.textContent?.trim() ?? "";
+    }).filter(Boolean);
+    return { matches, count: matches.length, method: "css-selector" };
+  } catch (e) {
+    return { matches: [], count: 0, method: "css-selector-error" };
+  }
 }
-export function extractByXPath(html, xpath) {
-    try {
-        const { document, XPathResult } = parseHTML(html);
-        const result = document.evaluate(xpath, document, null, XPathResult?.ANY_TYPE ?? 0, null);
-        const matches = [];
-        let node = result.iterateNext?.();
-        while (node) {
-            const text = node.textContent?.trim() ?? node.nodeValue?.trim() ?? '';
-            if (text)
-                matches.push(text);
-            node = result.iterateNext?.();
-        }
-        return { matches, count: matches.length, method: 'xpath' };
+function extractByXPath(html, xpath) {
+  try {
+    const { document, XPathResult } = parseHTML(html);
+    const result = document.evaluate(xpath, document, null, XPathResult?.ANY_TYPE ?? 0, null);
+    const matches = [];
+    let node = result.iterateNext?.();
+    while (node) {
+      const text = node.textContent?.trim() ?? node.nodeValue?.trim() ?? "";
+      if (text) matches.push(text);
+      node = result.iterateNext?.();
     }
-    catch (e) {
-        return { matches: [], count: 0, method: 'xpath-error' };
+    return { matches, count: matches.length, method: "xpath" };
+  } catch (e) {
+    return { matches: [], count: 0, method: "xpath-error" };
+  }
+}
+function extractByRegex(text, pattern, flags) {
+  try {
+    const re = new RegExp(pattern, (flags ?? "g").includes("g") ? flags : (flags ?? "") + "g");
+    const matches = [];
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      matches.push(m[1] ?? m[0]);
+      if (!re.global) break;
     }
+    return { matches, count: matches.length, method: "regex" };
+  } catch {
+    return { matches: [], count: 0, method: "regex-error" };
+  }
 }
-export function extractByRegex(text, pattern, flags) {
-    try {
-        const re = new RegExp(pattern, (flags ?? 'g').includes('g') ? flags : (flags ?? '') + 'g');
-        const matches = [];
-        let m;
-        while ((m = re.exec(text)) !== null) {
-            matches.push(m[1] ?? m[0]);
-            if (!re.global)
-                break;
+function extractTables(html) {
+  try {
+    const { document } = parseHTML(html);
+    const tables = Array.from(document.querySelectorAll("table"));
+    return tables.map((table) => {
+      const headers = [];
+      const rows = [];
+      const ths = [...table.querySelectorAll("thead th, thead td, tr:first-child th")];
+      if (ths.length > 0) {
+        ths.forEach((th, i) => headers.push(th.textContent?.trim() || `col${i + 1}`));
+      }
+      const trs = [...table.querySelectorAll("tbody tr, tr")];
+      for (const tr of trs) {
+        const cells = [...tr.querySelectorAll("td, th")];
+        if (cells.length === 0) continue;
+        if (headers.length === 0) {
+          cells.forEach((_, i) => headers.push(`col${i + 1}`));
         }
-        return { matches, count: matches.length, method: 'regex' };
-    }
-    catch {
-        return { matches: [], count: 0, method: 'regex-error' };
-    }
+        const row = {};
+        cells.forEach((cell, i) => {
+          const key = headers[i] || `col${i + 1}`;
+          row[key] = cell.textContent?.trim() ?? "";
+        });
+        if (Object.values(row).some((v) => v)) rows.push(row);
+      }
+      return rows;
+    }).filter((t) => t.length > 0);
+  } catch {
+    return [];
+  }
 }
-export function extractTables(html) {
-    try {
-        const { document } = parseHTML(html);
-        const tables = Array.from(document.querySelectorAll('table'));
-        return tables.map((table) => {
-            const headers = [];
-            const rows = [];
-            const ths = [...table.querySelectorAll('thead th, thead td, tr:first-child th')];
-            if (ths.length > 0) {
-                ths.forEach((th, i) => headers.push(th.textContent?.trim() || `col${i + 1}`));
-            }
-            const trs = [...table.querySelectorAll('tbody tr, tr')];
-            for (const tr of trs) {
-                const cells = [...tr.querySelectorAll('td, th')];
-                if (cells.length === 0)
-                    continue;
-                if (headers.length === 0) {
-                    cells.forEach((_, i) => headers.push(`col${i + 1}`));
-                }
-                const row = {};
-                cells.forEach((cell, i) => {
-                    const key = headers[i] || `col${i + 1}`;
-                    row[key] = cell.textContent?.trim() ?? '';
-                });
-                if (Object.values(row).some(v => v))
-                    rows.push(row);
-            }
-            return rows;
-        }).filter(t => t.length > 0);
+function extractMetadata(html, url) {
+  const meta = {
+    title: null,
+    description: null,
+    keywords: null,
+    ogTitle: null,
+    ogDescription: null,
+    ogImage: null,
+    ogUrl: null,
+    ogType: null,
+    ogSiteName: null,
+    ogLocale: null,
+    twitterCard: null,
+    twitterTitle: null,
+    twitterDescription: null,
+    twitterImage: null,
+    twitterSite: null,
+    canonical: null,
+    robots: null,
+    hreflang: [],
+    themeColor: null,
+    favicon: null,
+    jsonLd: [],
+    schemaTypes: [],
+    lang: null
+  };
+  try {
+    const { document } = parseHTML(html);
+    meta.title = document.title ?? document.querySelector("title")?.textContent?.trim() ?? null;
+    meta.lang = document.documentElement?.getAttribute("lang") ?? null;
+    const metas = [...document.querySelectorAll("meta")];
+    for (const m of metas) {
+      const name = m.getAttribute("name")?.toLowerCase() ?? "";
+      const prop = m.getAttribute("property")?.toLowerCase() ?? "";
+      const content = m.getAttribute("content") ?? "";
+      if (name === "description") meta.description = content;
+      if (name === "keywords") meta.keywords = content;
+      if (name === "robots") meta.robots = content;
+      if (name === "twitter:card") meta.twitterCard = content;
+      if (name === "twitter:title") meta.twitterTitle = content;
+      if (name === "twitter:description") meta.twitterDescription = content;
+      if (name === "twitter:image") meta.twitterImage = content;
+      if (name === "twitter:site") meta.twitterSite = content;
+      if (name === "theme-color") meta.themeColor = content;
+      if (prop === "og:title") meta.ogTitle = content;
+      if (prop === "og:description") meta.ogDescription = content;
+      if (prop === "og:image") meta.ogImage = content;
+      if (prop === "og:url") meta.ogUrl = content;
+      if (prop === "og:type") meta.ogType = content;
+      if (prop === "og:site_name") meta.ogSiteName = content;
+      if (prop === "og:locale") meta.ogLocale = content;
     }
-    catch {
-        return [];
+    const canonical = document.querySelector('link[rel="canonical"]');
+    if (canonical) meta.canonical = canonical.getAttribute("href") ?? null;
+    const faviconEl = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]');
+    if (faviconEl) meta.favicon = faviconEl.getAttribute("href") ?? null;
+    const hreflangEls = [...document.querySelectorAll('link[rel="alternate"][hreflang]')];
+    for (const el of hreflangEls) {
+      const lang = el.getAttribute("hreflang") ?? "";
+      const href = el.getAttribute("href") ?? "";
+      if (lang && href) meta.hreflang.push({ lang, href });
     }
-}
-export function extractMetadata(html, url) {
-    const meta = {
-        title: null, description: null, keywords: null,
-        ogTitle: null, ogDescription: null, ogImage: null, ogUrl: null, ogType: null,
-        ogSiteName: null, ogLocale: null,
-        twitterCard: null, twitterTitle: null, twitterDescription: null,
-        twitterImage: null, twitterSite: null,
-        canonical: null, robots: null,
-        hreflang: [], themeColor: null, favicon: null,
-        jsonLd: [], schemaTypes: [], lang: null,
-    };
-    try {
-        const { document } = parseHTML(html);
-        meta.title = document.title ?? document.querySelector('title')?.textContent?.trim() ?? null;
-        meta.lang = document.documentElement?.getAttribute('lang') ?? null;
-        const metas = [...document.querySelectorAll('meta')];
-        for (const m of metas) {
-            const name = m.getAttribute('name')?.toLowerCase() ?? '';
-            const prop = m.getAttribute('property')?.toLowerCase() ?? '';
-            const content = m.getAttribute('content') ?? '';
-            if (name === 'description')
-                meta.description = content;
-            if (name === 'keywords')
-                meta.keywords = content;
-            if (name === 'robots')
-                meta.robots = content;
-            if (name === 'twitter:card')
-                meta.twitterCard = content;
-            if (name === 'twitter:title')
-                meta.twitterTitle = content;
-            if (name === 'twitter:description')
-                meta.twitterDescription = content;
-            if (name === 'twitter:image')
-                meta.twitterImage = content;
-            if (name === 'twitter:site')
-                meta.twitterSite = content;
-            if (name === 'theme-color')
-                meta.themeColor = content;
-            if (prop === 'og:title')
-                meta.ogTitle = content;
-            if (prop === 'og:description')
-                meta.ogDescription = content;
-            if (prop === 'og:image')
-                meta.ogImage = content;
-            if (prop === 'og:url')
-                meta.ogUrl = content;
-            if (prop === 'og:type')
-                meta.ogType = content;
-            if (prop === 'og:site_name')
-                meta.ogSiteName = content;
-            if (prop === 'og:locale')
-                meta.ogLocale = content;
-        }
-        const canonical = document.querySelector('link[rel="canonical"]');
-        if (canonical)
-            meta.canonical = canonical.getAttribute('href') ?? null;
-        const faviconEl = document.querySelector('link[rel="icon"], link[rel="shortcut icon"]');
-        if (faviconEl)
-            meta.favicon = faviconEl.getAttribute('href') ?? null;
-        const hreflangEls = [...document.querySelectorAll('link[rel="alternate"][hreflang]')];
-        for (const el of hreflangEls) {
-            const lang = el.getAttribute('hreflang') ?? '';
-            const href = el.getAttribute('href') ?? '';
-            if (lang && href)
-                meta.hreflang.push({ lang, href });
-        }
-        const jsonLdScripts = [...document.querySelectorAll('script[type="application/ld+json"]')];
-        for (const s of jsonLdScripts) {
-            try {
-                const parsed = JSON.parse(s.textContent ?? '{}');
-                meta.jsonLd.push(parsed);
-                const types = Array.isArray(parsed) ? parsed.map((p) => p['@type']).flat() : [parsed['@type']];
-                meta.schemaTypes.push(...types.filter(Boolean));
-            }
-            catch { /* skip invalid JSON-LD */ }
-        }
-    }
-    catch { /* non-fatal */ }
-    return meta;
-}
-export function extractStructuredProduct(jsonLd) {
-    for (const item of jsonLd) {
-        try {
-            const type = item['@type'];
-            const isProduct = Array.isArray(type)
-                ? type.some((t) => typeof t === 'string' && t.toLowerCase() === 'product')
-                : typeof type === 'string' && type.toLowerCase() === 'product';
-            if (!isProduct)
-                continue;
-            const i = item;
-            const offers = Array.isArray(i.offers) ? i.offers[0] : i.offers;
-            const rawPrice = offers?.price;
-            const price = rawPrice != null ? parseFloat(rawPrice) : NaN;
-            const rawRating = i.aggregateRating?.ratingValue;
-            const rating = rawRating != null ? parseFloat(rawRating) : NaN;
-            const rawReviewCount = i.aggregateRating?.reviewCount;
-            const reviewCount = rawReviewCount != null ? parseInt(rawReviewCount, 10) : NaN;
-            const rawAvailability = offers?.availability;
-            const availability = rawAvailability
-                ? rawAvailability.replace(/^https?:\/\/schema\.org\//, '')
-                : null;
-            const rawImage = i.image;
-            const image = Array.isArray(rawImage) ? (rawImage[0] ?? null) : (rawImage ?? null);
-            const brand = i.brand?.name ?? i.brand ?? null;
-            return {
-                name: i.name ?? null,
-                description: i.description ?? null,
-                brand: typeof brand === 'string' ? brand : null,
-                sku: i.sku ?? null,
-                price: isNaN(price) ? null : price,
-                priceCurrency: offers?.priceCurrency ?? null,
-                availability,
-                rating: isNaN(rating) ? null : rating,
-                reviewCount: isNaN(reviewCount) ? null : reviewCount,
-                image,
-                url: i.url ?? null,
-            };
-        }
-        catch { /* skip malformed item */ }
+    const jsonLdScripts = [...document.querySelectorAll('script[type="application/ld+json"]')];
+    for (const s of jsonLdScripts) {
+      try {
+        const parsed = JSON.parse(s.textContent ?? "{}");
+        meta.jsonLd.push(parsed);
+        const types = Array.isArray(parsed) ? parsed.map((p) => p["@type"]).flat() : [parsed["@type"]];
+        meta.schemaTypes.push(...types.filter(Boolean));
+      } catch {
+      }
     }
-    return null;
+  } catch {
+  }
+  return meta;
 }
-export function extractImages(html, baseUrl) {
+function extractStructuredProduct(jsonLd) {
+  for (const item of jsonLd) {
     try {
-        const { document } = parseHTML(html);
-        const base = new URL(baseUrl);
-        return Array.from(document.querySelectorAll('img'))
-            .map((img) => {
-            const src = img.getAttribute('src') ?? img.getAttribute('data-src') ?? '';
-            if (!src)
-                return null;
-            try {
-                return {
-                    src: new URL(src, base).href,
-                    alt: (img.getAttribute('alt') ?? ''),
-                    width: img.getAttribute('width') ? parseInt(img.getAttribute('width')) : undefined,
-                    height: img.getAttribute('height') ? parseInt(img.getAttribute('height')) : undefined,
-                };
-            }
-            catch {
-                return null;
-            }
-        })
-            .filter((x) => x !== null);
-    }
-    catch {
-        return [];
+      const type = item["@type"];
+      const isProduct = Array.isArray(type) ? type.some((t) => typeof t === "string" && t.toLowerCase() === "product") : typeof type === "string" && type.toLowerCase() === "product";
+      if (!isProduct) continue;
+      const i = item;
+      const offers = Array.isArray(i.offers) ? i.offers[0] : i.offers;
+      const rawPrice = offers?.price;
+      const price = rawPrice != null ? parseFloat(rawPrice) : NaN;
+      const rawRating = i.aggregateRating?.ratingValue;
+      const rating = rawRating != null ? parseFloat(rawRating) : NaN;
+      const rawReviewCount = i.aggregateRating?.reviewCount;
+      const reviewCount = rawReviewCount != null ? parseInt(rawReviewCount, 10) : NaN;
+      const rawAvailability = offers?.availability;
+      const availability = rawAvailability ? rawAvailability.replace(/^https?:\/\/schema\.org\//, "") : null;
+      const rawImage = i.image;
+      const image = Array.isArray(rawImage) ? rawImage[0] ?? null : rawImage ?? null;
+      const brand = i.brand?.name ?? i.brand ?? null;
+      return {
+        name: i.name ?? null,
+        description: i.description ?? null,
+        brand: typeof brand === "string" ? brand : null,
+        sku: i.sku ?? null,
+        price: isNaN(price) ? null : price,
+        priceCurrency: offers?.priceCurrency ?? null,
+        availability,
+        rating: isNaN(rating) ? null : rating,
+        reviewCount: isNaN(reviewCount) ? null : reviewCount,
+        image,
+        url: i.url ?? null
+      };
+    } catch {
     }
+  }
+  return null;
 }
-export function extractMediaFiles(html, baseUrl) {
-    const IMAGE_EXT = /\.(png|jpe?g|gif|webp|svg|ico|avif)(\?.*)?$/i;
-    const VIDEO_EXT = /\.(mp4|webm|ogv|mov|avi)(\?.*)?$/i;
-    const AUDIO_EXT = /\.(mp3|ogg|wav|flac|aac)(\?.*)?$/i;
-    const DOC_EXT = /\.(pdf|docx?|xlsx?|pptx?|csv|zip)(\?.*)?$/i;
-    function classify(url, tag) {
-        if (tag === 'img' || IMAGE_EXT.test(url))
-            return 'image';
-        if (tag === 'video' || VIDEO_EXT.test(url))
-            return 'video';
-        if (tag === 'audio' || AUDIO_EXT.test(url))
-            return 'audio';
-        if (DOC_EXT.test(url))
-            return 'document';
-        return 'document';
-    }
-    try {
-        const { document } = parseHTML(html);
-        const base = new URL(baseUrl);
-        const results = [];
-        const seen = new Set();
-        const selectors = ['img[src],video[src],audio[src],source[src]', 'src'];
-        const attrPairs = [
-            ['img', 'src'], ['video', 'src'], ['audio', 'src'], ['source', 'src'],
-            ['a', 'href'],
-        ];
-        for (const [sel, attr] of attrPairs) {
-            for (const el of [...document.querySelectorAll(sel)]) {
-                const raw = el.getAttribute(attr) ?? '';
-                if (!raw)
-                    continue;
-                try {
-                    const resolved = new URL(raw, base).href;
-                    if (seen.has(resolved))
-                        continue;
-                    if (sel === 'a' && !DOC_EXT.test(resolved) && !IMAGE_EXT.test(resolved) && !VIDEO_EXT.test(resolved) && !AUDIO_EXT.test(resolved))
-                        continue;
-                    seen.add(resolved);
-                    results.push({ url: resolved, type: classify(resolved, sel), tag: sel });
-                }
-                catch { /* skip */ }
-            }
+function extractImages(html, baseUrl) {
+  try {
+    const { document } = parseHTML(html);
+    const base = new URL(baseUrl);
+    return Array.from(document.querySelectorAll("img")).map((img) => {
+      const src = img.getAttribute("src") ?? img.getAttribute("data-src") ?? "";
+      if (!src) return null;
+      try {
+        return {
+          src: new URL(src, base).href,
+          alt: img.getAttribute("alt") ?? "",
+          width: img.getAttribute("width") ? parseInt(img.getAttribute("width")) : void 0,
+          height: img.getAttribute("height") ? parseInt(img.getAttribute("height")) : void 0
+        };
+      } catch {
+        return null;
+      }
+    }).filter((x) => x !== null);
+  } catch {
+    return [];
+  }
+}
+function extractMediaFiles(html, baseUrl) {
+  const IMAGE_EXT = /\.(png|jpe?g|gif|webp|svg|ico|avif)(\?.*)?$/i;
+  const VIDEO_EXT = /\.(mp4|webm|ogv|mov|avi)(\?.*)?$/i;
+  const AUDIO_EXT = /\.(mp3|ogg|wav|flac|aac)(\?.*)?$/i;
+  const DOC_EXT = /\.(pdf|docx?|xlsx?|pptx?|csv|zip)(\?.*)?$/i;
+  function classify(url, tag) {
+    if (tag === "img" || IMAGE_EXT.test(url)) return "image";
+    if (tag === "video" || VIDEO_EXT.test(url)) return "video";
+    if (tag === "audio" || AUDIO_EXT.test(url)) return "audio";
+    if (DOC_EXT.test(url)) return "document";
+    return "document";
+  }
+  try {
+    const { document } = parseHTML(html);
+    const base = new URL(baseUrl);
+    const results = [];
+    const seen = /* @__PURE__ */ new Set();
+    const selectors = ["img[src],video[src],audio[src],source[src]", "src"];
+    const attrPairs = [
+      ["img", "src"],
+      ["video", "src"],
+      ["audio", "src"],
+      ["source", "src"],
+      ["a", "href"]
+    ];
+    for (const [sel, attr] of attrPairs) {
+      for (const el of [...document.querySelectorAll(sel)]) {
+        const raw = el.getAttribute(attr) ?? "";
+        if (!raw) continue;
+        try {
+          const resolved = new URL(raw, base).href;
+          if (seen.has(resolved)) continue;
+          if (sel === "a" && !DOC_EXT.test(resolved) && !IMAGE_EXT.test(resolved) && !VIDEO_EXT.test(resolved) && !AUDIO_EXT.test(resolved)) continue;
+          seen.add(resolved);
+          results.push({ url: resolved, type: classify(resolved, sel), tag: sel });
+        } catch {
         }
-        return results;
-    }
-    catch {
-        return [];
+      }
     }
+    return results;
+  } catch {
+    return [];
+  }
 }
+export {
+  extractByCssSelector,
+  extractByRegex,
+  extractByXPath,
+  extractImages,
+  extractMediaFiles,
+  extractMetadata,
+  extractStructuredProduct,
+  extractTables
+};
+//# sourceMappingURL=advanced.js.map