@detiq/crawler 1.1.9 → 1.1.11

package/dist/screen-reader.js

@@ -1,72 +1,47 @@
-export async function readScreen(page) {
-    return page.evaluate(() => {
-        const out = [];
-        const seen = new Set();
-        // Noise patterns to skip — cookie banners, footer chrome, version strings
-        // Intentionally excludes standalone "close" and "copy" — too many false positives
-        // (Close account, Close preview, Copy link, Copy to clipboard are all valid test targets)
-        const noiseLabel = /^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|got it|dismiss|skip to|powered by|copied|\s*$)/i;
-        // Include ARIA-role elements — Headless UI, Radix, Ant Design, Shadcn all use these instead of native tags
-        const nodes = document.querySelectorAll('input:not([type=hidden]), select, textarea, button, a[href], ' +
-            '[role="button"], [role="combobox"], [role="slider"], [role="switch"], ' +
-            '[role="tab"], [role="menuitem"], [role="searchbox"], [role="spinbutton"]');
-        nodes.forEach((el) => {
-            const htmlEl = el;
-            // Skip invisible elements
-            const style = window.getComputedStyle(htmlEl);
-            if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')
-                return;
-            const rect = el.getBoundingClientRect();
-            // Allow zero-size elements that carry an accessible label (sr-only icon buttons, skip-nav links)
-            const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby'));
-            if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel)
-                return;
-            // Derive role
-            const tag = el.tagName.toLowerCase();
-            const role = el.getAttribute('role') ||
-                (tag === 'input' ? (el.type || 'text') : tag);
-            // Derive the best human-readable label (prefer aria > placeholder > inner text)
-            const raw = (el.getAttribute('aria-label') ||
-                el.getAttribute('placeholder') ||
-                (el.getAttribute('title') ?? '') ||
-                htmlEl.innerText?.replace(/\s+/g, ' ').trim() ||
-                el.getAttribute('name') ||
-                '').substring(0, 80);
-            // Skip empty, noise patterns, or very long labels (likely container text)
-            if (!raw || noiseLabel.test(raw) || raw.length > 70)
-                return;
-            // Deduplicate by role+normalised label
-            const key = `${role}:${raw.toLowerCase()}`;
-            if (seen.has(key))
-                return;
-            seen.add(key);
-            let expectedData;
-            if (tag === 'input') {
-                const input = el;
-                if (input.type === 'email' || /email/i.test(raw))
-                    expectedData = 'valid_email';
-                else if (input.type === 'password' || /password/i.test(raw))
-                    expectedData = 'password';
-                else if (input.type === 'tel' || /phone|mobile/i.test(raw))
-                    expectedData = 'phone';
-                else if (input.type === 'date' || /\bdate\b/i.test(raw))
-                    expectedData = 'date';
-                else if (/\b(code|otp|token)\b/i.test(raw))
-                    expectedData = 'one_time_code';
-                else if (/\b(search|query)\b/i.test(raw))
-                    expectedData = 'search_term';
-            }
-            out.push({ role, label: raw, selectorHint: tag, expectedData });
-        });
-        // Prioritise: form fields > buttons > links; within each group keep order
-        const priority = (e) => {
-            if (['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role))
-                return 0;
-            if (e.role === 'button' || e.role === 'submit')
-                return 1;
-            return 2;
-        };
-        out.sort((a, b) => priority(a) - priority(b));
-        return out.slice(0, 25); // cap at 25 — enough context, not overwhelming
+async function readScreen(page) {
+  return page.evaluate(() => {
+    const out = [];
+    const seen = /* @__PURE__ */ new Set();
+    const noiseLabel = /^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|got it|dismiss|skip to|powered by|copied|\s*$)/i;
+    const nodes = document.querySelectorAll(
+      'input:not([type=hidden]), select, textarea, button, a[href], [role="button"], [role="combobox"], [role="slider"], [role="switch"], [role="tab"], [role="menuitem"], [role="searchbox"], [role="spinbutton"]'
+    );
+    nodes.forEach((el) => {
+      const htmlEl = el;
+      const style = window.getComputedStyle(htmlEl);
+      if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return;
+      const rect = el.getBoundingClientRect();
+      const hasAccessibleLabel = !!(el.getAttribute("aria-label") || el.getAttribute("aria-labelledby"));
+      if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return;
+      const tag = el.tagName.toLowerCase();
+      const role = el.getAttribute("role") || (tag === "input" ? el.type || "text" : tag);
+      const raw = (el.getAttribute("aria-label") || el.getAttribute("placeholder") || (el.getAttribute("title") ?? "") || htmlEl.innerText?.replace(/\s+/g, " ").trim() || el.getAttribute("name") || "").substring(0, 80);
+      if (!raw || noiseLabel.test(raw) || raw.length > 70) return;
+      const key = `${role}:${raw.toLowerCase()}`;
+      if (seen.has(key)) return;
+      seen.add(key);
+      let expectedData;
+      if (tag === "input") {
+        const input = el;
+        if (input.type === "email" || /email/i.test(raw)) expectedData = "valid_email";
+        else if (input.type === "password" || /password/i.test(raw)) expectedData = "password";
+        else if (input.type === "tel" || /phone|mobile/i.test(raw)) expectedData = "phone";
+        else if (input.type === "date" || /\bdate\b/i.test(raw)) expectedData = "date";
+        else if (/\b(code|otp|token)\b/i.test(raw)) expectedData = "one_time_code";
+        else if (/\b(search|query)\b/i.test(raw)) expectedData = "search_term";
+      }
+      out.push({ role, label: raw, selectorHint: tag, expectedData });
     });
+    const priority = (e) => {
+      if (["email", "password", "text", "tel", "date", "number", "select", "textarea"].includes(e.role)) return 0;
+      if (e.role === "button" || e.role === "submit") return 1;
+      return 2;
+    };
+    out.sort((a, b) => priority(a) - priority(b));
+    return out.slice(0, 25);
+  });
 }
+export {
+  readScreen
+};
+//# sourceMappingURL=screen-reader.js.map

package/dist/screenshot-masker.js

@@ -1,80 +1,62 @@
-/**
- * Screenshot masker — blur or black-box PII regions in captured screenshots.
- *
- * Strategy: inject a CSS overlay into the page before screenshot capture that
- * covers detected PII-likely elements (input[type=email], input[type=password],
- * elements with PII-containing text) with a blur or solid black mask.
- *
- * This is a CSS-based approach (no image processing library needed):
- * 1. Identify likely-PII DOM elements via selectors
- * 2. Add an overlaid <div> with blur/black filter at each element's bounding box
- * 3. Take screenshot
- * 4. Remove overlays
- *
- * Falls back gracefully if elements are not found or overlay injection fails.
- */
-// Default PII-sensitive selectors
 const DEFAULT_PII_SELECTORS = [
-    'input[type="email"]',
-    'input[type="password"]',
-    'input[type="tel"]',
-    'input[autocomplete="cc-number"]',
-    'input[autocomplete="cc-csc"]',
-    'input[autocomplete="ssn"]',
-    '[data-pii="true"]',
-    '[data-sensitive="true"]',
+  'input[type="email"]',
+  'input[type="password"]',
+  'input[type="tel"]',
+  'input[autocomplete="cc-number"]',
+  'input[autocomplete="cc-csc"]',
+  'input[autocomplete="ssn"]',
+  '[data-pii="true"]',
+  '[data-sensitive="true"]'
 ];
-export async function maskScreenshot(page, opts = {}) {
-    const { extraSelectors = [], style = 'blur', blurPx = 8 } = opts;
-    const selectors = [...DEFAULT_PII_SELECTORS, ...extraSelectors];
-    let maskedCount = 0;
-    const overlayIds = [];
-    // Inject CSS overlays
-    try {
-        const result = await page.evaluate(({ selectors, style, blurPx }) => {
-            const ids = [];
-            for (const sel of selectors) {
-                try {
-                    const elements = Array.from(document.querySelectorAll(sel));
-                    for (const el of elements) {
-                        const rect = el.getBoundingClientRect();
-                        if (rect.width === 0 || rect.height === 0)
-                            continue;
-                        const overlay = document.createElement('div');
-                        const id = `__zeta_mask_${Math.random().toString(36).slice(2)}`;
-                        overlay.id = id;
-                        overlay.style.cssText = [
-                            'position: fixed',
-                            `top: ${rect.top + window.scrollY}px`,
-                            `left: ${rect.left + window.scrollX}px`,
-                            `width: ${rect.width}px`,
-                            `height: ${rect.height}px`,
-                            'z-index: 2147483647',
-                            style === 'blur'
-                                ? `backdrop-filter: blur(${blurPx}px); -webkit-backdrop-filter: blur(${blurPx}px); background: rgba(0,0,0,0.1);`
-                                : 'background: #000000;',
-                            'pointer-events: none',
-                        ].join('; ');
-                        document.body.appendChild(overlay);
-                        ids.push(id);
-                    }
-                }
-                catch { /* selector error */ }
-            }
-            return ids;
-        }, { selectors, style, blurPx });
-        overlayIds.push(...result);
-        maskedCount = result.length;
-    }
-    catch { /* non-fatal */ }
-    // Take screenshot with overlays active
-    const screenshot = await page.screenshot({ type: 'png', fullPage: false });
-    // Remove overlays
-    if (overlayIds.length > 0) {
-        await page.evaluate((ids) => {
-            for (const id of ids)
-                document.getElementById(id)?.remove();
-        }, overlayIds).catch(() => { });
-    }
-    return { maskedCount, screenshot };
+async function maskScreenshot(page, opts = {}) {
+  const { extraSelectors = [], style = "blur", blurPx = 8 } = opts;
+  const selectors = [...DEFAULT_PII_SELECTORS, ...extraSelectors];
+  let maskedCount = 0;
+  const overlayIds = [];
+  try {
+    const result = await page.evaluate(({ selectors: selectors2, style: style2, blurPx: blurPx2 }) => {
+      const ids = [];
+      for (const sel of selectors2) {
+        try {
+          const elements = Array.from(document.querySelectorAll(sel));
+          for (const el of elements) {
+            const rect = el.getBoundingClientRect();
+            if (rect.width === 0 || rect.height === 0) continue;
+            const overlay = document.createElement("div");
+            const id = `__zeta_mask_${Math.random().toString(36).slice(2)}`;
+            overlay.id = id;
+            overlay.style.cssText = [
+              "position: fixed",
+              `top: ${rect.top + window.scrollY}px`,
+              `left: ${rect.left + window.scrollX}px`,
+              `width: ${rect.width}px`,
+              `height: ${rect.height}px`,
+              "z-index: 2147483647",
+              style2 === "blur" ? `backdrop-filter: blur(${blurPx2}px); -webkit-backdrop-filter: blur(${blurPx2}px); background: rgba(0,0,0,0.1);` : "background: #000000;",
+              "pointer-events: none"
+            ].join("; ");
+            document.body.appendChild(overlay);
+            ids.push(id);
+          }
+        } catch {
+        }
+      }
+      return ids;
+    }, { selectors, style, blurPx });
+    overlayIds.push(...result);
+    maskedCount = result.length;
+  } catch {
+  }
+  const screenshot = await page.screenshot({ type: "png", fullPage: false });
+  if (overlayIds.length > 0) {
+    await page.evaluate((ids) => {
+      for (const id of ids) document.getElementById(id)?.remove();
+    }, overlayIds).catch(() => {
+    });
+  }
+  return { maskedCount, screenshot };
 }
+export {
+  maskScreenshot
+};
+//# sourceMappingURL=screenshot-masker.js.map

package/dist/seo-checker.js

@@ -1,110 +1,91 @@
-/**
- * SEO indexability checker — determines if a page is crawlable and indexable.
- *
- * Checks:
- * 1. X-Robots-Tag response header (noindex, nofollow, noarchive)
- * 2. <meta name="robots"> tag (noindex, nofollow)
- * 3. Canonical URL (self-canonical vs cross-canonical)
- * 4. rel="nofollow" on outbound links
- * 5. hreflang tags (multilingual)
- * 6. Open Graph completeness (og:title, og:description, og:image)
- * 7. Structured data presence (JSON-LD)
- * 8. Meta description
- * 9. Title tag length (< 60 chars recommended)
- * 10. URL structure (query params, fragments, excessive depth)
- */
-export async function checkSeo(page, url, responseHeaders = {}) {
-    const issues = [];
-    const checkedAt = new Date().toISOString();
-    let isIndexable = true;
-    // 1. X-Robots-Tag
-    const robotsHeader = responseHeaders['x-robots-tag'] ?? '';
-    if (/noindex/i.test(robotsHeader)) {
-        isIndexable = false;
-        issues.push({ rule: 'x-robots-noindex', severity: 'critical', message: 'X-Robots-Tag: noindex found in response header', value: robotsHeader });
-    }
-    // 2-9. Page-level checks
-    const pageData = await page.evaluate(() => {
-        const meta = (name) => document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ?? '';
-        const og = (prop) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute('content') ?? '';
-        const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href') ?? '';
-        const hreflang = {};
-        document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el) => {
-            hreflang[el.getAttribute('hreflang')] = el.getAttribute('href') ?? '';
-        });
-        const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s) => s.textContent ?? '');
-        return {
-            robotsMeta: meta('robots'),
-            description: meta('description'),
-            title: document.title ?? '',
-            canonical,
-            hreflang,
-            ogTitle: og('title'),
-            ogDescription: og('description'),
-            ogImage: og('image'),
-            hasJsonLd: jsonLd.length > 0,
-            jsonLd,
-        };
-    }).catch(() => ({
-        robotsMeta: '', description: '', title: '', canonical: '', hreflang: {},
-        ogTitle: '', ogDescription: '', ogImage: '', hasJsonLd: false, jsonLd: [],
-    }));
-    // 2. Meta robots
-    if (/noindex/i.test(pageData.robotsMeta)) {
-        isIndexable = false;
-        issues.push({ rule: 'meta-robots-noindex', severity: 'critical', message: 'Meta robots noindex found', value: pageData.robotsMeta });
-    }
-    // 3. Canonical
-    let isSelfCanonical;
-    if (pageData.canonical) {
-        isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, '') === url.replace(/\/$/, '');
-        if (!isSelfCanonical) {
-            issues.push({ rule: 'cross-canonical', severity: 'warning', message: 'Cross-canonical points elsewhere', value: pageData.canonical });
-        }
-    }
-    else {
-        issues.push({ rule: 'missing-canonical', severity: 'warning', message: 'No canonical link tag' });
-    }
-    // 4. Title
-    if (!pageData.title) {
-        issues.push({ rule: 'missing-title', severity: 'critical', message: 'Page has no title tag' });
-    }
-    else if (pageData.title.length > 60) {
-        issues.push({ rule: 'title-too-long', severity: 'warning', message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });
-    }
-    // 5. Meta description
-    if (!pageData.description) {
-        issues.push({ rule: 'missing-description', severity: 'warning', message: 'Missing meta description' });
-    }
-    else if (pageData.description.length > 160) {
-        issues.push({ rule: 'description-too-long', severity: 'info', message: `Description ${pageData.description.length} chars (>160)` });
-    }
-    // 6. OG tags
-    if (!pageData.ogTitle)
-        issues.push({ rule: 'missing-og-title', severity: 'info', message: 'Missing og:title' });
-    if (!pageData.ogImage)
-        issues.push({ rule: 'missing-og-image', severity: 'info', message: 'Missing og:image' });
-    // 7. Structured data
-    if (!pageData.hasJsonLd)
-        issues.push({ rule: 'no-structured-data', severity: 'info', message: 'No JSON-LD structured data found' });
-    // 8. URL depth
-    try {
-        const depth = new URL(url).pathname.split('/').filter(Boolean).length;
-        if (depth > 5)
-            issues.push({ rule: 'deep-url', severity: 'info', message: `URL depth ${depth} (>5 levels)`, value: url });
-    }
-    catch { }
-    const criticals = issues.filter(i => i.severity === 'critical').length;
-    const warnings = issues.filter(i => i.severity === 'warning').length;
-    const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter(i => i.severity === 'info').length * 2);
+async function checkSeo(page, url, responseHeaders = {}) {
+  const issues = [];
+  const checkedAt = (/* @__PURE__ */ new Date()).toISOString();
+  let isIndexable = true;
+  const robotsHeader = responseHeaders["x-robots-tag"] ?? "";
+  if (/noindex/i.test(robotsHeader)) {
+    isIndexable = false;
+    issues.push({ rule: "x-robots-noindex", severity: "critical", message: "X-Robots-Tag: noindex found in response header", value: robotsHeader });
+  }
+  const pageData = await page.evaluate(() => {
+    const meta = (name) => document.querySelector(`meta[name="${name}"]`)?.getAttribute("content") ?? "";
+    const og = (prop) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute("content") ?? "";
+    const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute("href") ?? "";
+    const hreflang = {};
+    document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el) => {
+      hreflang[el.getAttribute("hreflang")] = el.getAttribute("href") ?? "";
+    });
+    const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s) => s.textContent ?? "");
     return {
-        url,
-        isIndexable,
-        canonicalUrl: pageData.canonical || undefined,
-        isSelfCanonical,
-        hreflang: pageData.hreflang,
-        issues,
-        score,
-        checkedAt,
+      robotsMeta: meta("robots"),
+      description: meta("description"),
+      title: document.title ?? "",
+      canonical,
+      hreflang,
+      ogTitle: og("title"),
+      ogDescription: og("description"),
+      ogImage: og("image"),
+      hasJsonLd: jsonLd.length > 0,
+      jsonLd
     };
+  }).catch(() => ({
+    robotsMeta: "",
+    description: "",
+    title: "",
+    canonical: "",
+    hreflang: {},
+    ogTitle: "",
+    ogDescription: "",
+    ogImage: "",
+    hasJsonLd: false,
+    jsonLd: []
+  }));
+  if (/noindex/i.test(pageData.robotsMeta)) {
+    isIndexable = false;
+    issues.push({ rule: "meta-robots-noindex", severity: "critical", message: "Meta robots noindex found", value: pageData.robotsMeta });
+  }
+  let isSelfCanonical;
+  if (pageData.canonical) {
+    isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, "") === url.replace(/\/$/, "");
+    if (!isSelfCanonical) {
+      issues.push({ rule: "cross-canonical", severity: "warning", message: "Cross-canonical points elsewhere", value: pageData.canonical });
+    }
+  } else {
+    issues.push({ rule: "missing-canonical", severity: "warning", message: "No canonical link tag" });
+  }
+  if (!pageData.title) {
+    issues.push({ rule: "missing-title", severity: "critical", message: "Page has no title tag" });
+  } else if (pageData.title.length > 60) {
+    issues.push({ rule: "title-too-long", severity: "warning", message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });
+  }
+  if (!pageData.description) {
+    issues.push({ rule: "missing-description", severity: "warning", message: "Missing meta description" });
+  } else if (pageData.description.length > 160) {
+    issues.push({ rule: "description-too-long", severity: "info", message: `Description ${pageData.description.length} chars (>160)` });
+  }
+  if (!pageData.ogTitle) issues.push({ rule: "missing-og-title", severity: "info", message: "Missing og:title" });
+  if (!pageData.ogImage) issues.push({ rule: "missing-og-image", severity: "info", message: "Missing og:image" });
+  if (!pageData.hasJsonLd) issues.push({ rule: "no-structured-data", severity: "info", message: "No JSON-LD structured data found" });
+  try {
+    const depth = new URL(url).pathname.split("/").filter(Boolean).length;
+    if (depth > 5) issues.push({ rule: "deep-url", severity: "info", message: `URL depth ${depth} (>5 levels)`, value: url });
+  } catch {
+  }
+  const criticals = issues.filter((i) => i.severity === "critical").length;
+  const warnings = issues.filter((i) => i.severity === "warning").length;
+  const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter((i) => i.severity === "info").length * 2);
+  return {
+    url,
+    isIndexable,
+    canonicalUrl: pageData.canonical || void 0,
+    isSelfCanonical,
+    hreflang: pageData.hreflang,
+    issues,
+    score,
+    checkedAt
+  };
 }
+export {
+  checkSeo
+};
+//# sourceMappingURL=seo-checker.js.map