@detiq/crawler 1.1.7 → 1.1.9

package/dist/screen-reader.js

@@ -1,47 +1,72 @@
-async function readScreen(page) {
-  return page.evaluate(() => {
-    const out = [];
-    const seen = /* @__PURE__ */ new Set();
-    const noiseLabel = /^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|got it|dismiss|skip to|powered by|copied|\s*$)/i;
-    const nodes = document.querySelectorAll(
-      'input:not([type=hidden]), select, textarea, button, a[href], [role="button"], [role="combobox"], [role="slider"], [role="switch"], [role="tab"], [role="menuitem"], [role="searchbox"], [role="spinbutton"]'
-    );
-    nodes.forEach((el) => {
-      const htmlEl = el;
-      const style = window.getComputedStyle(htmlEl);
-      if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return;
-      const rect = el.getBoundingClientRect();
-      const hasAccessibleLabel = !!(el.getAttribute("aria-label") || el.getAttribute("aria-labelledby"));
-      if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return;
-      const tag = el.tagName.toLowerCase();
-      const role = el.getAttribute("role") || (tag === "input" ? el.type || "text" : tag);
-      const raw = (el.getAttribute("aria-label") || el.getAttribute("placeholder") || (el.getAttribute("title") ?? "") || htmlEl.innerText?.replace(/\s+/g, " ").trim() || el.getAttribute("name") || "").substring(0, 80);
-      if (!raw || noiseLabel.test(raw) || raw.length > 70) return;
-      const key = `${role}:${raw.toLowerCase()}`;
-      if (seen.has(key)) return;
-      seen.add(key);
-      let expectedData;
-      if (tag === "input") {
-        const input = el;
-        if (input.type === "email" || /email/i.test(raw)) expectedData = "valid_email";
-        else if (input.type === "password" || /password/i.test(raw)) expectedData = "password";
-        else if (input.type === "tel" || /phone|mobile/i.test(raw)) expectedData = "phone";
-        else if (input.type === "date" || /\bdate\b/i.test(raw)) expectedData = "date";
-        else if (/\b(code|otp|token)\b/i.test(raw)) expectedData = "one_time_code";
-        else if (/\b(search|query)\b/i.test(raw)) expectedData = "search_term";
-      }
-      out.push({ role, label: raw, selectorHint: tag, expectedData });
+export async function readScreen(page) {
+    return page.evaluate(() => {
+        const out = [];
+        const seen = new Set();
+        // Noise patterns to skip — cookie banners, footer chrome, version strings
+        // Intentionally excludes standalone "close" and "copy" — too many false positives
+        // (Close account, Close preview, Copy link, Copy to clipboard are all valid test targets)
+        const noiseLabel = /^(cookie|consent|privacy policy|terms of use|copyright|©|\bv\d+\.\d+|got it|dismiss|skip to|powered by|copied|\s*$)/i;
+        // Include ARIA-role elements — Headless UI, Radix, Ant Design, Shadcn all use these instead of native tags
+        const nodes = document.querySelectorAll('input:not([type=hidden]), select, textarea, button, a[href], ' +
+            '[role="button"], [role="combobox"], [role="slider"], [role="switch"], ' +
+            '[role="tab"], [role="menuitem"], [role="searchbox"], [role="spinbutton"]');
+        nodes.forEach((el) => {
+            const htmlEl = el;
+            // Skip invisible elements
+            const style = window.getComputedStyle(htmlEl);
+            if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')
+                return;
+            const rect = el.getBoundingClientRect();
+            // Allow zero-size elements that carry an accessible label (sr-only icon buttons, skip-nav links)
+            const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby'));
+            if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel)
+                return;
+            // Derive role
+            const tag = el.tagName.toLowerCase();
+            const role = el.getAttribute('role') ||
+                (tag === 'input' ? (el.type || 'text') : tag);
+            // Derive the best human-readable label (prefer aria > placeholder > inner text)
+            const raw = (el.getAttribute('aria-label') ||
+                el.getAttribute('placeholder') ||
+                (el.getAttribute('title') ?? '') ||
+                htmlEl.innerText?.replace(/\s+/g, ' ').trim() ||
+                el.getAttribute('name') ||
+                '').substring(0, 80);
+            // Skip empty, noise patterns, or very long labels (likely container text)
+            if (!raw || noiseLabel.test(raw) || raw.length > 70)
+                return;
+            // Deduplicate by role+normalised label
+            const key = `${role}:${raw.toLowerCase()}`;
+            if (seen.has(key))
+                return;
+            seen.add(key);
+            let expectedData;
+            if (tag === 'input') {
+                const input = el;
+                if (input.type === 'email' || /email/i.test(raw))
+                    expectedData = 'valid_email';
+                else if (input.type === 'password' || /password/i.test(raw))
+                    expectedData = 'password';
+                else if (input.type === 'tel' || /phone|mobile/i.test(raw))
+                    expectedData = 'phone';
+                else if (input.type === 'date' || /\bdate\b/i.test(raw))
+                    expectedData = 'date';
+                else if (/\b(code|otp|token)\b/i.test(raw))
+                    expectedData = 'one_time_code';
+                else if (/\b(search|query)\b/i.test(raw))
+                    expectedData = 'search_term';
+            }
+            out.push({ role, label: raw, selectorHint: tag, expectedData });
+        });
+        // Prioritise: form fields > buttons > links; within each group keep order
+        const priority = (e) => {
+            if (['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea'].includes(e.role))
+                return 0;
+            if (e.role === 'button' || e.role === 'submit')
+                return 1;
+            return 2;
+        };
+        out.sort((a, b) => priority(a) - priority(b));
+        return out.slice(0, 25); // cap at 25 — enough context, not overwhelming
     });
-    const priority = (e) => {
-      if (["email", "password", "text", "tel", "date", "number", "select", "textarea"].includes(e.role)) return 0;
-      if (e.role === "button" || e.role === "submit") return 1;
-      return 2;
-    };
-    out.sort((a, b) => priority(a) - priority(b));
-    return out.slice(0, 25);
-  });
 }
-export {
-  readScreen
-};
-//# sourceMappingURL=screen-reader.js.map

package/dist/screenshot-masker.js

@@ -1,62 +1,80 @@
+/**
+ * Screenshot masker — blur or black-box PII regions in captured screenshots.
+ *
+ * Strategy: inject a CSS overlay into the page before screenshot capture that
+ * covers detected PII-likely elements (input[type=email], input[type=password],
+ * elements with PII-containing text) with a blur or solid black mask.
+ *
+ * This is a CSS-based approach (no image processing library needed):
+ * 1. Identify likely-PII DOM elements via selectors
+ * 2. Add an overlaid <div> with blur/black filter at each element's bounding box
+ * 3. Take screenshot
+ * 4. Remove overlays
+ *
+ * Falls back gracefully if elements are not found or overlay injection fails.
+ */
+// Default PII-sensitive selectors
 const DEFAULT_PII_SELECTORS = [
-  'input[type="email"]',
-  'input[type="password"]',
-  'input[type="tel"]',
-  'input[autocomplete="cc-number"]',
-  'input[autocomplete="cc-csc"]',
-  'input[autocomplete="ssn"]',
-  '[data-pii="true"]',
-  '[data-sensitive="true"]'
+    'input[type="email"]',
+    'input[type="password"]',
+    'input[type="tel"]',
+    'input[autocomplete="cc-number"]',
+    'input[autocomplete="cc-csc"]',
+    'input[autocomplete="ssn"]',
+    '[data-pii="true"]',
+    '[data-sensitive="true"]',
 ];
-async function maskScreenshot(page, opts = {}) {
-  const { extraSelectors = [], style = "blur", blurPx = 8 } = opts;
-  const selectors = [...DEFAULT_PII_SELECTORS, ...extraSelectors];
-  let maskedCount = 0;
-  const overlayIds = [];
-  try {
-    const result = await page.evaluate(({ selectors: selectors2, style: style2, blurPx: blurPx2 }) => {
-      const ids = [];
-      for (const sel of selectors2) {
-        try {
-          const elements = Array.from(document.querySelectorAll(sel));
-          for (const el of elements) {
-            const rect = el.getBoundingClientRect();
-            if (rect.width === 0 || rect.height === 0) continue;
-            const overlay = document.createElement("div");
-            const id = `__zeta_mask_${Math.random().toString(36).slice(2)}`;
-            overlay.id = id;
-            overlay.style.cssText = [
-              "position: fixed",
-              `top: ${rect.top + window.scrollY}px`,
-              `left: ${rect.left + window.scrollX}px`,
-              `width: ${rect.width}px`,
-              `height: ${rect.height}px`,
-              "z-index: 2147483647",
-              style2 === "blur" ? `backdrop-filter: blur(${blurPx2}px); -webkit-backdrop-filter: blur(${blurPx2}px); background: rgba(0,0,0,0.1);` : "background: #000000;",
-              "pointer-events: none"
-            ].join("; ");
-            document.body.appendChild(overlay);
-            ids.push(id);
-          }
-        } catch {
-        }
-      }
-      return ids;
-    }, { selectors, style, blurPx });
-    overlayIds.push(...result);
-    maskedCount = result.length;
-  } catch {
-  }
-  const screenshot = await page.screenshot({ type: "png", fullPage: false });
-  if (overlayIds.length > 0) {
-    await page.evaluate((ids) => {
-      for (const id of ids) document.getElementById(id)?.remove();
-    }, overlayIds).catch(() => {
-    });
-  }
-  return { maskedCount, screenshot };
+export async function maskScreenshot(page, opts = {}) {
+    const { extraSelectors = [], style = 'blur', blurPx = 8 } = opts;
+    const selectors = [...DEFAULT_PII_SELECTORS, ...extraSelectors];
+    let maskedCount = 0;
+    const overlayIds = [];
+    // Inject CSS overlays
+    try {
+        const result = await page.evaluate(({ selectors, style, blurPx }) => {
+            const ids = [];
+            for (const sel of selectors) {
+                try {
+                    const elements = Array.from(document.querySelectorAll(sel));
+                    for (const el of elements) {
+                        const rect = el.getBoundingClientRect();
+                        if (rect.width === 0 || rect.height === 0)
+                            continue;
+                        const overlay = document.createElement('div');
+                        const id = `__zeta_mask_${Math.random().toString(36).slice(2)}`;
+                        overlay.id = id;
+                        overlay.style.cssText = [
+                            'position: fixed',
+                            `top: ${rect.top + window.scrollY}px`,
+                            `left: ${rect.left + window.scrollX}px`,
+                            `width: ${rect.width}px`,
+                            `height: ${rect.height}px`,
+                            'z-index: 2147483647',
+                            style === 'blur'
+                                ? `backdrop-filter: blur(${blurPx}px); -webkit-backdrop-filter: blur(${blurPx}px); background: rgba(0,0,0,0.1);`
+                                : 'background: #000000;',
+                            'pointer-events: none',
+                        ].join('; ');
+                        document.body.appendChild(overlay);
+                        ids.push(id);
+                    }
+                }
+                catch { /* selector error */ }
+            }
+            return ids;
+        }, { selectors, style, blurPx });
+        overlayIds.push(...result);
+        maskedCount = result.length;
+    }
+    catch { /* non-fatal */ }
+    // Take screenshot with overlays active
+    const screenshot = await page.screenshot({ type: 'png', fullPage: false });
+    // Remove overlays
+    if (overlayIds.length > 0) {
+        await page.evaluate((ids) => {
+            for (const id of ids)
+                document.getElementById(id)?.remove();
+        }, overlayIds).catch(() => { });
+    }
+    return { maskedCount, screenshot };
 }
-export {
-  maskScreenshot
-};
-//# sourceMappingURL=screenshot-masker.js.map

package/dist/seo-checker.js

@@ -1,91 +1,110 @@
-async function checkSeo(page, url, responseHeaders = {}) {
-  const issues = [];
-  const checkedAt = (/* @__PURE__ */ new Date()).toISOString();
-  let isIndexable = true;
-  const robotsHeader = responseHeaders["x-robots-tag"] ?? "";
-  if (/noindex/i.test(robotsHeader)) {
-    isIndexable = false;
-    issues.push({ rule: "x-robots-noindex", severity: "critical", message: "X-Robots-Tag: noindex found in response header", value: robotsHeader });
-  }
-  const pageData = await page.evaluate(() => {
-    const meta = (name) => document.querySelector(`meta[name="${name}"]`)?.getAttribute("content") ?? "";
-    const og = (prop) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute("content") ?? "";
-    const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute("href") ?? "";
-    const hreflang = {};
-    document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el) => {
-      hreflang[el.getAttribute("hreflang")] = el.getAttribute("href") ?? "";
-    });
-    const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s) => s.textContent ?? "");
+/**
+ * SEO indexability checker — determines if a page is crawlable and indexable.
+ *
+ * Checks:
+ * 1. X-Robots-Tag response header (noindex, nofollow, noarchive)
+ * 2. <meta name="robots"> tag (noindex, nofollow)
+ * 3. Canonical URL (self-canonical vs cross-canonical)
+ * 4. rel="nofollow" on outbound links
+ * 5. hreflang tags (multilingual)
+ * 6. Open Graph completeness (og:title, og:description, og:image)
+ * 7. Structured data presence (JSON-LD)
+ * 8. Meta description
+ * 9. Title tag length (< 60 chars recommended)
+ * 10. URL structure (query params, fragments, excessive depth)
+ */
+export async function checkSeo(page, url, responseHeaders = {}) {
+    const issues = [];
+    const checkedAt = new Date().toISOString();
+    let isIndexable = true;
+    // 1. X-Robots-Tag
+    const robotsHeader = responseHeaders['x-robots-tag'] ?? '';
+    if (/noindex/i.test(robotsHeader)) {
+        isIndexable = false;
+        issues.push({ rule: 'x-robots-noindex', severity: 'critical', message: 'X-Robots-Tag: noindex found in response header', value: robotsHeader });
+    }
+    // 2-9. Page-level checks
+    const pageData = await page.evaluate(() => {
+        const meta = (name) => document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ?? '';
+        const og = (prop) => document.querySelector(`meta[property="og:${prop}"]`)?.getAttribute('content') ?? '';
+        const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href') ?? '';
+        const hreflang = {};
+        document.querySelectorAll('link[rel="alternate"][hreflang]').forEach((el) => {
+            hreflang[el.getAttribute('hreflang')] = el.getAttribute('href') ?? '';
+        });
+        const jsonLd = Array.from(document.querySelectorAll('script[type="application/ld+json"]')).map((s) => s.textContent ?? '');
+        return {
+            robotsMeta: meta('robots'),
+            description: meta('description'),
+            title: document.title ?? '',
+            canonical,
+            hreflang,
+            ogTitle: og('title'),
+            ogDescription: og('description'),
+            ogImage: og('image'),
+            hasJsonLd: jsonLd.length > 0,
+            jsonLd,
+        };
+    }).catch(() => ({
+        robotsMeta: '', description: '', title: '', canonical: '', hreflang: {},
+        ogTitle: '', ogDescription: '', ogImage: '', hasJsonLd: false, jsonLd: [],
+    }));
+    // 2. Meta robots
+    if (/noindex/i.test(pageData.robotsMeta)) {
+        isIndexable = false;
+        issues.push({ rule: 'meta-robots-noindex', severity: 'critical', message: 'Meta robots noindex found', value: pageData.robotsMeta });
+    }
+    // 3. Canonical
+    let isSelfCanonical;
+    if (pageData.canonical) {
+        isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, '') === url.replace(/\/$/, '');
+        if (!isSelfCanonical) {
+            issues.push({ rule: 'cross-canonical', severity: 'warning', message: 'Cross-canonical points elsewhere', value: pageData.canonical });
+        }
+    }
+    else {
+        issues.push({ rule: 'missing-canonical', severity: 'warning', message: 'No canonical link tag' });
+    }
+    // 4. Title
+    if (!pageData.title) {
+        issues.push({ rule: 'missing-title', severity: 'critical', message: 'Page has no title tag' });
+    }
+    else if (pageData.title.length > 60) {
+        issues.push({ rule: 'title-too-long', severity: 'warning', message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });
+    }
+    // 5. Meta description
+    if (!pageData.description) {
+        issues.push({ rule: 'missing-description', severity: 'warning', message: 'Missing meta description' });
+    }
+    else if (pageData.description.length > 160) {
+        issues.push({ rule: 'description-too-long', severity: 'info', message: `Description ${pageData.description.length} chars (>160)` });
+    }
+    // 6. OG tags
+    if (!pageData.ogTitle)
+        issues.push({ rule: 'missing-og-title', severity: 'info', message: 'Missing og:title' });
+    if (!pageData.ogImage)
+        issues.push({ rule: 'missing-og-image', severity: 'info', message: 'Missing og:image' });
+    // 7. Structured data
+    if (!pageData.hasJsonLd)
+        issues.push({ rule: 'no-structured-data', severity: 'info', message: 'No JSON-LD structured data found' });
+    // 8. URL depth
+    try {
+        const depth = new URL(url).pathname.split('/').filter(Boolean).length;
+        if (depth > 5)
+            issues.push({ rule: 'deep-url', severity: 'info', message: `URL depth ${depth} (>5 levels)`, value: url });
+    }
+    catch { }
+    const criticals = issues.filter(i => i.severity === 'critical').length;
+    const warnings = issues.filter(i => i.severity === 'warning').length;
+    const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter(i => i.severity === 'info').length * 2);
     return {
-      robotsMeta: meta("robots"),
-      description: meta("description"),
-      title: document.title ?? "",
-      canonical,
-      hreflang,
-      ogTitle: og("title"),
-      ogDescription: og("description"),
-      ogImage: og("image"),
-      hasJsonLd: jsonLd.length > 0,
-      jsonLd
+        url,
+        isIndexable,
+        canonicalUrl: pageData.canonical || undefined,
+        isSelfCanonical,
+        hreflang: pageData.hreflang,
+        issues,
+        score,
+        checkedAt,
     };
-  }).catch(() => ({
-    robotsMeta: "",
-    description: "",
-    title: "",
-    canonical: "",
-    hreflang: {},
-    ogTitle: "",
-    ogDescription: "",
-    ogImage: "",
-    hasJsonLd: false,
-    jsonLd: []
-  }));
-  if (/noindex/i.test(pageData.robotsMeta)) {
-    isIndexable = false;
-    issues.push({ rule: "meta-robots-noindex", severity: "critical", message: "Meta robots noindex found", value: pageData.robotsMeta });
-  }
-  let isSelfCanonical;
-  if (pageData.canonical) {
-    isSelfCanonical = pageData.canonical === url || pageData.canonical.replace(/\/$/, "") === url.replace(/\/$/, "");
-    if (!isSelfCanonical) {
-      issues.push({ rule: "cross-canonical", severity: "warning", message: "Cross-canonical points elsewhere", value: pageData.canonical });
-    }
-  } else {
-    issues.push({ rule: "missing-canonical", severity: "warning", message: "No canonical link tag" });
-  }
-  if (!pageData.title) {
-    issues.push({ rule: "missing-title", severity: "critical", message: "Page has no title tag" });
-  } else if (pageData.title.length > 60) {
-    issues.push({ rule: "title-too-long", severity: "warning", message: `Title ${pageData.title.length} chars (>60)`, value: pageData.title });
-  }
-  if (!pageData.description) {
-    issues.push({ rule: "missing-description", severity: "warning", message: "Missing meta description" });
-  } else if (pageData.description.length > 160) {
-    issues.push({ rule: "description-too-long", severity: "info", message: `Description ${pageData.description.length} chars (>160)` });
-  }
-  if (!pageData.ogTitle) issues.push({ rule: "missing-og-title", severity: "info", message: "Missing og:title" });
-  if (!pageData.ogImage) issues.push({ rule: "missing-og-image", severity: "info", message: "Missing og:image" });
-  if (!pageData.hasJsonLd) issues.push({ rule: "no-structured-data", severity: "info", message: "No JSON-LD structured data found" });
-  try {
-    const depth = new URL(url).pathname.split("/").filter(Boolean).length;
-    if (depth > 5) issues.push({ rule: "deep-url", severity: "info", message: `URL depth ${depth} (>5 levels)`, value: url });
-  } catch {
-  }
-  const criticals = issues.filter((i) => i.severity === "critical").length;
-  const warnings = issues.filter((i) => i.severity === "warning").length;
-  const score = Math.max(0, 100 - criticals * 20 - warnings * 5 - issues.filter((i) => i.severity === "info").length * 2);
-  return {
-    url,
-    isIndexable,
-    canonicalUrl: pageData.canonical || void 0,
-    isSelfCanonical,
-    hreflang: pageData.hreflang,
-    issues,
-    score,
-    checkedAt
-  };
 }
-export {
-  checkSeo
-};
-//# sourceMappingURL=seo-checker.js.map