@detiq/crawler 1.1.9 → 1.1.11

package/dist/custom-js-injector.js

@@ -1,66 +1,50 @@
-/**
- * Custom JS injector — run user-provided JavaScript in the context of each crawled page.
- *
- * Use cases:
- * - Extract custom data points not captured by standard extractors
- * - Set localStorage/sessionStorage values before crawl
- * - Override fetch() to intercept API calls
- * - Simulate user events (keydown, focus, etc.)
- * - Bypass specific anti-bot checks
- *
- * Security: scripts run in the page context (sandboxed by Chromium's renderer).
- * They cannot access the Node.js process or file system.
- * ZeTa validates scripts are strings before injection.
- */
-export async function injectCustomJs(page, opts) {
-    const result = {
-        initScriptApplied: false,
-        postLoadResult: null,
-        storageSet: false,
-        cookiesSet: 0,
-    };
-    // Set cookies via Playwright context
-    if (opts.cookies?.length) {
-        try {
-            await page.context().addCookies(opts.cookies.map(c => ({ name: c.name, value: c.value, domain: c.domain ?? '', path: c.path ?? '/', url: page.url() || undefined })));
-            result.cookiesSet = opts.cookies.length;
-        }
-        catch (e) {
-            result.error = `Cookie injection failed: ${e.message}`;
-        }
+async function injectCustomJs(page, opts) {
+  const result = {
+    initScriptApplied: false,
+    postLoadResult: null,
+    storageSet: false,
+    cookiesSet: 0
+  };
+  if (opts.cookies?.length) {
+    try {
+      await page.context().addCookies(
+        opts.cookies.map((c) => ({ name: c.name, value: c.value, domain: c.domain ?? "", path: c.path ?? "/", url: page.url() || void 0 }))
+      );
+      result.cookiesSet = opts.cookies.length;
+    } catch (e) {
+      result.error = `Cookie injection failed: ${e.message}`;
     }
-    // Init script (runs before page load on every navigation)
-    if (opts.initScript || opts.localStorage || opts.sessionStorage) {
-        const storageSetup = [
-            opts.localStorage ? `Object.entries(${JSON.stringify(opts.localStorage)}).forEach(([k,v])=>localStorage.setItem(k,v));` : '',
-            opts.sessionStorage ? `Object.entries(${JSON.stringify(opts.sessionStorage)}).forEach(([k,v])=>sessionStorage.setItem(k,v));` : '',
-        ].filter(Boolean).join('\n');
-        const combinedInit = [storageSetup, opts.initScript ?? ''].filter(Boolean).join('\n');
-        try {
-            await page.addInitScript(combinedInit);
-            result.initScriptApplied = true;
-            result.storageSet = !!(opts.localStorage || opts.sessionStorage);
-        }
-        catch (e) {
-            result.error = `Init script failed: ${e.message}`;
-        }
+  }
+  if (opts.initScript || opts.localStorage || opts.sessionStorage) {
+    const storageSetup = [
+      opts.localStorage ? `Object.entries(${JSON.stringify(opts.localStorage)}).forEach(([k,v])=>localStorage.setItem(k,v));` : "",
+      opts.sessionStorage ? `Object.entries(${JSON.stringify(opts.sessionStorage)}).forEach(([k,v])=>sessionStorage.setItem(k,v));` : ""
+    ].filter(Boolean).join("\n");
+    const combinedInit = [storageSetup, opts.initScript ?? ""].filter(Boolean).join("\n");
+    try {
+      await page.addInitScript(combinedInit);
+      result.initScriptApplied = true;
+      result.storageSet = !!(opts.localStorage || opts.sessionStorage);
+    } catch (e) {
+      result.error = `Init script failed: ${e.message}`;
     }
-    // Post-load script (runs after navigation)
-    if (opts.postLoadScript) {
-        try {
-            result.postLoadResult = await page.evaluate(opts.postLoadScript);
-        }
-        catch (e) {
-            result.error = `Post-load script failed: ${e.message}`;
-        }
+  }
+  if (opts.postLoadScript) {
+    try {
+      result.postLoadResult = await page.evaluate(opts.postLoadScript);
+    } catch (e) {
+      result.error = `Post-load script failed: ${e.message}`;
     }
-    return result;
+  }
+  return result;
 }
-/** Safe string validator — ensures script is a string, not an object/function reference */
-export function validateScript(script) {
-    if (typeof script !== 'string')
-        throw new Error('Script must be a string');
-    if (script.length > 50_000)
-        throw new Error('Script too large (>50KB)');
-    return script;
+function validateScript(script) {
+  if (typeof script !== "string") throw new Error("Script must be a string");
+  if (script.length > 5e4) throw new Error("Script too large (>50KB)");
+  return script;
 }
+export {
+  injectCustomJs,
+  validateScript
+};
+//# sourceMappingURL=custom-js-injector.js.map

package/dist/dark-mode-capture.js

@@ -1,89 +1,75 @@
-/**
- * Dark mode capture — screenshot pages in both light and dark color schemes.
- *
- * Uses Playwright's page.emulateMedia({ colorScheme: 'dark' }) to force dark mode,
- * then captures a screenshot. Compares with light mode to determine if dark mode
- * is supported (significant visual difference).
- *
- * Also detects prefers-color-scheme media query support in CSS.
- */
 async function getBackgroundColor(page) {
-    try {
-        return await page.evaluate(() => {
-            const body = document.body;
-            return window.getComputedStyle(body).backgroundColor;
-        });
-    }
-    catch {
-        return 'unknown';
-    }
+  try {
+    return await page.evaluate(() => {
+      const body = document.body;
+      return window.getComputedStyle(body).backgroundColor;
+    });
+  } catch {
+    return "unknown";
+  }
 }
 function colorsAreDifferent(a, b) {
-    if (a === b)
-        return false;
-    if (a === 'unknown' || b === 'unknown')
-        return false;
-    // Parse rgb(r, g, b) and compare luminance
-    const parse = (c) => {
-        const m = c.match(/\d+/g);
-        return m ? m.map(Number) : [0, 0, 0];
-    };
-    const [ar, ag, ab] = parse(a);
-    const [br, bg, bb] = parse(b);
-    const lumA = 0.299 * ar + 0.587 * ag + 0.114 * ab;
-    const lumB = 0.299 * br + 0.587 * bg + 0.114 * bb;
-    return Math.abs(lumA - lumB) > 30; // significant luminance difference
+  if (a === b) return false;
+  if (a === "unknown" || b === "unknown") return false;
+  const parse = (c) => {
+    const m = c.match(/\d+/g);
+    return m ? m.map(Number) : [0, 0, 0];
+  };
+  const [ar, ag, ab] = parse(a);
+  const [br, bg, bb] = parse(b);
+  const lumA = 0.299 * ar + 0.587 * ag + 0.114 * ab;
+  const lumB = 0.299 * br + 0.587 * bg + 0.114 * bb;
+  return Math.abs(lumA - lumB) > 30;
 }
 async function hasCssDarkModeSupport(page) {
-    try {
-        return await page.evaluate(() => {
-            const sheets = Array.from(document.styleSheets);
-            for (const sheet of sheets) {
-                try {
-                    const rules = Array.from(sheet.cssRules ?? []);
-                    for (const rule of rules) {
-                        if (rule instanceof CSSMediaRule && rule.conditionText?.includes('prefers-color-scheme')) {
-                            return true;
-                        }
-                    }
-                }
-                catch { /* cross-origin stylesheet — skip */ }
+  try {
+    return await page.evaluate(() => {
+      const sheets = Array.from(document.styleSheets);
+      for (const sheet of sheets) {
+        try {
+          const rules = Array.from(sheet.cssRules ?? []);
+          for (const rule of rules) {
+            if (rule instanceof CSSMediaRule && rule.conditionText?.includes("prefers-color-scheme")) {
+              return true;
             }
-            return false;
-        });
-    }
-    catch {
-        return false;
-    }
+          }
+        } catch {
+        }
+      }
+      return false;
+    });
+  } catch {
+    return false;
+  }
 }
-export async function captureDarkMode(page, url, opts = {}) {
-    const capturedAt = new Date().toISOString();
-    const { takeScreenshots = false } = opts;
-    // Light mode
-    await page.emulateMedia({ colorScheme: 'light' }).catch(() => { });
-    const backgroundColorLight = await getBackgroundColor(page);
-    const hasCssDark = await hasCssDarkModeSupport(page);
-    const lightScreenshot = takeScreenshots
-        ? await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined)
-        : undefined;
-    // Dark mode
-    await page.emulateMedia({ colorScheme: 'dark' }).catch(() => { });
-    await page.waitForTimeout(300); // allow CSS transitions
-    const backgroundColorDark = await getBackgroundColor(page);
-    const darkScreenshot = takeScreenshots
-        ? await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined)
-        : undefined;
-    // Restore light mode
-    await page.emulateMedia({ colorScheme: 'light' }).catch(() => { });
-    const hasDarkMode = colorsAreDifferent(backgroundColorLight, backgroundColorDark) || hasCssDark;
-    return {
-        url,
-        lightScreenshot,
-        darkScreenshot,
-        hasDarkMode,
-        hasCssDarkMode: hasCssDark,
-        backgroundColorLight,
-        backgroundColorDark,
-        capturedAt,
-    };
+async function captureDarkMode(page, url, opts = {}) {
+  const capturedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const { takeScreenshots = false } = opts;
+  await page.emulateMedia({ colorScheme: "light" }).catch(() => {
+  });
+  const backgroundColorLight = await getBackgroundColor(page);
+  const hasCssDark = await hasCssDarkModeSupport(page);
+  const lightScreenshot = takeScreenshots ? await page.screenshot({ type: "jpeg", quality: 60 }).catch(() => void 0) : void 0;
+  await page.emulateMedia({ colorScheme: "dark" }).catch(() => {
+  });
+  await page.waitForTimeout(300);
+  const backgroundColorDark = await getBackgroundColor(page);
+  const darkScreenshot = takeScreenshots ? await page.screenshot({ type: "jpeg", quality: 60 }).catch(() => void 0) : void 0;
+  await page.emulateMedia({ colorScheme: "light" }).catch(() => {
+  });
+  const hasDarkMode = colorsAreDifferent(backgroundColorLight, backgroundColorDark) || hasCssDark;
+  return {
+    url,
+    lightScreenshot,
+    darkScreenshot,
+    hasDarkMode,
+    hasCssDarkMode: hasCssDark,
+    backgroundColorLight,
+    backgroundColorDark,
+    capturedAt
+  };
 }
+export {
+  captureDarkMode
+};
+//# sourceMappingURL=dark-mode-capture.js.map

package/dist/dead-link-detector.js

@@ -1,58 +1,50 @@
-/**
- * Dead link detector — tracks broken links discovered during crawl.
- *
- * A "dead link" is a URL that:
- * - Returns 404, 410, or other 4xx/5xx responses
- * - Times out after maxRetries
- * - Redirects to an error page (detected by title heuristic)
- *
- * Results: list of broken links with source page, status code, and anchor text.
- * Used for: SEO audit export, test report enrichment.
- */
-export class DeadLinkTracker {
-    deadLinks = [];
-    checked = new Map(); // url → statusCode
-    record(url, statusCode, source) {
-        if (this.checked.has(url))
-            return;
-        this.checked.set(url, statusCode);
-        if (statusCode >= 400) {
-            this.deadLinks.push({
-                url,
-                statusCode,
-                sourceUrl: source?.sourceUrl,
-                anchorText: source?.anchorText,
-                checkedAt: new Date().toISOString(),
-                reason: statusCode === 0 ? 'timeout' : 'http-error',
-            });
-        }
+class DeadLinkTracker {
+  deadLinks = [];
+  checked = /* @__PURE__ */ new Map();
+  // url → statusCode
+  record(url, statusCode, source) {
+    if (this.checked.has(url)) return;
+    this.checked.set(url, statusCode);
+    if (statusCode >= 400) {
+      this.deadLinks.push({
+        url,
+        statusCode,
+        sourceUrl: source?.sourceUrl,
+        anchorText: source?.anchorText,
+        checkedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        reason: statusCode === 0 ? "timeout" : "http-error"
+      });
     }
-    recordTimeout(url, sourceUrl) {
-        if (this.checked.has(url))
-            return;
-        this.checked.set(url, 0);
-        this.deadLinks.push({ url, statusCode: 0, sourceUrl, checkedAt: new Date().toISOString(), reason: 'timeout' });
-    }
-    getDeadLinks() {
-        return [...this.deadLinks];
-    }
-    isKnownBroken(url) {
-        const code = this.checked.get(url);
-        return code !== undefined && code >= 400;
-    }
-    stats() {
-        return {
-            total: this.deadLinks.length,
-            by404: this.deadLinks.filter(l => l.statusCode === 404).length,
-            by5xx: this.deadLinks.filter(l => l.statusCode >= 500).length,
-            byTimeout: this.deadLinks.filter(l => l.reason === 'timeout').length,
-        };
-    }
-    toCsv() {
-        const rows = [['URL', 'Status', 'Source URL', 'Anchor Text', 'Checked At']];
-        for (const l of this.deadLinks) {
-            rows.push([l.url, String(l.statusCode), l.sourceUrl ?? '', l.anchorText ?? '', l.checkedAt]);
-        }
-        return rows.map(r => r.map(c => `"${c.replace(/"/g, '""')}"`).join(',')).join('\n');
+  }
+  recordTimeout(url, sourceUrl) {
+    if (this.checked.has(url)) return;
+    this.checked.set(url, 0);
+    this.deadLinks.push({ url, statusCode: 0, sourceUrl, checkedAt: (/* @__PURE__ */ new Date()).toISOString(), reason: "timeout" });
+  }
+  getDeadLinks() {
+    return [...this.deadLinks];
+  }
+  isKnownBroken(url) {
+    const code = this.checked.get(url);
+    return code !== void 0 && code >= 400;
+  }
+  stats() {
+    return {
+      total: this.deadLinks.length,
+      by404: this.deadLinks.filter((l) => l.statusCode === 404).length,
+      by5xx: this.deadLinks.filter((l) => l.statusCode >= 500).length,
+      byTimeout: this.deadLinks.filter((l) => l.reason === "timeout").length
+    };
+  }
+  toCsv() {
+    const rows = [["URL", "Status", "Source URL", "Anchor Text", "Checked At"]];
+    for (const l of this.deadLinks) {
+      rows.push([l.url, String(l.statusCode), l.sourceUrl ?? "", l.anchorText ?? "", l.checkedAt]);
     }
+    return rows.map((r) => r.map((c) => `"${c.replace(/"/g, '""')}"`).join(",")).join("\n");
+  }
 }
+export {
+  DeadLinkTracker
+};
+//# sourceMappingURL=dead-link-detector.js.map

package/dist/element-hash.js

@@ -1,13 +1,9 @@
-import { createHash } from 'node:crypto';
-/**
- * Compute a stable fingerprint of a screen's element set.
- * Sorted so insertion-order changes (React re-renders) don't produce false positives.
- * Returns first 16 hex chars of SHA-256.
- */
-export function computeElementHash(elements) {
-    const pairs = elements
-        .map((e) => `${e.role.toLowerCase()}:${e.meaning.toLowerCase().trim()}`)
-        .sort()
-        .join('|');
-    return createHash('sha256').update(pairs).digest('hex').slice(0, 16);
+import { createHash } from "node:crypto";
+function computeElementHash(elements) {
+  const pairs = elements.map((e) => `${e.role.toLowerCase()}:${e.meaning.toLowerCase().trim()}`).sort().join("|");
+  return createHash("sha256").update(pairs).digest("hex").slice(0, 16);
 }
+export {
+  computeElementHash
+};
+//# sourceMappingURL=element-hash.js.map