@detiq/crawler 1.1.7 → 1.1.9

package/dist/custom-js-injector.js

@@ -1,50 +1,66 @@
-async function injectCustomJs(page, opts) {
-  const result = {
-    initScriptApplied: false,
-    postLoadResult: null,
-    storageSet: false,
-    cookiesSet: 0
-  };
-  if (opts.cookies?.length) {
-    try {
-      await page.context().addCookies(
-        opts.cookies.map((c) => ({ name: c.name, value: c.value, domain: c.domain ?? "", path: c.path ?? "/", url: page.url() || void 0 }))
-      );
-      result.cookiesSet = opts.cookies.length;
-    } catch (e) {
-      result.error = `Cookie injection failed: ${e.message}`;
+/**
+ * Custom JS injector — run user-provided JavaScript in the context of each crawled page.
+ *
+ * Use cases:
+ * - Extract custom data points not captured by standard extractors
+ * - Set localStorage/sessionStorage values before crawl
+ * - Override fetch() to intercept API calls
+ * - Simulate user events (keydown, focus, etc.)
+ * - Bypass specific anti-bot checks
+ *
+ * Security: scripts run in the page context (sandboxed by Chromium's renderer).
+ * They cannot access the Node.js process or file system.
+ * ZeTa validates scripts are strings before injection.
+ */
+export async function injectCustomJs(page, opts) {
+    const result = {
+        initScriptApplied: false,
+        postLoadResult: null,
+        storageSet: false,
+        cookiesSet: 0,
+    };
+    // Set cookies via Playwright context
+    if (opts.cookies?.length) {
+        try {
+            await page.context().addCookies(opts.cookies.map(c => ({ name: c.name, value: c.value, domain: c.domain ?? '', path: c.path ?? '/', url: page.url() || undefined })));
+            result.cookiesSet = opts.cookies.length;
+        }
+        catch (e) {
+            result.error = `Cookie injection failed: ${e.message}`;
+        }
     }
-  }
-  if (opts.initScript || opts.localStorage || opts.sessionStorage) {
-    const storageSetup = [
-      opts.localStorage ? `Object.entries(${JSON.stringify(opts.localStorage)}).forEach(([k,v])=>localStorage.setItem(k,v));` : "",
-      opts.sessionStorage ? `Object.entries(${JSON.stringify(opts.sessionStorage)}).forEach(([k,v])=>sessionStorage.setItem(k,v));` : ""
-    ].filter(Boolean).join("\n");
-    const combinedInit = [storageSetup, opts.initScript ?? ""].filter(Boolean).join("\n");
-    try {
-      await page.addInitScript(combinedInit);
-      result.initScriptApplied = true;
-      result.storageSet = !!(opts.localStorage || opts.sessionStorage);
-    } catch (e) {
-      result.error = `Init script failed: ${e.message}`;
+    // Init script (runs before page load on every navigation)
+    if (opts.initScript || opts.localStorage || opts.sessionStorage) {
+        const storageSetup = [
+            opts.localStorage ? `Object.entries(${JSON.stringify(opts.localStorage)}).forEach(([k,v])=>localStorage.setItem(k,v));` : '',
+            opts.sessionStorage ? `Object.entries(${JSON.stringify(opts.sessionStorage)}).forEach(([k,v])=>sessionStorage.setItem(k,v));` : '',
+        ].filter(Boolean).join('\n');
+        const combinedInit = [storageSetup, opts.initScript ?? ''].filter(Boolean).join('\n');
+        try {
+            await page.addInitScript(combinedInit);
+            result.initScriptApplied = true;
+            result.storageSet = !!(opts.localStorage || opts.sessionStorage);
+        }
+        catch (e) {
+            result.error = `Init script failed: ${e.message}`;
+        }
     }
-  }
-  if (opts.postLoadScript) {
-    try {
-      result.postLoadResult = await page.evaluate(opts.postLoadScript);
-    } catch (e) {
-      result.error = `Post-load script failed: ${e.message}`;
+    // Post-load script (runs after navigation)
+    if (opts.postLoadScript) {
+        try {
+            result.postLoadResult = await page.evaluate(opts.postLoadScript);
+        }
+        catch (e) {
+            result.error = `Post-load script failed: ${e.message}`;
+        }
     }
-  }
-  return result;
+    return result;
 }
-function validateScript(script) {
-  if (typeof script !== "string") throw new Error("Script must be a string");
-  if (script.length > 5e4) throw new Error("Script too large (>50KB)");
-  return script;
+/** Safe string validator — ensures script is a string, not an object/function reference */
+export function validateScript(script) {
+    if (typeof script !== 'string')
+        throw new Error('Script must be a string');
+    if (script.length > 50_000)
+        throw new Error('Script too large (>50KB)');
+    return script;
 }
-export {
-  injectCustomJs,
-  validateScript
-};
-//# sourceMappingURL=custom-js-injector.js.map

package/dist/dark-mode-capture.js

@@ -1,75 +1,89 @@
+/**
+ * Dark mode capture — screenshot pages in both light and dark color schemes.
+ *
+ * Uses Playwright's page.emulateMedia({ colorScheme: 'dark' }) to force dark mode,
+ * then captures a screenshot. Compares with light mode to determine if dark mode
+ * is supported (significant visual difference).
+ *
+ * Also detects prefers-color-scheme media query support in CSS.
+ */
 async function getBackgroundColor(page) {
-  try {
-    return await page.evaluate(() => {
-      const body = document.body;
-      return window.getComputedStyle(body).backgroundColor;
-    });
-  } catch {
-    return "unknown";
-  }
+    try {
+        return await page.evaluate(() => {
+            const body = document.body;
+            return window.getComputedStyle(body).backgroundColor;
+        });
+    }
+    catch {
+        return 'unknown';
+    }
 }
 function colorsAreDifferent(a, b) {
-  if (a === b) return false;
-  if (a === "unknown" || b === "unknown") return false;
-  const parse = (c) => {
-    const m = c.match(/\d+/g);
-    return m ? m.map(Number) : [0, 0, 0];
-  };
-  const [ar, ag, ab] = parse(a);
-  const [br, bg, bb] = parse(b);
-  const lumA = 0.299 * ar + 0.587 * ag + 0.114 * ab;
-  const lumB = 0.299 * br + 0.587 * bg + 0.114 * bb;
-  return Math.abs(lumA - lumB) > 30;
+    if (a === b)
+        return false;
+    if (a === 'unknown' || b === 'unknown')
+        return false;
+    // Parse rgb(r, g, b) and compare luminance
+    const parse = (c) => {
+        const m = c.match(/\d+/g);
+        return m ? m.map(Number) : [0, 0, 0];
+    };
+    const [ar, ag, ab] = parse(a);
+    const [br, bg, bb] = parse(b);
+    const lumA = 0.299 * ar + 0.587 * ag + 0.114 * ab;
+    const lumB = 0.299 * br + 0.587 * bg + 0.114 * bb;
+    return Math.abs(lumA - lumB) > 30; // significant luminance difference
 }
 async function hasCssDarkModeSupport(page) {
-  try {
-    return await page.evaluate(() => {
-      const sheets = Array.from(document.styleSheets);
-      for (const sheet of sheets) {
-        try {
-          const rules = Array.from(sheet.cssRules ?? []);
-          for (const rule of rules) {
-            if (rule instanceof CSSMediaRule && rule.conditionText?.includes("prefers-color-scheme")) {
-              return true;
+    try {
+        return await page.evaluate(() => {
+            const sheets = Array.from(document.styleSheets);
+            for (const sheet of sheets) {
+                try {
+                    const rules = Array.from(sheet.cssRules ?? []);
+                    for (const rule of rules) {
+                        if (rule instanceof CSSMediaRule && rule.conditionText?.includes('prefers-color-scheme')) {
+                            return true;
+                        }
+                    }
+                }
+                catch { /* cross-origin stylesheet — skip */ }
             }
-          }
-        } catch {
-        }
-      }
-      return false;
-    });
-  } catch {
-    return false;
-  }
+            return false;
+        });
+    }
+    catch {
+        return false;
+    }
 }
-async function captureDarkMode(page, url, opts = {}) {
-  const capturedAt = (/* @__PURE__ */ new Date()).toISOString();
-  const { takeScreenshots = false } = opts;
-  await page.emulateMedia({ colorScheme: "light" }).catch(() => {
-  });
-  const backgroundColorLight = await getBackgroundColor(page);
-  const hasCssDark = await hasCssDarkModeSupport(page);
-  const lightScreenshot = takeScreenshots ? await page.screenshot({ type: "jpeg", quality: 60 }).catch(() => void 0) : void 0;
-  await page.emulateMedia({ colorScheme: "dark" }).catch(() => {
-  });
-  await page.waitForTimeout(300);
-  const backgroundColorDark = await getBackgroundColor(page);
-  const darkScreenshot = takeScreenshots ? await page.screenshot({ type: "jpeg", quality: 60 }).catch(() => void 0) : void 0;
-  await page.emulateMedia({ colorScheme: "light" }).catch(() => {
-  });
-  const hasDarkMode = colorsAreDifferent(backgroundColorLight, backgroundColorDark) || hasCssDark;
-  return {
-    url,
-    lightScreenshot,
-    darkScreenshot,
-    hasDarkMode,
-    hasCssDarkMode: hasCssDark,
-    backgroundColorLight,
-    backgroundColorDark,
-    capturedAt
-  };
+export async function captureDarkMode(page, url, opts = {}) {
+    const capturedAt = new Date().toISOString();
+    const { takeScreenshots = false } = opts;
+    // Light mode
+    await page.emulateMedia({ colorScheme: 'light' }).catch(() => { });
+    const backgroundColorLight = await getBackgroundColor(page);
+    const hasCssDark = await hasCssDarkModeSupport(page);
+    const lightScreenshot = takeScreenshots
+        ? await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined)
+        : undefined;
+    // Dark mode
+    await page.emulateMedia({ colorScheme: 'dark' }).catch(() => { });
+    await page.waitForTimeout(300); // allow CSS transitions
+    const backgroundColorDark = await getBackgroundColor(page);
+    const darkScreenshot = takeScreenshots
+        ? await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined)
+        : undefined;
+    // Restore light mode
+    await page.emulateMedia({ colorScheme: 'light' }).catch(() => { });
+    const hasDarkMode = colorsAreDifferent(backgroundColorLight, backgroundColorDark) || hasCssDark;
+    return {
+        url,
+        lightScreenshot,
+        darkScreenshot,
+        hasDarkMode,
+        hasCssDarkMode: hasCssDark,
+        backgroundColorLight,
+        backgroundColorDark,
+        capturedAt,
+    };
 }
-export {
-  captureDarkMode
-};
-//# sourceMappingURL=dark-mode-capture.js.map

package/dist/dead-link-detector.js

@@ -1,50 +1,58 @@
-class DeadLinkTracker {
-  deadLinks = [];
-  checked = /* @__PURE__ */ new Map();
-  // url → statusCode
-  record(url, statusCode, source) {
-    if (this.checked.has(url)) return;
-    this.checked.set(url, statusCode);
-    if (statusCode >= 400) {
-      this.deadLinks.push({
-        url,
-        statusCode,
-        sourceUrl: source?.sourceUrl,
-        anchorText: source?.anchorText,
-        checkedAt: (/* @__PURE__ */ new Date()).toISOString(),
-        reason: statusCode === 0 ? "timeout" : "http-error"
-      });
+/**
+ * Dead link detector — tracks broken links discovered during crawl.
+ *
+ * A "dead link" is a URL that:
+ * - Returns 404, 410, or other 4xx/5xx responses
+ * - Times out after maxRetries
+ * - Redirects to an error page (detected by title heuristic)
+ *
+ * Results: list of broken links with source page, status code, and anchor text.
+ * Used for: SEO audit export, test report enrichment.
+ */
+export class DeadLinkTracker {
+    deadLinks = [];
+    checked = new Map(); // url → statusCode
+    record(url, statusCode, source) {
+        if (this.checked.has(url))
+            return;
+        this.checked.set(url, statusCode);
+        if (statusCode >= 400) {
+            this.deadLinks.push({
+                url,
+                statusCode,
+                sourceUrl: source?.sourceUrl,
+                anchorText: source?.anchorText,
+                checkedAt: new Date().toISOString(),
+                reason: statusCode === 0 ? 'timeout' : 'http-error',
+            });
+        }
     }
-  }
-  recordTimeout(url, sourceUrl) {
-    if (this.checked.has(url)) return;
-    this.checked.set(url, 0);
-    this.deadLinks.push({ url, statusCode: 0, sourceUrl, checkedAt: (/* @__PURE__ */ new Date()).toISOString(), reason: "timeout" });
-  }
-  getDeadLinks() {
-    return [...this.deadLinks];
-  }
-  isKnownBroken(url) {
-    const code = this.checked.get(url);
-    return code !== void 0 && code >= 400;
-  }
-  stats() {
-    return {
-      total: this.deadLinks.length,
-      by404: this.deadLinks.filter((l) => l.statusCode === 404).length,
-      by5xx: this.deadLinks.filter((l) => l.statusCode >= 500).length,
-      byTimeout: this.deadLinks.filter((l) => l.reason === "timeout").length
-    };
-  }
-  toCsv() {
-    const rows = [["URL", "Status", "Source URL", "Anchor Text", "Checked At"]];
-    for (const l of this.deadLinks) {
-      rows.push([l.url, String(l.statusCode), l.sourceUrl ?? "", l.anchorText ?? "", l.checkedAt]);
+    recordTimeout(url, sourceUrl) {
+        if (this.checked.has(url))
+            return;
+        this.checked.set(url, 0);
+        this.deadLinks.push({ url, statusCode: 0, sourceUrl, checkedAt: new Date().toISOString(), reason: 'timeout' });
+    }
+    getDeadLinks() {
+        return [...this.deadLinks];
+    }
+    isKnownBroken(url) {
+        const code = this.checked.get(url);
+        return code !== undefined && code >= 400;
+    }
+    stats() {
+        return {
+            total: this.deadLinks.length,
+            by404: this.deadLinks.filter(l => l.statusCode === 404).length,
+            by5xx: this.deadLinks.filter(l => l.statusCode >= 500).length,
+            byTimeout: this.deadLinks.filter(l => l.reason === 'timeout').length,
+        };
+    }
+    toCsv() {
+        const rows = [['URL', 'Status', 'Source URL', 'Anchor Text', 'Checked At']];
+        for (const l of this.deadLinks) {
+            rows.push([l.url, String(l.statusCode), l.sourceUrl ?? '', l.anchorText ?? '', l.checkedAt]);
+        }
+        return rows.map(r => r.map(c => `"${c.replace(/"/g, '""')}"`).join(',')).join('\n');
     }
-    return rows.map((r) => r.map((c) => `"${c.replace(/"/g, '""')}"`).join(",")).join("\n");
-  }
 }
-export {
-  DeadLinkTracker
-};
-//# sourceMappingURL=dead-link-detector.js.map

package/dist/element-hash.js

@@ -1,9 +1,13 @@
-import { createHash } from "node:crypto";
-function computeElementHash(elements) {
-  const pairs = elements.map((e) => `${e.role.toLowerCase()}:${e.meaning.toLowerCase().trim()}`).sort().join("|");
-  return createHash("sha256").update(pairs).digest("hex").slice(0, 16);
+import { createHash } from 'node:crypto';
+/**
+ * Compute a stable fingerprint of a screen's element set.
+ * Sorted so insertion-order changes (React re-renders) don't produce false positives.
+ * Returns first 16 hex chars of SHA-256.
+ */
+export function computeElementHash(elements) {
+    const pairs = elements
+        .map((e) => `${e.role.toLowerCase()}:${e.meaning.toLowerCase().trim()}`)
+        .sort()
+        .join('|');
+    return createHash('sha256').update(pairs).digest('hex').slice(0, 16);
 }
-export {
-  computeElementHash
-};
-//# sourceMappingURL=element-hash.js.map