@detiq/crawler 1.1.8 → 1.1.9

package/dist/response-cache.js

@@ -1,140 +1,153 @@
-import { createHash } from "node:crypto";
-import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, unlinkSync } from "node:fs";
-import { join } from "node:path";
-class ResponseCache {
-  dir;
-  totalHits = 0;
-  totalRequests = 0;
-  constructor(cacheDir) {
-    this.dir = cacheDir;
-    if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true });
-  }
-  key(url) {
-    return createHash("sha256").update(url).digest("hex").slice(0, 16);
-  }
-  path(url) {
-    return join(this.dir, `${this.key(url)}.json`);
-  }
-  get(url) {
-    this.totalRequests++;
-    try {
-      const p = this.path(url);
-      if (!existsSync(p)) return null;
-      const entry = JSON.parse(readFileSync(p, "utf-8"));
-      this.totalHits++;
-      return entry;
-    } catch {
-      return null;
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, unlinkSync } from 'node:fs';
+import { join } from 'node:path';
+export class ResponseCache {
+    dir;
+    totalHits = 0;
+    totalRequests = 0;
+    constructor(cacheDir) {
+        this.dir = cacheDir;
+        if (!existsSync(cacheDir))
+            mkdirSync(cacheDir, { recursive: true });
     }
-  }
-  read(url) {
-    try {
-      const p = this.path(url);
-      if (!existsSync(p)) return null;
-      return JSON.parse(readFileSync(p, "utf-8"));
-    } catch {
-      return null;
+    key(url) {
+        return createHash('sha256').update(url).digest('hex').slice(0, 16);
     }
-  }
-  isFresh(url) {
-    const entry = this.read(url);
-    if (!entry?.cacheControl) return false;
-    const m = /max-age=(\d+)/i.exec(entry.cacheControl);
-    if (!m) return false;
-    const maxAgeMs = parseInt(m[1], 10) * 1e3;
-    return entry.cachedAt + maxAgeMs > Date.now();
-  }
-  staleIfError(url) {
-    const entry = this.read(url);
-    if (!entry?.cacheControl) return false;
-    const m = /stale-if-error=(\d+)/i.exec(entry.cacheControl);
-    if (!m) return false;
-    const staleIfErrorMs = parseInt(m[1], 10) * 1e3;
-    return entry.cachedAt + staleIfErrorMs >= Date.now();
-  }
-  set(url, entry) {
-    try {
-      const full = { ...entry, cachedAt: Date.now(), hitCount: 0 };
-      writeFileSync(this.path(url), JSON.stringify(full), "utf-8");
-    } catch {
+    path(url) {
+        return join(this.dir, `${this.key(url)}.json`);
     }
-  }
-  getValidationHeaders(url) {
-    const entry = this.get(url);
-    if (!entry) return {};
-    const headers = {};
-    if (entry.etag) headers["If-None-Match"] = entry.etag;
-    if (entry.lastModified) headers["If-Modified-Since"] = entry.lastModified;
-    return headers;
-  }
-  markHit(url) {
-    try {
-      const p = this.path(url);
-      if (!existsSync(p)) return;
-      const entry = JSON.parse(readFileSync(p, "utf-8"));
-      entry.hitCount++;
-      writeFileSync(p, JSON.stringify(entry), "utf-8");
-    } catch {
+    get(url) {
+        this.totalRequests++;
+        try {
+            const p = this.path(url);
+            if (!existsSync(p))
+                return null;
+            const entry = JSON.parse(readFileSync(p, 'utf-8'));
+            this.totalHits++;
+            return entry;
+        }
+        catch {
+            return null;
+        }
+    }
+    read(url) {
+        try {
+            const p = this.path(url);
+            if (!existsSync(p))
+                return null;
+            return JSON.parse(readFileSync(p, 'utf-8'));
+        }
+        catch {
+            return null;
+        }
+    }
+    isFresh(url) {
+        const entry = this.read(url);
+        if (!entry?.cacheControl)
+            return false;
+        const m = /max-age=(\d+)/i.exec(entry.cacheControl);
+        if (!m)
+            return false;
+        const maxAgeMs = parseInt(m[1], 10) * 1000;
+        return entry.cachedAt + maxAgeMs > Date.now();
+    }
+    staleIfError(url) {
+        const entry = this.read(url);
+        if (!entry?.cacheControl)
+            return false;
+        const m = /stale-if-error=(\d+)/i.exec(entry.cacheControl);
+        if (!m)
+            return false;
+        const staleIfErrorMs = parseInt(m[1], 10) * 1000;
+        return entry.cachedAt + staleIfErrorMs >= Date.now();
+    }
+    set(url, entry) {
+        try {
+            const full = { ...entry, cachedAt: Date.now(), hitCount: 0 };
+            writeFileSync(this.path(url), JSON.stringify(full), 'utf-8');
+        }
+        catch { /* non-fatal */ }
+    }
+    getValidationHeaders(url) {
+        const entry = this.get(url);
+        if (!entry)
+            return {};
+        const headers = {};
+        if (entry.etag)
+            headers['If-None-Match'] = entry.etag;
+        if (entry.lastModified)
+            headers['If-Modified-Since'] = entry.lastModified;
+        return headers;
+    }
+    markHit(url) {
+        try {
+            const p = this.path(url);
+            if (!existsSync(p))
+                return;
+            const entry = JSON.parse(readFileSync(p, 'utf-8'));
+            entry.hitCount++;
+            writeFileSync(p, JSON.stringify(entry), 'utf-8');
+        }
+        catch { /* non-fatal */ }
     }
-  }
-  invalidate(url) {
-    try {
-      const p = this.path(url);
-      if (existsSync(p)) unlinkSync(p);
-    } catch {
+    invalidate(url) {
+        try {
+            const p = this.path(url);
+            if (existsSync(p))
+                unlinkSync(p);
+        }
+        catch { /* non-fatal */ }
     }
-  }
-  clear() {
-    try {
-      for (const f of readdirSync(this.dir)) {
-        if (f.endsWith(".json")) unlinkSync(join(this.dir, f));
-      }
-    } catch {
+    clear() {
+        try {
+            for (const f of readdirSync(this.dir)) {
+                if (f.endsWith('.json'))
+                    unlinkSync(join(this.dir, f));
+            }
+        }
+        catch { /* non-fatal */ }
     }
-  }
-  stats() {
-    let entries = 0;
-    let totalSizeBytes = 0;
-    try {
-      for (const f of readdirSync(this.dir)) {
-        if (!f.endsWith(".json")) continue;
-        entries++;
+    stats() {
+        let entries = 0;
+        let totalSizeBytes = 0;
         try {
-          totalSizeBytes += statSync(join(this.dir, f)).size;
-        } catch {
+            for (const f of readdirSync(this.dir)) {
+                if (!f.endsWith('.json'))
+                    continue;
+                entries++;
+                try {
+                    totalSizeBytes += statSync(join(this.dir, f)).size;
+                }
+                catch { /* skip */ }
+            }
         }
-      }
-    } catch {
+        catch { /* non-fatal */ }
+        return {
+            entries,
+            totalSizeBytes,
+            hitRate: this.totalRequests > 0 ? this.totalHits / this.totalRequests : 0,
+            totalHits: this.totalHits,
+        };
     }
-    return {
-      entries,
-      totalSizeBytes,
-      hitRate: this.totalRequests > 0 ? this.totalHits / this.totalRequests : 0,
-      totalHits: this.totalHits
-    };
-  }
-  prune(maxAgeMs) {
-    let deleted = 0;
-    const cutoff = Date.now() - maxAgeMs;
-    try {
-      for (const f of readdirSync(this.dir)) {
-        if (!f.endsWith(".json")) continue;
+    prune(maxAgeMs) {
+        let deleted = 0;
+        const cutoff = Date.now() - maxAgeMs;
         try {
-          const p = join(this.dir, f);
-          const entry = JSON.parse(readFileSync(p, "utf-8"));
-          if (entry.cachedAt < cutoff) {
-            unlinkSync(p);
-            deleted++;
-          }
-        } catch {
+            for (const f of readdirSync(this.dir)) {
+                if (!f.endsWith('.json'))
+                    continue;
+                try {
+                    const p = join(this.dir, f);
+                    const entry = JSON.parse(readFileSync(p, 'utf-8'));
+                    if (entry.cachedAt < cutoff) {
+                        unlinkSync(p);
+                        deleted++;
+                    }
+                }
+                catch { /* skip */ }
+            }
         }
-      }
-    } catch {
+        catch { /* non-fatal */ }
+        return deleted;
     }
-    return deleted;
-  }
 }
-export {
-  ResponseCache
-};
-//# sourceMappingURL=response-cache.js.map

package/dist/responsive-detector.js

@@ -1,46 +1,50 @@
+/**
+ * Responsive design detector — captures page at multiple breakpoints
+ * and detects layout differences between desktop and mobile viewports.
+ *
+ * Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)
+ * Detects: viewport-specific content visibility, menu collapse state,
+ * font size changes, hidden/shown elements.
+ */
 const BREAKPOINTS = [
-  { width: 375, height: 812, label: "mobile" },
-  { width: 768, height: 1024, label: "tablet" },
-  { width: 1280, height: 800, label: "desktop" }
+    { width: 375, height: 812, label: 'mobile' },
+    { width: 768, height: 1024, label: 'tablet' },
+    { width: 1280, height: 800, label: 'desktop' },
 ];
-async function detectResponsive(page, url, opts = {}) {
-  const captures = [];
-  const issues = [];
-  for (const bp of BREAKPOINTS) {
-    try {
-      await page.setViewportSize({ width: bp.width, height: bp.height });
-      await page.waitForTimeout(500);
-      const data = await page.evaluate(() => ({
-        visibleTextLength: (document.body.innerText ?? "").length,
-        hasHamburgerMenu: !!document.querySelector('[class*="hamburger"],[class*="menu-toggle"],[class*="nav-toggle"],[aria-label*="menu"],[aria-label*="Menu"]'),
-        hiddenElements: Array.from(document.querySelectorAll("*")).filter(
-          (el) => window.getComputedStyle(el).display === "none"
-        ).length,
-        scrollHeight: document.body.scrollHeight
-      })).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));
-      let screenshotBuffer;
-      if (opts.captureScreenshots) {
-        screenshotBuffer = await page.screenshot({ type: "jpeg", quality: 60 }).catch(() => void 0);
-      }
-      captures.push({ ...bp, ...data, screenshotBuffer });
-    } catch {
+export async function detectResponsive(page, url, opts = {}) {
+    const captures = [];
+    const issues = [];
+    for (const bp of BREAKPOINTS) {
+        try {
+            await page.setViewportSize({ width: bp.width, height: bp.height });
+            await page.waitForTimeout(500); // allow CSS reflow
+            const data = await page.evaluate(() => ({
+                visibleTextLength: (document.body.innerText ?? '').length,
+                hasHamburgerMenu: !!(document.querySelector('[class*="hamburger"],[class*="menu-toggle"],[class*="nav-toggle"],[aria-label*="menu"],[aria-label*="Menu"]')),
+                hiddenElements: Array.from(document.querySelectorAll('*')).filter((el) => window.getComputedStyle(el).display === 'none').length,
+                scrollHeight: document.body.scrollHeight,
+            })).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));
+            let screenshotBuffer;
+            if (opts.captureScreenshots) {
+                screenshotBuffer = await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined);
+            }
+            captures.push({ ...bp, ...data, screenshotBuffer });
+        }
+        catch { /* viewport failed — skip */ }
     }
-  }
-  await page.setViewportSize({ width: 1280, height: 800 }).catch(() => {
-  });
-  const mobile = captures.find((c) => c.label === "mobile");
-  const desktop = captures.find((c) => c.label === "desktop");
-  const isResponsive = !!(mobile && desktop && (mobile.hiddenElements !== desktop.hiddenElements || mobile.scrollHeight !== desktop.scrollHeight || mobile.hasHamburgerMenu));
-  const hasMobileMenu = captures.some((c) => c.label === "mobile" && c.hasHamburgerMenu);
-  if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {
-    issues.push("Mobile shows significantly more text than desktop \u2014 possible responsive issue");
-  }
-  if (!isResponsive && captures.length > 1) {
-    issues.push("Layout appears identical across breakpoints \u2014 may not be responsive");
-  }
-  return { url, captures, isResponsive, hasMobileMenu, issues };
+    // Restore to desktop
+    await page.setViewportSize({ width: 1280, height: 800 }).catch(() => { });
+    const mobile = captures.find(c => c.label === 'mobile');
+    const desktop = captures.find(c => c.label === 'desktop');
+    const isResponsive = !!(mobile && desktop && (mobile.hiddenElements !== desktop.hiddenElements ||
+        mobile.scrollHeight !== desktop.scrollHeight ||
+        mobile.hasHamburgerMenu));
+    const hasMobileMenu = captures.some(c => c.label === 'mobile' && c.hasHamburgerMenu);
+    if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {
+        issues.push('Mobile shows significantly more text than desktop — possible responsive issue');
+    }
+    if (!isResponsive && captures.length > 1) {
+        issues.push('Layout appears identical across breakpoints — may not be responsive');
+    }
+    return { url, captures, isResponsive, hasMobileMenu, issues };
 }
-export {
-  detectResponsive
-};
-//# sourceMappingURL=responsive-detector.js.map

package/dist/retry-policy.js

@@ -1,54 +1,58 @@
+/**
+ * Retry policy — configurable exponential backoff with jitter.
+ * Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.
+ * NOT retried: 4xx client errors (except 429 — handled separately), redirects.
+ */
 const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
 function shouldRetry(error, retryOn) {
-  if (error?.code && ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EPIPE"].includes(error.code)) return true;
-  if (error?.message && /timeout|net::ERR_/i.test(error.message)) return true;
-  if (typeof error?.status === "number") return retryOn.includes(error.status);
-  return false;
+    // Network-level errors
+    if (error?.code && ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EPIPE'].includes(error.code))
+        return true;
+    // Playwright navigation timeout
+    if (error?.message && /timeout|net::ERR_/i.test(error.message))
+        return true;
+    // HTTP status errors
+    if (typeof error?.status === 'number')
+        return retryOn.includes(error.status);
+    return false;
 }
 function delay(ms) {
-  return new Promise((r) => setTimeout(r, ms));
+    return new Promise(r => setTimeout(r, ms));
 }
-async function withRetry(fn, opts = {}) {
-  const {
-    maxRetries = 3,
-    baseDelayMs = 1e3,
-    maxDelayMs = 3e4,
-    jitterMs = 500,
-    retryOn = DEFAULT_RETRY_STATUSES
-  } = opts;
-  let lastError;
-  let totalDelayMs = 0;
-  for (let attempt = 0; attempt <= maxRetries; attempt++) {
-    try {
-      const value = await fn(attempt);
-      return { value, attempts: attempt + 1, totalDelayMs };
-    } catch (err) {
-      lastError = err;
-      if (attempt === maxRetries) break;
-      if (!shouldRetry(err, retryOn)) throw err;
-      const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
-      const jitter = Math.random() * jitterMs;
-      const waitMs = Math.round(backoff + jitter);
-      totalDelayMs += waitMs;
-      await delay(waitMs);
+export async function withRetry(fn, opts = {}) {
+    const { maxRetries = 3, baseDelayMs = 1_000, maxDelayMs = 30_000, jitterMs = 500, retryOn = DEFAULT_RETRY_STATUSES, } = opts;
+    let lastError;
+    let totalDelayMs = 0;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+        try {
+            const value = await fn(attempt);
+            return { value, attempts: attempt + 1, totalDelayMs };
+        }
+        catch (err) {
+            lastError = err;
+            if (attempt === maxRetries)
+                break;
+            if (!shouldRetry(err, retryOn))
+                throw err; // non-retryable — fail fast
+            const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
+            const jitter = Math.random() * jitterMs;
+            const waitMs = Math.round(backoff + jitter);
+            totalDelayMs += waitMs;
+            await delay(waitMs);
+        }
     }
-  }
-  throw lastError;
+    throw lastError;
 }
-async function fetchWithRetry(url, init = {}, retryOpts = {}) {
-  const result = await withRetry(async () => {
-    const res = await fetch(url, init);
-    if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {
-      const err = new Error(`HTTP ${res.status}`);
-      err.status = res.status;
-      throw err;
-    }
-    return res;
-  }, retryOpts);
-  return result.value;
+/** Wraps fetch() with retry policy */
+export async function fetchWithRetry(url, init = {}, retryOpts = {}) {
+    const result = await withRetry(async () => {
+        const res = await fetch(url, init);
+        if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {
+            const err = new Error(`HTTP ${res.status}`);
+            err.status = res.status;
+            throw err;
+        }
+        return res;
+    }, retryOpts);
+    return result.value;
 }
-export {
-  fetchWithRetry,
-  withRetry
-};
-//# sourceMappingURL=retry-policy.js.map

package/dist/robots-txt.js

@@ -1,77 +1,91 @@
+/**
+ * Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
+ * Implements the subset of robots.txt spec used in practice:
+ *   User-agent, Disallow, Allow — longest match wins (RFC 9309).
+ */
 function parseRobots(text) {
-  const rules = [];
-  let current = null;
-  for (const rawLine of text.split("\n")) {
-    const line = rawLine.split("#")[0].trim();
-    if (!line) {
-      current = null;
-      continue;
+    const rules = [];
+    let current = null;
+    for (const rawLine of text.split('\n')) {
+        const line = rawLine.split('#')[0].trim();
+        if (!line) {
+            current = null;
+            continue;
+        }
+        const colonIdx = line.indexOf(':');
+        if (colonIdx === -1)
+            continue;
+        const key = line.slice(0, colonIdx).trim().toLowerCase();
+        const value = line.slice(colonIdx + 1).trim();
+        if (key === 'user-agent') {
+            // Start new block only when current block already has rules, otherwise
+            // accumulate multiple User-agent lines into one block.
+            if (!current || current.disallow.length > 0 || current.allow.length > 0) {
+                current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
+                rules.push(current);
+            }
+            else {
+                current.userAgent = value.toLowerCase();
+            }
+        }
+        else if (key === 'disallow' && current && value) {
+            current.disallow.push(value);
+        }
+        else if (key === 'allow' && current && value) {
+            current.allow.push(value);
+        }
     }
-    const colonIdx = line.indexOf(":");
-    if (colonIdx === -1) continue;
-    const key = line.slice(0, colonIdx).trim().toLowerCase();
-    const value = line.slice(colonIdx + 1).trim();
-    if (key === "user-agent") {
-      if (!current || current.disallow.length > 0 || current.allow.length > 0) {
-        current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
-        rules.push(current);
-      } else {
-        current.userAgent = value.toLowerCase();
-      }
-    } else if (key === "disallow" && current && value) {
-      current.disallow.push(value);
-    } else if (key === "allow" && current && value) {
-      current.allow.push(value);
-    }
-  }
-  return rules;
+    return rules;
 }
 function isAllowedByRules(rules, userAgent, pathname) {
-  const ua = userAgent.toLowerCase();
-  const exactRules = rules.filter((r) => r.userAgent === ua);
-  const wildcardRules = rules.filter((r) => r.userAgent === "*");
-  const applicable = exactRules.length > 0 ? exactRules : wildcardRules;
-  if (applicable.length === 0) return true;
-  const matched = [];
-  for (const rule of applicable) {
-    for (const p of rule.allow) {
-      if (pathname.startsWith(p)) matched.push({ path: p, allow: true });
-    }
-    for (const p of rule.disallow) {
-      if (pathname.startsWith(p)) matched.push({ path: p, allow: false });
+    const ua = userAgent.toLowerCase();
+    const exactRules = rules.filter(r => r.userAgent === ua);
+    const wildcardRules = rules.filter(r => r.userAgent === '*');
+    const applicable = exactRules.length > 0 ? exactRules : wildcardRules;
+    if (applicable.length === 0)
+        return true;
+    // Collect all matched allow/disallow directives; longest path wins.
+    const matched = [];
+    for (const rule of applicable) {
+        for (const p of rule.allow) {
+            if (pathname.startsWith(p))
+                matched.push({ path: p, allow: true });
+        }
+        for (const p of rule.disallow) {
+            if (pathname.startsWith(p))
+                matched.push({ path: p, allow: false });
+        }
     }
-  }
-  if (matched.length === 0) return true;
-  matched.sort((a, b) => b.path.length - a.path.length);
-  return matched[0].allow;
+    if (matched.length === 0)
+        return true;
+    matched.sort((a, b) => b.path.length - a.path.length);
+    return matched[0].allow;
 }
-class RobotsCache {
-  cache = /* @__PURE__ */ new Map();
-  async isAllowed(url, userAgent = "ZeTa-Crawler") {
-    let origin;
-    let pathname;
-    try {
-      const u = new URL(url);
-      origin = u.origin;
-      pathname = u.pathname || "/";
-    } catch {
-      return true;
-    }
-    if (!this.cache.has(origin)) {
-      try {
-        const res = await fetch(`${origin}/robots.txt`, {
-          headers: { "User-Agent": userAgent },
-          signal: AbortSignal.timeout(5e3)
-        });
-        this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
-      } catch {
-        this.cache.set(origin, []);
-      }
+export class RobotsCache {
+    cache = new Map();
+    async isAllowed(url, userAgent = 'ZeTa-Crawler') {
+        let origin;
+        let pathname;
+        try {
+            const u = new URL(url);
+            origin = u.origin;
+            pathname = u.pathname || '/';
+        }
+        catch {
+            return true;
+        }
+        if (!this.cache.has(origin)) {
+            try {
+                const res = await fetch(`${origin}/robots.txt`, {
+                    headers: { 'User-Agent': userAgent },
+                    signal: AbortSignal.timeout(5_000),
+                });
+                this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
+            }
+            catch {
+                this.cache.set(origin, []); // network error or timeout = no restrictions
+            }
+        }
+        return isAllowedByRules(this.cache.get(origin), userAgent, pathname);
     }
-    return isAllowedByRules(this.cache.get(origin), userAgent, pathname);
-  }
 }
-export {
-  RobotsCache
-};
-//# sourceMappingURL=robots-txt.js.map