@detiq/crawler 1.1.9 → 1.1.11

package/dist/response-cache.js

@@ -1,153 +1,140 @@
-import { createHash } from 'node:crypto';
-import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, unlinkSync } from 'node:fs';
-import { join } from 'node:path';
-export class ResponseCache {
-    dir;
-    totalHits = 0;
-    totalRequests = 0;
-    constructor(cacheDir) {
-        this.dir = cacheDir;
-        if (!existsSync(cacheDir))
-            mkdirSync(cacheDir, { recursive: true });
+import { createHash } from "node:crypto";
+import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, statSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+class ResponseCache {
+  dir;
+  totalHits = 0;
+  totalRequests = 0;
+  constructor(cacheDir) {
+    this.dir = cacheDir;
+    if (!existsSync(cacheDir)) mkdirSync(cacheDir, { recursive: true });
+  }
+  key(url) {
+    return createHash("sha256").update(url).digest("hex").slice(0, 16);
+  }
+  path(url) {
+    return join(this.dir, `${this.key(url)}.json`);
+  }
+  get(url) {
+    this.totalRequests++;
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return null;
+      const entry = JSON.parse(readFileSync(p, "utf-8"));
+      this.totalHits++;
+      return entry;
+    } catch {
+      return null;
     }
-    key(url) {
-        return createHash('sha256').update(url).digest('hex').slice(0, 16);
+  }
+  read(url) {
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return null;
+      return JSON.parse(readFileSync(p, "utf-8"));
+    } catch {
+      return null;
     }
-    path(url) {
-        return join(this.dir, `${this.key(url)}.json`);
+  }
+  isFresh(url) {
+    const entry = this.read(url);
+    if (!entry?.cacheControl) return false;
+    const m = /max-age=(\d+)/i.exec(entry.cacheControl);
+    if (!m) return false;
+    const maxAgeMs = parseInt(m[1], 10) * 1e3;
+    return entry.cachedAt + maxAgeMs > Date.now();
+  }
+  staleIfError(url) {
+    const entry = this.read(url);
+    if (!entry?.cacheControl) return false;
+    const m = /stale-if-error=(\d+)/i.exec(entry.cacheControl);
+    if (!m) return false;
+    const staleIfErrorMs = parseInt(m[1], 10) * 1e3;
+    return entry.cachedAt + staleIfErrorMs >= Date.now();
+  }
+  set(url, entry) {
+    try {
+      const full = { ...entry, cachedAt: Date.now(), hitCount: 0 };
+      writeFileSync(this.path(url), JSON.stringify(full), "utf-8");
+    } catch {
     }
-    get(url) {
-        this.totalRequests++;
-        try {
-            const p = this.path(url);
-            if (!existsSync(p))
-                return null;
-            const entry = JSON.parse(readFileSync(p, 'utf-8'));
-            this.totalHits++;
-            return entry;
-        }
-        catch {
-            return null;
-        }
-    }
-    read(url) {
-        try {
-            const p = this.path(url);
-            if (!existsSync(p))
-                return null;
-            return JSON.parse(readFileSync(p, 'utf-8'));
-        }
-        catch {
-            return null;
-        }
-    }
-    isFresh(url) {
-        const entry = this.read(url);
-        if (!entry?.cacheControl)
-            return false;
-        const m = /max-age=(\d+)/i.exec(entry.cacheControl);
-        if (!m)
-            return false;
-        const maxAgeMs = parseInt(m[1], 10) * 1000;
-        return entry.cachedAt + maxAgeMs > Date.now();
-    }
-    staleIfError(url) {
-        const entry = this.read(url);
-        if (!entry?.cacheControl)
-            return false;
-        const m = /stale-if-error=(\d+)/i.exec(entry.cacheControl);
-        if (!m)
-            return false;
-        const staleIfErrorMs = parseInt(m[1], 10) * 1000;
-        return entry.cachedAt + staleIfErrorMs >= Date.now();
-    }
-    set(url, entry) {
-        try {
-            const full = { ...entry, cachedAt: Date.now(), hitCount: 0 };
-            writeFileSync(this.path(url), JSON.stringify(full), 'utf-8');
-        }
-        catch { /* non-fatal */ }
-    }
-    getValidationHeaders(url) {
-        const entry = this.get(url);
-        if (!entry)
-            return {};
-        const headers = {};
-        if (entry.etag)
-            headers['If-None-Match'] = entry.etag;
-        if (entry.lastModified)
-            headers['If-Modified-Since'] = entry.lastModified;
-        return headers;
-    }
-    markHit(url) {
-        try {
-            const p = this.path(url);
-            if (!existsSync(p))
-                return;
-            const entry = JSON.parse(readFileSync(p, 'utf-8'));
-            entry.hitCount++;
-            writeFileSync(p, JSON.stringify(entry), 'utf-8');
-        }
-        catch { /* non-fatal */ }
+  }
+  getValidationHeaders(url) {
+    const entry = this.get(url);
+    if (!entry) return {};
+    const headers = {};
+    if (entry.etag) headers["If-None-Match"] = entry.etag;
+    if (entry.lastModified) headers["If-Modified-Since"] = entry.lastModified;
+    return headers;
+  }
+  markHit(url) {
+    try {
+      const p = this.path(url);
+      if (!existsSync(p)) return;
+      const entry = JSON.parse(readFileSync(p, "utf-8"));
+      entry.hitCount++;
+      writeFileSync(p, JSON.stringify(entry), "utf-8");
+    } catch {
     }
-    invalidate(url) {
-        try {
-            const p = this.path(url);
-            if (existsSync(p))
-                unlinkSync(p);
-        }
-        catch { /* non-fatal */ }
+  }
+  invalidate(url) {
+    try {
+      const p = this.path(url);
+      if (existsSync(p)) unlinkSync(p);
+    } catch {
     }
-    clear() {
-        try {
-            for (const f of readdirSync(this.dir)) {
-                if (f.endsWith('.json'))
-                    unlinkSync(join(this.dir, f));
-            }
-        }
-        catch { /* non-fatal */ }
+  }
+  clear() {
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (f.endsWith(".json")) unlinkSync(join(this.dir, f));
+      }
+    } catch {
     }
-    stats() {
-        let entries = 0;
-        let totalSizeBytes = 0;
+  }
+  stats() {
+    let entries = 0;
+    let totalSizeBytes = 0;
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (!f.endsWith(".json")) continue;
+        entries++;
         try {
-            for (const f of readdirSync(this.dir)) {
-                if (!f.endsWith('.json'))
-                    continue;
-                entries++;
-                try {
-                    totalSizeBytes += statSync(join(this.dir, f)).size;
-                }
-                catch { /* skip */ }
-            }
+          totalSizeBytes += statSync(join(this.dir, f)).size;
+        } catch {
         }
-        catch { /* non-fatal */ }
-        return {
-            entries,
-            totalSizeBytes,
-            hitRate: this.totalRequests > 0 ? this.totalHits / this.totalRequests : 0,
-            totalHits: this.totalHits,
-        };
+      }
+    } catch {
     }
-    prune(maxAgeMs) {
-        let deleted = 0;
-        const cutoff = Date.now() - maxAgeMs;
+    return {
+      entries,
+      totalSizeBytes,
+      hitRate: this.totalRequests > 0 ? this.totalHits / this.totalRequests : 0,
+      totalHits: this.totalHits
+    };
+  }
+  prune(maxAgeMs) {
+    let deleted = 0;
+    const cutoff = Date.now() - maxAgeMs;
+    try {
+      for (const f of readdirSync(this.dir)) {
+        if (!f.endsWith(".json")) continue;
         try {
-            for (const f of readdirSync(this.dir)) {
-                if (!f.endsWith('.json'))
-                    continue;
-                try {
-                    const p = join(this.dir, f);
-                    const entry = JSON.parse(readFileSync(p, 'utf-8'));
-                    if (entry.cachedAt < cutoff) {
-                        unlinkSync(p);
-                        deleted++;
-                    }
-                }
-                catch { /* skip */ }
-            }
+          const p = join(this.dir, f);
+          const entry = JSON.parse(readFileSync(p, "utf-8"));
+          if (entry.cachedAt < cutoff) {
+            unlinkSync(p);
+            deleted++;
+          }
+        } catch {
         }
-        catch { /* non-fatal */ }
-        return deleted;
+      }
+    } catch {
     }
+    return deleted;
+  }
 }
+export {
+  ResponseCache
+};
+//# sourceMappingURL=response-cache.js.map

package/dist/responsive-detector.js

@@ -1,50 +1,46 @@
-/**
- * Responsive design detector — captures page at multiple breakpoints
- * and detects layout differences between desktop and mobile viewports.
- *
- * Breakpoints: mobile(375), tablet(768), desktop(1280), wide(1920)
- * Detects: viewport-specific content visibility, menu collapse state,
- * font size changes, hidden/shown elements.
- */
 const BREAKPOINTS = [
-    { width: 375, height: 812, label: 'mobile' },
-    { width: 768, height: 1024, label: 'tablet' },
-    { width: 1280, height: 800, label: 'desktop' },
+  { width: 375, height: 812, label: "mobile" },
+  { width: 768, height: 1024, label: "tablet" },
+  { width: 1280, height: 800, label: "desktop" }
 ];
-export async function detectResponsive(page, url, opts = {}) {
-    const captures = [];
-    const issues = [];
-    for (const bp of BREAKPOINTS) {
-        try {
-            await page.setViewportSize({ width: bp.width, height: bp.height });
-            await page.waitForTimeout(500); // allow CSS reflow
-            const data = await page.evaluate(() => ({
-                visibleTextLength: (document.body.innerText ?? '').length,
-                hasHamburgerMenu: !!(document.querySelector('[class*="hamburger"],[class*="menu-toggle"],[class*="nav-toggle"],[aria-label*="menu"],[aria-label*="Menu"]')),
-                hiddenElements: Array.from(document.querySelectorAll('*')).filter((el) => window.getComputedStyle(el).display === 'none').length,
-                scrollHeight: document.body.scrollHeight,
-            })).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));
-            let screenshotBuffer;
-            if (opts.captureScreenshots) {
-                screenshotBuffer = await page.screenshot({ type: 'jpeg', quality: 60 }).catch(() => undefined);
-            }
-            captures.push({ ...bp, ...data, screenshotBuffer });
-        }
-        catch { /* viewport failed — skip */ }
+async function detectResponsive(page, url, opts = {}) {
+  const captures = [];
+  const issues = [];
+  for (const bp of BREAKPOINTS) {
+    try {
+      await page.setViewportSize({ width: bp.width, height: bp.height });
+      await page.waitForTimeout(500);
+      const data = await page.evaluate(() => ({
+        visibleTextLength: (document.body.innerText ?? "").length,
+        hasHamburgerMenu: !!document.querySelector('[class*="hamburger"],[class*="menu-toggle"],[class*="nav-toggle"],[aria-label*="menu"],[aria-label*="Menu"]'),
+        hiddenElements: Array.from(document.querySelectorAll("*")).filter(
+          (el) => window.getComputedStyle(el).display === "none"
+        ).length,
+        scrollHeight: document.body.scrollHeight
+      })).catch(() => ({ visibleTextLength: 0, hasHamburgerMenu: false, hiddenElements: 0, scrollHeight: 0 }));
+      let screenshotBuffer;
+      if (opts.captureScreenshots) {
+        screenshotBuffer = await page.screenshot({ type: "jpeg", quality: 60 }).catch(() => void 0);
+      }
+      captures.push({ ...bp, ...data, screenshotBuffer });
+    } catch {
     }
-    // Restore to desktop
-    await page.setViewportSize({ width: 1280, height: 800 }).catch(() => { });
-    const mobile = captures.find(c => c.label === 'mobile');
-    const desktop = captures.find(c => c.label === 'desktop');
-    const isResponsive = !!(mobile && desktop && (mobile.hiddenElements !== desktop.hiddenElements ||
-        mobile.scrollHeight !== desktop.scrollHeight ||
-        mobile.hasHamburgerMenu));
-    const hasMobileMenu = captures.some(c => c.label === 'mobile' && c.hasHamburgerMenu);
-    if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {
-        issues.push('Mobile shows significantly more text than desktop — possible responsive issue');
-    }
-    if (!isResponsive && captures.length > 1) {
-        issues.push('Layout appears identical across breakpoints — may not be responsive');
-    }
-    return { url, captures, isResponsive, hasMobileMenu, issues };
+  }
+  await page.setViewportSize({ width: 1280, height: 800 }).catch(() => {
+  });
+  const mobile = captures.find((c) => c.label === "mobile");
+  const desktop = captures.find((c) => c.label === "desktop");
+  const isResponsive = !!(mobile && desktop && (mobile.hiddenElements !== desktop.hiddenElements || mobile.scrollHeight !== desktop.scrollHeight || mobile.hasHamburgerMenu));
+  const hasMobileMenu = captures.some((c) => c.label === "mobile" && c.hasHamburgerMenu);
+  if (mobile && desktop && mobile.visibleTextLength > desktop.visibleTextLength * 1.3) {
+    issues.push("Mobile shows significantly more text than desktop \u2014 possible responsive issue");
+  }
+  if (!isResponsive && captures.length > 1) {
+    issues.push("Layout appears identical across breakpoints \u2014 may not be responsive");
+  }
+  return { url, captures, isResponsive, hasMobileMenu, issues };
 }
+export {
+  detectResponsive
+};
+//# sourceMappingURL=responsive-detector.js.map

package/dist/retry-policy.js

@@ -1,58 +1,54 @@
-/**
- * Retry policy — configurable exponential backoff with jitter.
- * Covers: 5xx server errors, network timeouts, ECONNREFUSED, ECONNRESET.
- * NOT retried: 4xx client errors (except 429 — handled separately), redirects.
- */
 const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
 function shouldRetry(error, retryOn) {
-    // Network-level errors
-    if (error?.code && ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND', 'EPIPE'].includes(error.code))
-        return true;
-    // Playwright navigation timeout
-    if (error?.message && /timeout|net::ERR_/i.test(error.message))
-        return true;
-    // HTTP status errors
-    if (typeof error?.status === 'number')
-        return retryOn.includes(error.status);
-    return false;
+  if (error?.code && ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "ENOTFOUND", "EPIPE"].includes(error.code)) return true;
+  if (error?.message && /timeout|net::ERR_/i.test(error.message)) return true;
+  if (typeof error?.status === "number") return retryOn.includes(error.status);
+  return false;
 }
 function delay(ms) {
-    return new Promise(r => setTimeout(r, ms));
+  return new Promise((r) => setTimeout(r, ms));
 }
-export async function withRetry(fn, opts = {}) {
-    const { maxRetries = 3, baseDelayMs = 1_000, maxDelayMs = 30_000, jitterMs = 500, retryOn = DEFAULT_RETRY_STATUSES, } = opts;
-    let lastError;
-    let totalDelayMs = 0;
-    for (let attempt = 0; attempt <= maxRetries; attempt++) {
-        try {
-            const value = await fn(attempt);
-            return { value, attempts: attempt + 1, totalDelayMs };
-        }
-        catch (err) {
-            lastError = err;
-            if (attempt === maxRetries)
-                break;
-            if (!shouldRetry(err, retryOn))
-                throw err; // non-retryable — fail fast
-            const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
-            const jitter = Math.random() * jitterMs;
-            const waitMs = Math.round(backoff + jitter);
-            totalDelayMs += waitMs;
-            await delay(waitMs);
-        }
+async function withRetry(fn, opts = {}) {
+  const {
+    maxRetries = 3,
+    baseDelayMs = 1e3,
+    maxDelayMs = 3e4,
+    jitterMs = 500,
+    retryOn = DEFAULT_RETRY_STATUSES
+  } = opts;
+  let lastError;
+  let totalDelayMs = 0;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const value = await fn(attempt);
+      return { value, attempts: attempt + 1, totalDelayMs };
+    } catch (err) {
+      lastError = err;
+      if (attempt === maxRetries) break;
+      if (!shouldRetry(err, retryOn)) throw err;
+      const backoff = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attempt));
+      const jitter = Math.random() * jitterMs;
+      const waitMs = Math.round(backoff + jitter);
+      totalDelayMs += waitMs;
+      await delay(waitMs);
     }
-    throw lastError;
+  }
+  throw lastError;
 }
-/** Wraps fetch() with retry policy */
-export async function fetchWithRetry(url, init = {}, retryOpts = {}) {
-    const result = await withRetry(async () => {
-        const res = await fetch(url, init);
-        if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {
-            const err = new Error(`HTTP ${res.status}`);
-            err.status = res.status;
-            throw err;
-        }
-        return res;
-    }, retryOpts);
-    return result.value;
+async function fetchWithRetry(url, init = {}, retryOpts = {}) {
+  const result = await withRetry(async () => {
+    const res = await fetch(url, init);
+    if (retryOpts.retryOn?.includes(res.status) ?? DEFAULT_RETRY_STATUSES.includes(res.status)) {
+      const err = new Error(`HTTP ${res.status}`);
+      err.status = res.status;
+      throw err;
+    }
+    return res;
+  }, retryOpts);
+  return result.value;
 }
+export {
+  fetchWithRetry,
+  withRetry
+};
+//# sourceMappingURL=retry-policy.js.map

package/dist/robots-txt.js

@@ -1,91 +1,77 @@
-/**
- * Minimal robots.txt fetcher + parser. No external deps. Caches per origin.
- * Implements the subset of robots.txt spec used in practice:
- *   User-agent, Disallow, Allow — longest match wins (RFC 9309).
- */
 function parseRobots(text) {
-    const rules = [];
-    let current = null;
-    for (const rawLine of text.split('\n')) {
-        const line = rawLine.split('#')[0].trim();
-        if (!line) {
-            current = null;
-            continue;
-        }
-        const colonIdx = line.indexOf(':');
-        if (colonIdx === -1)
-            continue;
-        const key = line.slice(0, colonIdx).trim().toLowerCase();
-        const value = line.slice(colonIdx + 1).trim();
-        if (key === 'user-agent') {
-            // Start new block only when current block already has rules, otherwise
-            // accumulate multiple User-agent lines into one block.
-            if (!current || current.disallow.length > 0 || current.allow.length > 0) {
-                current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
-                rules.push(current);
-            }
-            else {
-                current.userAgent = value.toLowerCase();
-            }
-        }
-        else if (key === 'disallow' && current && value) {
-            current.disallow.push(value);
-        }
-        else if (key === 'allow' && current && value) {
-            current.allow.push(value);
-        }
+  const rules = [];
+  let current = null;
+  for (const rawLine of text.split("\n")) {
+    const line = rawLine.split("#")[0].trim();
+    if (!line) {
+      current = null;
+      continue;
     }
-    return rules;
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const key = line.slice(0, colonIdx).trim().toLowerCase();
+    const value = line.slice(colonIdx + 1).trim();
+    if (key === "user-agent") {
+      if (!current || current.disallow.length > 0 || current.allow.length > 0) {
+        current = { userAgent: value.toLowerCase(), disallow: [], allow: [] };
+        rules.push(current);
+      } else {
+        current.userAgent = value.toLowerCase();
+      }
+    } else if (key === "disallow" && current && value) {
+      current.disallow.push(value);
+    } else if (key === "allow" && current && value) {
+      current.allow.push(value);
+    }
+  }
+  return rules;
 }
 function isAllowedByRules(rules, userAgent, pathname) {
-    const ua = userAgent.toLowerCase();
-    const exactRules = rules.filter(r => r.userAgent === ua);
-    const wildcardRules = rules.filter(r => r.userAgent === '*');
-    const applicable = exactRules.length > 0 ? exactRules : wildcardRules;
-    if (applicable.length === 0)
-        return true;
-    // Collect all matched allow/disallow directives; longest path wins.
-    const matched = [];
-    for (const rule of applicable) {
-        for (const p of rule.allow) {
-            if (pathname.startsWith(p))
-                matched.push({ path: p, allow: true });
-        }
-        for (const p of rule.disallow) {
-            if (pathname.startsWith(p))
-                matched.push({ path: p, allow: false });
-        }
+  const ua = userAgent.toLowerCase();
+  const exactRules = rules.filter((r) => r.userAgent === ua);
+  const wildcardRules = rules.filter((r) => r.userAgent === "*");
+  const applicable = exactRules.length > 0 ? exactRules : wildcardRules;
+  if (applicable.length === 0) return true;
+  const matched = [];
+  for (const rule of applicable) {
+    for (const p of rule.allow) {
+      if (pathname.startsWith(p)) matched.push({ path: p, allow: true });
+    }
+    for (const p of rule.disallow) {
+      if (pathname.startsWith(p)) matched.push({ path: p, allow: false });
     }
-    if (matched.length === 0)
-        return true;
-    matched.sort((a, b) => b.path.length - a.path.length);
-    return matched[0].allow;
+  }
+  if (matched.length === 0) return true;
+  matched.sort((a, b) => b.path.length - a.path.length);
+  return matched[0].allow;
 }
-export class RobotsCache {
-    cache = new Map();
-    async isAllowed(url, userAgent = 'ZeTa-Crawler') {
-        let origin;
-        let pathname;
-        try {
-            const u = new URL(url);
-            origin = u.origin;
-            pathname = u.pathname || '/';
-        }
-        catch {
-            return true;
-        }
-        if (!this.cache.has(origin)) {
-            try {
-                const res = await fetch(`${origin}/robots.txt`, {
-                    headers: { 'User-Agent': userAgent },
-                    signal: AbortSignal.timeout(5_000),
-                });
-                this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
-            }
-            catch {
-                this.cache.set(origin, []); // network error or timeout = no restrictions
-            }
-        }
-        return isAllowedByRules(this.cache.get(origin), userAgent, pathname);
+class RobotsCache {
+  cache = /* @__PURE__ */ new Map();
+  async isAllowed(url, userAgent = "ZeTa-Crawler") {
+    let origin;
+    let pathname;
+    try {
+      const u = new URL(url);
+      origin = u.origin;
+      pathname = u.pathname || "/";
+    } catch {
+      return true;
+    }
+    if (!this.cache.has(origin)) {
+      try {
+        const res = await fetch(`${origin}/robots.txt`, {
+          headers: { "User-Agent": userAgent },
+          signal: AbortSignal.timeout(5e3)
+        });
+        this.cache.set(origin, res.ok ? parseRobots(await res.text()) : []);
+      } catch {
+        this.cache.set(origin, []);
+      }
     }
+    return isAllowedByRules(this.cache.get(origin), userAgent, pathname);
+  }
 }
+export {
+  RobotsCache
+};
+//# sourceMappingURL=robots-txt.js.map