@detiq/crawler 1.1.9 → 1.1.11

package/dist/cookie-consent.js

@@ -1,93 +1,78 @@
-/**
- * Cookie consent auto-acceptor.
- *
- * Detects and clicks "Accept" on common cookie consent banners before the
- * crawler proceeds. Without this, many EU sites show a consent wall that
- * blocks content or shows a different layout, making crawl results unreliable.
- *
- * Supports: Cookiebot, OneTrust, TrustArc, Usercentrics, GDPR Cookie Compliance,
- * generic "Accept All" / "Accept Cookies" buttons.
- *
- * Strategy: try known selectors first, then fall back to text-based button search.
- * Non-blocking — if no banner is found or click fails, crawl proceeds normally.
- */
-// Known consent platform selectors
 const CONSENT_SELECTORS = [
-    // Cookiebot
-    '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',
-    '#CybotCookiebotDialogBodyButtonAccept',
-    // OneTrust
-    '#onetrust-accept-btn-handler',
-    '.ot-sdk-accept-btn',
-    // TrustArc
-    '.truste_popframe [title*="Accept"]',
-    '#truste-consent-button',
-    // Usercentrics
-    '[data-testid="uc-accept-all-button"]',
-    // Generic high-confidence
-    '[id*="accept"][id*="cookie"]',
-    '[class*="accept"][class*="cookie"]',
-    '[class*="cookie"][class*="accept"]',
-    // GDPR Cookie Compliance
-    '.gdpr-cookie-notice-accept',
-    // Quantcast
-    '.qc-cmp2-summary-buttons button:first-child',
+  // Cookiebot
+  "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
+  "#CybotCookiebotDialogBodyButtonAccept",
+  // OneTrust
+  "#onetrust-accept-btn-handler",
+  ".ot-sdk-accept-btn",
+  // TrustArc
+  '.truste_popframe [title*="Accept"]',
+  "#truste-consent-button",
+  // Usercentrics
+  '[data-testid="uc-accept-all-button"]',
+  // Generic high-confidence
+  '[id*="accept"][id*="cookie"]',
+  '[class*="accept"][class*="cookie"]',
+  '[class*="cookie"][class*="accept"]',
+  // GDPR Cookie Compliance
+  ".gdpr-cookie-notice-accept",
+  // Quantcast
+  ".qc-cmp2-summary-buttons button:first-child"
 ];
-// Fallback: button text matching
 const ACCEPT_TEXT_PATTERNS = [
-    /^accept all$/i,
-    /^accept all cookies$/i,
-    /^allow all$/i,
-    /^i accept$/i,
-    /^agree$/i,
-    /^ok$/i,
-    /^got it$/i,
-    /^allow cookies$/i,
+  /^accept all$/i,
+  /^accept all cookies$/i,
+  /^allow all$/i,
+  /^i accept$/i,
+  /^agree$/i,
+  /^ok$/i,
+  /^got it$/i,
+  /^allow cookies$/i
 ];
-export async function acceptCookieConsent(page, opts = {}) {
-    const { waitMs = 1_500 } = opts;
-    // Wait briefly for consent banners to appear
-    await page.waitForTimeout(waitMs).catch(() => { });
-    // Try known selectors first
-    for (const selector of CONSENT_SELECTORS) {
-        try {
-            const el = await page.$(selector);
-            if (el && await el.isVisible()) {
-                await el.click({ timeout: 2_000 });
-                await page.waitForTimeout(500);
-                return { found: true, clicked: true, method: `selector: ${selector}` };
-            }
-        }
-        catch { /* try next */ }
-    }
-    // Fallback: find buttons with accept-like text
+async function acceptCookieConsent(page, opts = {}) {
+  const { waitMs = 1500 } = opts;
+  await page.waitForTimeout(waitMs).catch(() => {
+  });
+  for (const selector of CONSENT_SELECTORS) {
     try {
-        const buttons = await page.$$('button, [role="button"], a[class*="cookie"]');
-        for (const btn of buttons) {
-            const text = ((await btn.textContent()) ?? '').trim();
-            if (ACCEPT_TEXT_PATTERNS.some(re => re.test(text))) {
-                if (await btn.isVisible()) {
-                    await btn.click({ timeout: 2_000 });
-                    await page.waitForTimeout(500);
-                    return { found: true, clicked: true, method: `text: "${text}"` };
-                }
-            }
-        }
+      const el = await page.$(selector);
+      if (el && await el.isVisible()) {
+        await el.click({ timeout: 2e3 });
+        await page.waitForTimeout(500);
+        return { found: true, clicked: true, method: `selector: ${selector}` };
+      }
+    } catch {
     }
-    catch { /* non-fatal */ }
-    return { found: false, clicked: false };
-}
-/** Check if a consent banner is still visible after attempted dismissal */
-export async function isConsentBannerVisible(page) {
-    try {
-        for (const sel of CONSENT_SELECTORS.slice(0, 5)) {
-            const el = await page.$(sel);
-            if (el && await el.isVisible())
-                return true;
+  }
+  try {
+    const buttons = await page.$$('button, [role="button"], a[class*="cookie"]');
+    for (const btn of buttons) {
+      const text = (await btn.textContent() ?? "").trim();
+      if (ACCEPT_TEXT_PATTERNS.some((re) => re.test(text))) {
+        if (await btn.isVisible()) {
+          await btn.click({ timeout: 2e3 });
+          await page.waitForTimeout(500);
+          return { found: true, clicked: true, method: `text: "${text}"` };
         }
-        return false;
+      }
     }
-    catch {
-        return false;
+  } catch {
+  }
+  return { found: false, clicked: false };
+}
+async function isConsentBannerVisible(page) {
+  try {
+    for (const sel of CONSENT_SELECTORS.slice(0, 5)) {
+      const el = await page.$(sel);
+      if (el && await el.isVisible()) return true;
     }
+    return false;
+  } catch {
+    return false;
+  }
 }
+export {
+  acceptCookieConsent,
+  isConsentBannerVisible
+};
+//# sourceMappingURL=cookie-consent.js.map

package/dist/cookie-jar.js

@@ -1,68 +1,61 @@
-/**
- * Cookie jar — persist and restore Playwright storageState cookies between
- * crawl segments and restarts. Enables continuous auth across session boundaries
- * (e.g. segment 1 logs in, segment 2–N inherit the session without re-login).
- *
- * Storage: JSON file per project in the crawl cache dir.
- * Format: Playwright's storageState shape (cookies + origins).
- */
-import * as fs from 'fs/promises';
-import * as path from 'path';
+import * as fs from "fs/promises";
+import * as path from "path";
 function jarPath(cacheDir, projectId) {
-    return path.join(cacheDir, `cookie-jar-${projectId}.json`);
+  return path.join(cacheDir, `cookie-jar-${projectId}.json`);
 }
-export async function saveCookieJar(cacheDir, projectId, tenantId, storageState, domain) {
-    await fs.mkdir(cacheDir, { recursive: true });
-    const entry = {
-        projectId,
-        tenantId,
-        storageState,
-        savedAt: new Date().toISOString(),
-        domain,
-    };
-    await fs.writeFile(jarPath(cacheDir, projectId), JSON.stringify(entry, null, 2), 'utf8');
+async function saveCookieJar(cacheDir, projectId, tenantId, storageState, domain) {
+  await fs.mkdir(cacheDir, { recursive: true });
+  const entry = {
+    projectId,
+    tenantId,
+    storageState,
+    savedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    domain
+  };
+  await fs.writeFile(jarPath(cacheDir, projectId), JSON.stringify(entry, null, 2), "utf8");
 }
-export async function loadCookieJar(cacheDir, projectId, maxAgeMs = 8 * 60 * 60 * 1000) {
-    try {
-        const raw = await fs.readFile(jarPath(cacheDir, projectId), 'utf8');
-        const entry = JSON.parse(raw);
-        const age = Date.now() - new Date(entry.savedAt).getTime();
-        if (age > maxAgeMs)
-            return null; // expired
-        return entry.storageState;
-    }
-    catch {
-        return null;
-    }
+async function loadCookieJar(cacheDir, projectId, maxAgeMs = 8 * 60 * 60 * 1e3) {
+  try {
+    const raw = await fs.readFile(jarPath(cacheDir, projectId), "utf8");
+    const entry = JSON.parse(raw);
+    const age = Date.now() - new Date(entry.savedAt).getTime();
+    if (age > maxAgeMs) return null;
+    return entry.storageState;
+  } catch {
+    return null;
+  }
 }
-export async function clearCookieJar(cacheDir, projectId) {
-    try {
-        await fs.unlink(jarPath(cacheDir, projectId));
-    }
-    catch { /* already gone */ }
+async function clearCookieJar(cacheDir, projectId) {
+  try {
+    await fs.unlink(jarPath(cacheDir, projectId));
+  } catch {
+  }
 }
-/** Merge two storageState JSON strings — second takes priority for same-domain cookies. */
-export function mergeStorageStates(base, overlay) {
-    try {
-        const a = JSON.parse(base);
-        const b = JSON.parse(overlay);
-        // Cookies: overlay replaces same-name+domain+path cookies from base
-        const cookies = [...(a.cookies ?? [])];
-        for (const ck of (b.cookies ?? [])) {
-            const idx = cookies.findIndex((c) => c.name === ck.name && c.domain === ck.domain && c.path === ck.path);
-            if (idx >= 0)
-                cookies[idx] = ck;
-            else
-                cookies.push(ck);
-        }
-        // Origins: merge origins, overlay wins on same-origin
-        const originsMap = new Map();
-        for (const o of [...(a.origins ?? []), ...(b.origins ?? [])]) {
-            originsMap.set(o.origin, o);
-        }
-        return JSON.stringify({ cookies, origins: [...originsMap.values()] });
+function mergeStorageStates(base, overlay) {
+  try {
+    const a = JSON.parse(base);
+    const b = JSON.parse(overlay);
+    const cookies = [...a.cookies ?? []];
+    for (const ck of b.cookies ?? []) {
+      const idx = cookies.findIndex(
+        (c) => c.name === ck.name && c.domain === ck.domain && c.path === ck.path
+      );
+      if (idx >= 0) cookies[idx] = ck;
+      else cookies.push(ck);
     }
-    catch {
-        return overlay || base;
+    const originsMap = /* @__PURE__ */ new Map();
+    for (const o of [...a.origins ?? [], ...b.origins ?? []]) {
+      originsMap.set(o.origin, o);
     }
+    return JSON.stringify({ cookies, origins: [...originsMap.values()] });
+  } catch {
+    return overlay || base;
+  }
 }
+export {
+  clearCookieJar,
+  loadCookieJar,
+  mergeStorageStates,
+  saveCookieJar
+};
+//# sourceMappingURL=cookie-jar.js.map