@detiq/crawler 1.1.7 → 1.1.9

package/dist/cookie-consent.js

@@ -1,78 +1,93 @@
+/**
+ * Cookie consent auto-acceptor.
+ *
+ * Detects and clicks "Accept" on common cookie consent banners before the
+ * crawler proceeds. Without this, many EU sites show a consent wall that
+ * blocks content or shows a different layout, making crawl results unreliable.
+ *
+ * Supports: Cookiebot, OneTrust, TrustArc, Usercentrics, GDPR Cookie Compliance,
+ * generic "Accept All" / "Accept Cookies" buttons.
+ *
+ * Strategy: try known selectors first, then fall back to text-based button search.
+ * Non-blocking — if no banner is found or click fails, crawl proceeds normally.
+ */
+// Known consent platform selectors
 const CONSENT_SELECTORS = [
-  // Cookiebot
-  "#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll",
-  "#CybotCookiebotDialogBodyButtonAccept",
-  // OneTrust
-  "#onetrust-accept-btn-handler",
-  ".ot-sdk-accept-btn",
-  // TrustArc
-  '.truste_popframe [title*="Accept"]',
-  "#truste-consent-button",
-  // Usercentrics
-  '[data-testid="uc-accept-all-button"]',
-  // Generic high-confidence
-  '[id*="accept"][id*="cookie"]',
-  '[class*="accept"][class*="cookie"]',
-  '[class*="cookie"][class*="accept"]',
-  // GDPR Cookie Compliance
-  ".gdpr-cookie-notice-accept",
-  // Quantcast
-  ".qc-cmp2-summary-buttons button:first-child"
+    // Cookiebot
+    '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',
+    '#CybotCookiebotDialogBodyButtonAccept',
+    // OneTrust
+    '#onetrust-accept-btn-handler',
+    '.ot-sdk-accept-btn',
+    // TrustArc
+    '.truste_popframe [title*="Accept"]',
+    '#truste-consent-button',
+    // Usercentrics
+    '[data-testid="uc-accept-all-button"]',
+    // Generic high-confidence
+    '[id*="accept"][id*="cookie"]',
+    '[class*="accept"][class*="cookie"]',
+    '[class*="cookie"][class*="accept"]',
+    // GDPR Cookie Compliance
+    '.gdpr-cookie-notice-accept',
+    // Quantcast
+    '.qc-cmp2-summary-buttons button:first-child',
 ];
+// Fallback: button text matching
 const ACCEPT_TEXT_PATTERNS = [
-  /^accept all$/i,
-  /^accept all cookies$/i,
-  /^allow all$/i,
-  /^i accept$/i,
-  /^agree$/i,
-  /^ok$/i,
-  /^got it$/i,
-  /^allow cookies$/i
+    /^accept all$/i,
+    /^accept all cookies$/i,
+    /^allow all$/i,
+    /^i accept$/i,
+    /^agree$/i,
+    /^ok$/i,
+    /^got it$/i,
+    /^allow cookies$/i,
 ];
-async function acceptCookieConsent(page, opts = {}) {
-  const { waitMs = 1500 } = opts;
-  await page.waitForTimeout(waitMs).catch(() => {
-  });
-  for (const selector of CONSENT_SELECTORS) {
-    try {
-      const el = await page.$(selector);
-      if (el && await el.isVisible()) {
-        await el.click({ timeout: 2e3 });
-        await page.waitForTimeout(500);
-        return { found: true, clicked: true, method: `selector: ${selector}` };
-      }
-    } catch {
+export async function acceptCookieConsent(page, opts = {}) {
+    const { waitMs = 1_500 } = opts;
+    // Wait briefly for consent banners to appear
+    await page.waitForTimeout(waitMs).catch(() => { });
+    // Try known selectors first
+    for (const selector of CONSENT_SELECTORS) {
+        try {
+            const el = await page.$(selector);
+            if (el && await el.isVisible()) {
+                await el.click({ timeout: 2_000 });
+                await page.waitForTimeout(500);
+                return { found: true, clicked: true, method: `selector: ${selector}` };
+            }
+        }
+        catch { /* try next */ }
     }
-  }
-  try {
-    const buttons = await page.$$('button, [role="button"], a[class*="cookie"]');
-    for (const btn of buttons) {
-      const text = (await btn.textContent() ?? "").trim();
-      if (ACCEPT_TEXT_PATTERNS.some((re) => re.test(text))) {
-        if (await btn.isVisible()) {
-          await btn.click({ timeout: 2e3 });
-          await page.waitForTimeout(500);
-          return { found: true, clicked: true, method: `text: "${text}"` };
+    // Fallback: find buttons with accept-like text
+    try {
+        const buttons = await page.$$('button, [role="button"], a[class*="cookie"]');
+        for (const btn of buttons) {
+            const text = ((await btn.textContent()) ?? '').trim();
+            if (ACCEPT_TEXT_PATTERNS.some(re => re.test(text))) {
+                if (await btn.isVisible()) {
+                    await btn.click({ timeout: 2_000 });
+                    await page.waitForTimeout(500);
+                    return { found: true, clicked: true, method: `text: "${text}"` };
+                }
+            }
         }
-      }
     }
-  } catch {
-  }
-  return { found: false, clicked: false };
+    catch { /* non-fatal */ }
+    return { found: false, clicked: false };
 }
-async function isConsentBannerVisible(page) {
-  try {
-    for (const sel of CONSENT_SELECTORS.slice(0, 5)) {
-      const el = await page.$(sel);
-      if (el && await el.isVisible()) return true;
+/** Check if a consent banner is still visible after attempted dismissal */
+export async function isConsentBannerVisible(page) {
+    try {
+        for (const sel of CONSENT_SELECTORS.slice(0, 5)) {
+            const el = await page.$(sel);
+            if (el && await el.isVisible())
+                return true;
+        }
+        return false;
+    }
+    catch {
+        return false;
     }
-    return false;
-  } catch {
-    return false;
-  }
 }
-export {
-  acceptCookieConsent,
-  isConsentBannerVisible
-};
-//# sourceMappingURL=cookie-consent.js.map

package/dist/cookie-jar.js

@@ -1,61 +1,68 @@
-import * as fs from "fs/promises";
-import * as path from "path";
+/**
+ * Cookie jar — persist and restore Playwright storageState cookies between
+ * crawl segments and restarts. Enables continuous auth across session boundaries
+ * (e.g. segment 1 logs in, segment 2–N inherit the session without re-login).
+ *
+ * Storage: JSON file per project in the crawl cache dir.
+ * Format: Playwright's storageState shape (cookies + origins).
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
 function jarPath(cacheDir, projectId) {
-  return path.join(cacheDir, `cookie-jar-${projectId}.json`);
+    return path.join(cacheDir, `cookie-jar-${projectId}.json`);
 }
-async function saveCookieJar(cacheDir, projectId, tenantId, storageState, domain) {
-  await fs.mkdir(cacheDir, { recursive: true });
-  const entry = {
-    projectId,
-    tenantId,
-    storageState,
-    savedAt: (/* @__PURE__ */ new Date()).toISOString(),
-    domain
-  };
-  await fs.writeFile(jarPath(cacheDir, projectId), JSON.stringify(entry, null, 2), "utf8");
+export async function saveCookieJar(cacheDir, projectId, tenantId, storageState, domain) {
+    await fs.mkdir(cacheDir, { recursive: true });
+    const entry = {
+        projectId,
+        tenantId,
+        storageState,
+        savedAt: new Date().toISOString(),
+        domain,
+    };
+    await fs.writeFile(jarPath(cacheDir, projectId), JSON.stringify(entry, null, 2), 'utf8');
 }
-async function loadCookieJar(cacheDir, projectId, maxAgeMs = 8 * 60 * 60 * 1e3) {
-  try {
-    const raw = await fs.readFile(jarPath(cacheDir, projectId), "utf8");
-    const entry = JSON.parse(raw);
-    const age = Date.now() - new Date(entry.savedAt).getTime();
-    if (age > maxAgeMs) return null;
-    return entry.storageState;
-  } catch {
-    return null;
-  }
+export async function loadCookieJar(cacheDir, projectId, maxAgeMs = 8 * 60 * 60 * 1000) {
+    try {
+        const raw = await fs.readFile(jarPath(cacheDir, projectId), 'utf8');
+        const entry = JSON.parse(raw);
+        const age = Date.now() - new Date(entry.savedAt).getTime();
+        if (age > maxAgeMs)
+            return null; // expired
+        return entry.storageState;
+    }
+    catch {
+        return null;
+    }
 }
-async function clearCookieJar(cacheDir, projectId) {
-  try {
-    await fs.unlink(jarPath(cacheDir, projectId));
-  } catch {
-  }
+export async function clearCookieJar(cacheDir, projectId) {
+    try {
+        await fs.unlink(jarPath(cacheDir, projectId));
+    }
+    catch { /* already gone */ }
 }
-function mergeStorageStates(base, overlay) {
-  try {
-    const a = JSON.parse(base);
-    const b = JSON.parse(overlay);
-    const cookies = [...a.cookies ?? []];
-    for (const ck of b.cookies ?? []) {
-      const idx = cookies.findIndex(
-        (c) => c.name === ck.name && c.domain === ck.domain && c.path === ck.path
-      );
-      if (idx >= 0) cookies[idx] = ck;
-      else cookies.push(ck);
+/** Merge two storageState JSON strings — second takes priority for same-domain cookies. */
+export function mergeStorageStates(base, overlay) {
+    try {
+        const a = JSON.parse(base);
+        const b = JSON.parse(overlay);
+        // Cookies: overlay replaces same-name+domain+path cookies from base
+        const cookies = [...(a.cookies ?? [])];
+        for (const ck of (b.cookies ?? [])) {
+            const idx = cookies.findIndex((c) => c.name === ck.name && c.domain === ck.domain && c.path === ck.path);
+            if (idx >= 0)
+                cookies[idx] = ck;
+            else
+                cookies.push(ck);
+        }
+        // Origins: merge origins, overlay wins on same-origin
+        const originsMap = new Map();
+        for (const o of [...(a.origins ?? []), ...(b.origins ?? [])]) {
+            originsMap.set(o.origin, o);
+        }
+        return JSON.stringify({ cookies, origins: [...originsMap.values()] });
     }
-    const originsMap = /* @__PURE__ */ new Map();
-    for (const o of [...a.origins ?? [], ...b.origins ?? []]) {
-      originsMap.set(o.origin, o);
+    catch {
+        return overlay || base;
     }
-    return JSON.stringify({ cookies, origins: [...originsMap.values()] });
-  } catch {
-    return overlay || base;
-  }
 }
-export {
-  clearCookieJar,
-  loadCookieJar,
-  mergeStorageStates,
-  saveCookieJar
-};
-//# sourceMappingURL=cookie-jar.js.map