@detiq/crawler 0.1.3

package/src/captcha-solver.ts

@@ -0,0 +1,143 @@
+/**
+ * CAPTCHA auto-solving via 2captcha or CapSolver APIs.
+ * Supports reCAPTCHA v2, hCaptcha, and Cloudflare Turnstile.
+ */
+
+export type CaptchaSolverProvider = '2captcha' | 'capsolver';
+
+interface SolveResult {
+  token: string;
+  provider: CaptchaSolverProvider;
+}
+
+// Poll interval and max attempts
+const POLL_INTERVAL_MS = 3_000;
+const MAX_ATTEMPTS = 40; // 2 min max
+
+/**
+ * Detect CAPTCHA type and sitekey on the page, then solve it.
+ * Returns null if no CAPTCHA found or solver not configured.
+ * Throws on solve failure.
+ */
+export async function solveCaptcha(
+  page: any,
+  pageUrl: string,
+  apiKey: string,
+  provider: CaptchaSolverProvider = '2captcha'
+): Promise<SolveResult | null> {
+  // Detect CAPTCHA type
+  const captchaInfo = await page.evaluate(() => {
+    // reCAPTCHA v2
+    const recaptchaEl = document.querySelector('.g-recaptcha, [data-sitekey]') as HTMLElement | null;
+    if (recaptchaEl) {
+      const sitekey = recaptchaEl.getAttribute('data-sitekey') || '';
+      return { type: 'recaptcha', sitekey };
+    }
+    // hCaptcha
+    const hcaptchaEl = document.querySelector('.h-captcha, [data-hcaptcha-sitekey]') as HTMLElement | null;
+    if (hcaptchaEl) {
+      const sitekey = hcaptchaEl.getAttribute('data-sitekey') || hcaptchaEl.getAttribute('data-hcaptcha-sitekey') || '';
+      return { type: 'hcaptcha', sitekey };
+    }
+    // Cloudflare Turnstile
+    const turnstileEl = document.querySelector('.cf-turnstile') as HTMLElement | null;
+    if (turnstileEl) {
+      const sitekey = turnstileEl.getAttribute('data-sitekey') || '';
+      return { type: 'turnstile', sitekey };
+    }
+    return null;
+  });
+
+  if (!captchaInfo?.sitekey) return null;
+
+  const token = provider === 'capsolver'
+    ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey)
+    : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);
+
+  // Inject the token into the page
+  await page.evaluate((t: string, type: string) => {
+    // reCAPTCHA
+    if (type === 'recaptcha') {
+      const el = document.querySelector('textarea.g-recaptcha-response') as HTMLTextAreaElement | null;
+      if (el) { el.value = t; }
+      // Also trigger the callback if registered
+      if ((window as any).grecaptcha?.getResponse && document.querySelector('[data-callback]')) {
+        const cb = (document.querySelector('[data-callback]') as HTMLElement)?.getAttribute('data-callback');
+        if (cb && (window as any)[cb]) (window as any)[cb](t);
+      }
+    }
+    // hCaptcha
+    if (type === 'hcaptcha') {
+      const el = document.querySelector('textarea[name="h-captcha-response"]') as HTMLTextAreaElement | null;
+      if (el) { el.value = t; }
+    }
+    // Turnstile
+    if (type === 'turnstile') {
+      const el = document.querySelector('[name="cf-turnstile-response"]') as HTMLInputElement | null;
+      if (el) { el.value = t; }
+    }
+  }, token, captchaInfo.type);
+
+  return { token, provider };
+}
+
+async function solveWith2Captcha(type: string, sitekey: string, pageUrl: string, apiKey: string): Promise<string> {
+  const method = type === 'hcaptcha' ? 'hcaptcha' : type === 'turnstile' ? 'turnstile' : 'userrecaptcha';
+
+  const submitRes = await fetch('http://2captcha.com/in.php', {
+    method: 'POST',
+    body: new URLSearchParams({
+      key: apiKey,
+      method,
+      googlekey: sitekey,
+      pageurl: pageUrl,
+      json: '1',
+    }),
+  });
+  const submitData = await submitRes.json() as { status: number; request: string };
+  if (submitData.status !== 1) throw new Error(`2captcha submit failed: ${submitData.request}`);
+  const taskId = submitData.request;
+
+  // Poll for result
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+    const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);
+    const pollData = await pollRes.json() as { status: number; request: string };
+    if (pollData.status === 1) return pollData.request;
+    if (pollData.request !== 'CAPCHA_NOT_READY') throw new Error(`2captcha error: ${pollData.request}`);
+  }
+  throw new Error('2captcha timed out after 2 minutes');
+}
+
+async function solveWithCapSolver(type: string, sitekey: string, pageUrl: string, apiKey: string): Promise<string> {
+  const taskType = type === 'hcaptcha' ? 'HCaptchaTaskProxyless'
+    : type === 'turnstile' ? 'AntiTurnstileTaskProxyless'
+    : 'ReCaptchaV2TaskProxyless';
+
+  const createRes = await fetch('https://api.capsolver.com/createTask', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      clientKey: apiKey,
+      task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey },
+    }),
+  });
+  const createData = await createRes.json() as { errorId: number; taskId?: string; errorDescription?: string };
+  if (createData.errorId !== 0) throw new Error(`CapSolver create failed: ${createData.errorDescription}`);
+  const taskId = createData.taskId!;
+
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+    const pollRes = await fetch('https://api.capsolver.com/getTaskResult', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ clientKey: apiKey, taskId }),
+    });
+    const pollData = await pollRes.json() as { errorId: number; status: string; solution?: { gRecaptchaResponse?: string; token?: string }; errorDescription?: string };
+    if (pollData.errorId !== 0) throw new Error(`CapSolver error: ${pollData.errorDescription}`);
+    if (pollData.status === 'ready') {
+      return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? '';
+    }
+  }
+  throw new Error('CapSolver timed out after 2 minutes');
+}

package/src/color-contrast.ts

@@ -0,0 +1,116 @@
+/**
+ * WCAG color contrast checker.
+ *
+ * Evaluates text contrast ratios against backgrounds for all visible text elements.
+ * Reports elements that fail WCAG 2.1 AA (4.5:1 for normal text, 3:1 for large text)
+ * or AAA (7:1 for normal, 4.5:1 for large text) requirements.
+ *
+ * Uses computedStyle to get actual rendered colors — accounts for CSS variables
+ * and inheritance.
+ */
+
+import type { Page } from 'playwright';
+
+export interface ContrastViolation {
+  selector: string;
+  text: string;
+  foreground: string;   // rgb(r, g, b)
+  background: string;   // rgb(r, g, b)
+  ratio: number;        // actual contrast ratio
+  required: number;     // required ratio (4.5 or 3.0)
+  level: 'AA' | 'AAA';
+  isLargeText: boolean; // font-size >= 18px or bold >= 14px
+  severity: 'fail' | 'warn'; // fail = below AA, warn = below AAA
+}
+
+export interface ContrastAuditResult {
+  url: string;
+  violations: ContrastViolation[];
+  passCount: number;
+  score: number; // 0-100
+  auditedAt: string;
+}
+
+export async function auditColorContrast(page: Page, url: string): Promise<ContrastAuditResult> {
+  const auditedAt = new Date().toISOString();
+
+  const violations: ContrastViolation[] = await page.evaluate(() => {
+    function parseRgb(color: string): [number, number, number, number] | null {
+      const m = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?/);
+      return m ? [+m[1], +m[2], +m[3], m[4] !== undefined ? +m[4] : 1] : null;
+    }
+    function luminance([r, g, b]: [number, number, number, number]): number {
+      const toLinear = (c: number) => {
+        const n = c / 255;
+        return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);
+      };
+      return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);
+    }
+    function contrastRatio(fg: [number,number,number,number], bg: [number,number,number,number]): number {
+      const l1 = luminance(fg);
+      const l2 = luminance(bg);
+      const lighter = Math.max(l1, l2);
+      const darker = Math.min(l1, l2);
+      return Math.round(((lighter + 0.05) / (darker + 0.05)) * 100) / 100;
+    }
+
+    const violations: any[] = [];
+    const elements = Array.from(document.querySelectorAll('p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button'));
+    let checked = 0;
+
+    for (const el of elements as HTMLElement[]) {
+      if (checked >= 100) break; // limit to avoid perf hit
+      const text = (el.textContent ?? '').trim().slice(0, 60);
+      if (!text) continue;
+
+      const style = window.getComputedStyle(el);
+      if (style.display === 'none' || style.visibility === 'hidden') continue;
+
+      const fg = parseRgb(style.color);
+      const bg = parseRgb(style.backgroundColor);
+      if (!fg || !bg || bg[3] === 0) continue; // transparent bg — can't check
+
+      const fontSize = parseFloat(style.fontSize);
+      const fontWeight = style.fontWeight;
+      const isLargeText = fontSize >= 18 || (fontSize >= 14 && (fontWeight === 'bold' || +fontWeight >= 700));
+      const ratio = contrastRatio(fg, bg);
+      const aaRequired = isLargeText ? 3.0 : 4.5;
+      const aaaRequired = isLargeText ? 4.5 : 7.0;
+
+      if (ratio < aaRequired) {
+        violations.push({
+          selector: el.tagName.toLowerCase(),
+          text: text.slice(0, 50),
+          foreground: style.color,
+          background: style.backgroundColor,
+          ratio,
+          required: aaRequired,
+          level: 'AA',
+          isLargeText,
+          severity: 'fail',
+        });
+      } else if (ratio < aaaRequired) {
+        violations.push({
+          selector: el.tagName.toLowerCase(),
+          text: text.slice(0, 50),
+          foreground: style.color,
+          background: style.backgroundColor,
+          ratio,
+          required: aaaRequired,
+          level: 'AAA',
+          isLargeText,
+          severity: 'warn',
+        });
+      } else {
+        checked++;
+      }
+    }
+    return violations;
+  }).catch(() => [] as ContrastViolation[]);
+
+  const passCount = Math.max(0, 100 - violations.length);
+  const criticals = violations.filter(v => v.severity === 'fail').length;
+  const score = Math.max(0, 100 - criticals * 15 - violations.filter(v => v.severity === 'warn').length * 3);
+
+  return { url, violations, passCount, score, auditedAt };
+}

package/src/content-chunker.ts

@@ -0,0 +1,170 @@
+export type ChunkingStrategy = 'fixed' | 'overlap' | 'sentence' | 'paragraph' | 'markdown-heading' | 'semantic' | 'recursive';
+
+export interface ChunkOpts {
+  strategy: ChunkingStrategy;
+  chunkSize?: number;   // token estimate target (1 token ≈ 4 chars). Default 2000.
+  overlap?: number;     // overlap tokens for 'overlap' strategy. Default 200.
+  separators?: string[]; // for 'recursive' strategy. Default ['\\n\\n', '\\n', '. ', ' ']
+}
+
+export interface Chunk {
+  index: number;
+  text: string;
+  tokenEstimate: number;
+  startChar: number;
+  endChar: number;
+  heading?: string;
+}
+
+export function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+export function chunkContent(text: string, opts: ChunkOpts): Chunk[] {
+  const size = opts.chunkSize ?? 2000;
+  const charSize = size * 4;
+  const overlap = opts.overlap ?? 200;
+  const overlapChars = overlap * 4;
+
+  switch (opts.strategy) {
+    case 'fixed': return fixedChunk(text, charSize);
+    case 'overlap': return overlapChunk(text, charSize, overlapChars);
+    case 'sentence': return sentenceChunk(text, charSize);
+    case 'paragraph': return paragraphChunk(text, charSize);
+    case 'markdown-heading': return markdownHeadingChunk(text, charSize);
+    case 'semantic': return semanticChunk(text, charSize);
+    case 'recursive': return recursiveChunk(text, charSize, opts.separators ?? ['\n\n', '\n', '. ', ' ']);
+    default: return fixedChunk(text, charSize);
+  }
+}
+
+function makeChunk(text: string, index: number, startChar: number, heading?: string): Chunk {
+  return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };
+}
+
+function fixedChunk(text: string, charSize: number): Chunk[] {
+  const chunks: Chunk[] = [];
+  let i = 0;
+  while (i < text.length) {
+    const slice = text.slice(i, i + charSize);
+    chunks.push(makeChunk(slice, chunks.length, i));
+    i += charSize;
+  }
+  return chunks;
+}
+
+function overlapChunk(text: string, charSize: number, overlapChars: number): Chunk[] {
+  const chunks: Chunk[] = [];
+  let i = 0;
+  while (i < text.length) {
+    const slice = text.slice(i, i + charSize);
+    chunks.push(makeChunk(slice, chunks.length, i));
+    i += charSize - overlapChars;
+    if (i <= 0) break;
+  }
+  return chunks;
+}
+
+function sentenceChunk(text: string, charSize: number): Chunk[] {
+  const sentences = text.match(/[^.!?]*[.!?]+["']?\s*/g) ?? [text];
+  return groupIntoChunks(sentences, charSize);
+}
+
+function paragraphChunk(text: string, charSize: number): Chunk[] {
+  const paras = text.split(/\n\n+/).filter(p => p.trim());
+  return groupIntoChunks(paras, charSize);
+}
+
+function markdownHeadingChunk(text: string, charSize: number): Chunk[] {
+  const chunks: Chunk[] = [];
+  const headingRe = /^(#{1,3}\s.+)$/m;
+  const sections = text.split(/(?=^#{1,3}\s)/m).filter(s => s.trim());
+  let pos = 0;
+  for (const section of sections) {
+    const headingMatch = section.match(headingRe);
+    const heading = headingMatch ? headingMatch[1].replace(/^#+\s*/, '').trim() : undefined;
+    if (section.length <= charSize) {
+      chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });
+    } else {
+      // Section too large — sub-chunk by paragraph
+      const sub = paragraphChunk(section, charSize);
+      for (const s of sub) {
+        chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });
+      }
+    }
+    pos += section.length;
+  }
+  return chunks.length > 0 ? chunks : fixedChunk(text, charSize);
+}
+
+function semanticChunk(text: string, charSize: number): Chunk[] {
+  // Like paragraph but merges short paragraphs with the next
+  const paras = text.split(/\n\n+/).filter(p => p.trim());
+  const merged: string[] = [];
+  let current = '';
+  for (const para of paras) {
+    if (current.length + para.length + 2 <= charSize) {
+      current = current ? current + '\n\n' + para : para;
+    } else {
+      if (current) merged.push(current);
+      current = para;
+    }
+  }
+  if (current) merged.push(current);
+  return groupIntoChunks(merged, charSize);
+}
+
+function recursiveChunk(text: string, charSize: number, separators: string[]): Chunk[] {
+  if (text.length <= charSize) return [makeChunk(text, 0, 0)];
+  const [sep, ...rest] = separators;
+  if (!sep) return fixedChunk(text, charSize);
+  const parts = text.split(sep).filter(p => p);
+  const chunks: Chunk[] = [];
+  let pos = 0;
+  for (const part of parts) {
+    if (part.length <= charSize) {
+      chunks.push(makeChunk(part, chunks.length, pos));
+    } else {
+      const sub = recursiveChunk(part, charSize, rest);
+      for (const s of sub) chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });
+    }
+    pos += part.length + sep.length;
+  }
+  return chunks;
+}
+
+function groupIntoChunks(parts: string[], charSize: number): Chunk[] {
+  const chunks: Chunk[] = [];
+  let current = '';
+  let startChar = 0;
+  let pos = 0;
+  for (const part of parts) {
+    if (current.length + part.length + 1 > charSize && current) {
+      chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+      startChar = pos;
+      current = part;
+    } else {
+      current = current ? current + ' ' + part : part;
+    }
+    pos += part.length + 1;
+  }
+  if (current.trim()) chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+  return chunks;
+}
+
+export function mergeSmallChunks(chunks: Chunk[], minTokens: number): Chunk[] {
+  const minChars = minTokens * 4;
+  const result: Chunk[] = [];
+  let i = 0;
+  while (i < chunks.length) {
+    if (chunks[i].text.length < minChars && i + 1 < chunks.length) {
+      const merged = chunks[i].text + '\n\n' + chunks[i + 1].text;
+      result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));
+      i += 2;
+    } else {
+      result.push({ ...chunks[i], index: result.length });
+      i++;
+    }
+  }
+  return result;
+}

package/src/content-dedup.ts

@@ -0,0 +1,99 @@
+/**
+ * Content deduplication — detect near-duplicate pages during crawl.
+ *
+ * Uses MinHash-inspired approach: instead of full MinHash (needs many hash functions),
+ * we use 8 structural fingerprints extracted from the markdown to detect pages that
+ * share the same layout/template even if the data differs.
+ *
+ * Typical duplicates: /products?page=1 vs /products?page=2 (same template, different data).
+ * These waste crawl budget and generate redundant test cases.
+ */
+
+/** Extract structural fingerprints from markdown — layout-sensitive, data-insensitive */
+function extractFingerprints(markdown: string): string[] {
+  const lines = markdown.split('\n').filter(l => l.trim().length > 0);
+  const fps: string[] = [];
+
+  // FP1: heading structure (H1-H6 count distribution)
+  const headings = lines.filter(l => /^#{1,6} /.test(l));
+  const hCounts = [0, 0, 0, 0, 0, 0];
+  for (const h of headings) {
+    const lvl = h.match(/^(#{1,6}) /)?.[1].length ?? 1;
+    hCounts[lvl - 1]++;
+  }
+  fps.push('h:' + hCounts.join(','));
+
+  // FP2: link count (grouped into buckets)
+  const linkCount = (markdown.match(/\[.+?\]\(.+?\)/g) ?? []).length;
+  fps.push('lk:' + Math.floor(linkCount / 5));
+
+  // FP3: list item count (grouped)
+  const listCount = lines.filter(l => /^[\-\*\+] |^\d+\. /.test(l)).length;
+  fps.push('li:' + Math.floor(listCount / 3));
+
+  // FP4: code block presence
+  fps.push('code:' + (markdown.includes('```') ? '1' : '0'));
+
+  // FP5: table presence
+  fps.push('tbl:' + (markdown.includes('|') ? '1' : '0'));
+
+  // FP6: approximate paragraph count
+  const paraCount = markdown.split(/\n{2,}/).filter(p => p.trim().length > 30).length;
+  fps.push('p:' + Math.floor(paraCount / 2));
+
+  // FP7: first heading text (normalized)
+  const firstH = headings[0]?.replace(/^#{1,6} /, '').toLowerCase().replace(/[^a-z0-9]/g, '') ?? '';
+  fps.push('fh:' + firstH.slice(0, 20));
+
+  // FP8: word count bucket
+  const wordCount = markdown.split(/\s+/).length;
+  fps.push('wc:' + Math.floor(wordCount / 50));
+
+  return fps;
+}
+
+function similarity(a: string[], b: string[]): number {
+  if (a.length !== b.length) return 0;
+  let matches = 0;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] === b[i]) matches++;
+  }
+  return matches / a.length;
+}
+
+export interface DedupResult {
+  isDuplicate: boolean;
+  similarUrl?: string;
+  similarityScore: number;
+}
+
+export class ContentDeduplicator {
+  private seen = new Map<string, { fps: string[]; url: string }>();
+  private readonly threshold: number;
+
+  constructor(threshold = 0.85) {
+    this.threshold = threshold;
+  }
+
+  check(url: string, markdown: string): DedupResult {
+    const fps = extractFingerprints(markdown);
+
+    for (const [key, entry] of this.seen) {
+      const score = similarity(fps, entry.fps);
+      if (score >= this.threshold) {
+        return { isDuplicate: true, similarUrl: entry.url, similarityScore: score };
+      }
+    }
+
+    this.seen.set(url, { fps, url });
+    return { isDuplicate: false, similarityScore: 1.0 };
+  }
+
+  size(): number {
+    return this.seen.size;
+  }
+
+  reset(): void {
+    this.seen.clear();
+  }
+}

package/src/cookie-consent.ts

@@ -0,0 +1,106 @@
+/**
+ * Cookie consent auto-acceptor.
+ *
+ * Detects and clicks "Accept" on common cookie consent banners before the
+ * crawler proceeds. Without this, many EU sites show a consent wall that
+ * blocks content or shows a different layout, making crawl results unreliable.
+ *
+ * Supports: Cookiebot, OneTrust, TrustArc, Usercentrics, GDPR Cookie Compliance,
+ * generic "Accept All" / "Accept Cookies" buttons.
+ *
+ * Strategy: try known selectors first, then fall back to text-based button search.
+ * Non-blocking — if no banner is found or click fails, crawl proceeds normally.
+ */
+
+import type { Page } from 'playwright';
+
+export interface ConsentResult {
+  found: boolean;
+  clicked: boolean;
+  method?: string; // which selector/strategy matched
+  error?: string;
+}
+
+// Known consent platform selectors
+const CONSENT_SELECTORS = [
+  // Cookiebot
+  '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',
+  '#CybotCookiebotDialogBodyButtonAccept',
+  // OneTrust
+  '#onetrust-accept-btn-handler',
+  '.ot-sdk-accept-btn',
+  // TrustArc
+  '.truste_popframe [title*="Accept"]',
+  '#truste-consent-button',
+  // Usercentrics
+  '[data-testid="uc-accept-all-button"]',
+  // Generic high-confidence
+  '[id*="accept"][id*="cookie"]',
+  '[class*="accept"][class*="cookie"]',
+  '[class*="cookie"][class*="accept"]',
+  // GDPR Cookie Compliance
+  '.gdpr-cookie-notice-accept',
+  // Quantcast
+  '.qc-cmp2-summary-buttons button:first-child',
+];
+
+// Fallback: button text matching
+const ACCEPT_TEXT_PATTERNS = [
+  /^accept all$/i,
+  /^accept all cookies$/i,
+  /^allow all$/i,
+  /^i accept$/i,
+  /^agree$/i,
+  /^ok$/i,
+  /^got it$/i,
+  /^allow cookies$/i,
+];
+
+export async function acceptCookieConsent(page: Page, opts: { waitMs?: number } = {}): Promise<ConsentResult> {
+  const { waitMs = 1_500 } = opts;
+
+  // Wait briefly for consent banners to appear
+  await page.waitForTimeout(waitMs).catch(() => {});
+
+  // Try known selectors first
+  for (const selector of CONSENT_SELECTORS) {
+    try {
+      const el = await page.$(selector);
+      if (el && await el.isVisible()) {
+        await el.click({ timeout: 2_000 });
+        await page.waitForTimeout(500);
+        return { found: true, clicked: true, method: `selector: ${selector}` };
+      }
+    } catch { /* try next */ }
+  }
+
+  // Fallback: find buttons with accept-like text
+  try {
+    const buttons = await page.$$('button, [role="button"], a[class*="cookie"]');
+    for (const btn of buttons) {
+      const text = ((await btn.textContent()) ?? '').trim();
+      if (ACCEPT_TEXT_PATTERNS.some(re => re.test(text))) {
+        if (await btn.isVisible()) {
+          await btn.click({ timeout: 2_000 });
+          await page.waitForTimeout(500);
+          return { found: true, clicked: true, method: `text: "${text}"` };
+        }
+      }
+    }
+  } catch { /* non-fatal */ }
+
+  return { found: false, clicked: false };
+}
+
+/** Check if a consent banner is still visible after attempted dismissal */
+export async function isConsentBannerVisible(page: Page): Promise<boolean> {
+  try {
+    for (const sel of CONSENT_SELECTORS.slice(0, 5)) {
+      const el = await page.$(sel);
+      if (el && await el.isVisible()) return true;
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}