@detiq/crawler 0.1.3

package/dist/captcha-solver.js

@@ -0,0 +1,105 @@
+const POLL_INTERVAL_MS = 3e3;
+const MAX_ATTEMPTS = 40;
+async function solveCaptcha(page, pageUrl, apiKey, provider = "2captcha") {
+  const captchaInfo = await page.evaluate(() => {
+    const recaptchaEl = document.querySelector(".g-recaptcha, [data-sitekey]");
+    if (recaptchaEl) {
+      const sitekey = recaptchaEl.getAttribute("data-sitekey") || "";
+      return { type: "recaptcha", sitekey };
+    }
+    const hcaptchaEl = document.querySelector(".h-captcha, [data-hcaptcha-sitekey]");
+    if (hcaptchaEl) {
+      const sitekey = hcaptchaEl.getAttribute("data-sitekey") || hcaptchaEl.getAttribute("data-hcaptcha-sitekey") || "";
+      return { type: "hcaptcha", sitekey };
+    }
+    const turnstileEl = document.querySelector(".cf-turnstile");
+    if (turnstileEl) {
+      const sitekey = turnstileEl.getAttribute("data-sitekey") || "";
+      return { type: "turnstile", sitekey };
+    }
+    return null;
+  });
+  if (!captchaInfo?.sitekey) return null;
+  const token = provider === "capsolver" ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey) : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);
+  await page.evaluate((t, type) => {
+    if (type === "recaptcha") {
+      const el = document.querySelector("textarea.g-recaptcha-response");
+      if (el) {
+        el.value = t;
+      }
+      if (window.grecaptcha?.getResponse && document.querySelector("[data-callback]")) {
+        const cb = document.querySelector("[data-callback]")?.getAttribute("data-callback");
+        if (cb && window[cb]) window[cb](t);
+      }
+    }
+    if (type === "hcaptcha") {
+      const el = document.querySelector('textarea[name="h-captcha-response"]');
+      if (el) {
+        el.value = t;
+      }
+    }
+    if (type === "turnstile") {
+      const el = document.querySelector('[name="cf-turnstile-response"]');
+      if (el) {
+        el.value = t;
+      }
+    }
+  }, token, captchaInfo.type);
+  return { token, provider };
+}
+async function solveWith2Captcha(type, sitekey, pageUrl, apiKey) {
+  const method = type === "hcaptcha" ? "hcaptcha" : type === "turnstile" ? "turnstile" : "userrecaptcha";
+  const submitRes = await fetch("http://2captcha.com/in.php", {
+    method: "POST",
+    body: new URLSearchParams({
+      key: apiKey,
+      method,
+      googlekey: sitekey,
+      pageurl: pageUrl,
+      json: "1"
+    })
+  });
+  const submitData = await submitRes.json();
+  if (submitData.status !== 1) throw new Error(`2captcha submit failed: ${submitData.request}`);
+  const taskId = submitData.request;
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+    const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);
+    const pollData = await pollRes.json();
+    if (pollData.status === 1) return pollData.request;
+    if (pollData.request !== "CAPCHA_NOT_READY") throw new Error(`2captcha error: ${pollData.request}`);
+  }
+  throw new Error("2captcha timed out after 2 minutes");
+}
+async function solveWithCapSolver(type, sitekey, pageUrl, apiKey) {
+  const taskType = type === "hcaptcha" ? "HCaptchaTaskProxyless" : type === "turnstile" ? "AntiTurnstileTaskProxyless" : "ReCaptchaV2TaskProxyless";
+  const createRes = await fetch("https://api.capsolver.com/createTask", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      clientKey: apiKey,
+      task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey }
+    })
+  });
+  const createData = await createRes.json();
+  if (createData.errorId !== 0) throw new Error(`CapSolver create failed: ${createData.errorDescription}`);
+  const taskId = createData.taskId;
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+    const pollRes = await fetch("https://api.capsolver.com/getTaskResult", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ clientKey: apiKey, taskId })
+    });
+    const pollData = await pollRes.json();
+    if (pollData.errorId !== 0) throw new Error(`CapSolver error: ${pollData.errorDescription}`);
+    if (pollData.status === "ready") {
+      return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? "";
+    }
+  }
+  throw new Error("CapSolver timed out after 2 minutes");
+}
+export {
+  solveCaptcha
+};
+//# sourceMappingURL=captcha-solver.js.map

package/dist/captcha-solver.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/captcha-solver.ts"],
+  "sourcesContent": ["/**\n * CAPTCHA auto-solving via 2captcha or CapSolver APIs.\n * Supports reCAPTCHA v2, hCaptcha, and Cloudflare Turnstile.\n */\n\nexport type CaptchaSolverProvider = '2captcha' | 'capsolver';\n\ninterface SolveResult {\n  token: string;\n  provider: CaptchaSolverProvider;\n}\n\n// Poll interval and max attempts\nconst POLL_INTERVAL_MS = 3_000;\nconst MAX_ATTEMPTS = 40; // 2 min max\n\n/**\n * Detect CAPTCHA type and sitekey on the page, then solve it.\n * Returns null if no CAPTCHA found or solver not configured.\n * Throws on solve failure.\n */\nexport async function solveCaptcha(\n  page: any,\n  pageUrl: string,\n  apiKey: string,\n  provider: CaptchaSolverProvider = '2captcha'\n): Promise<SolveResult | null> {\n  // Detect CAPTCHA type\n  const captchaInfo = await page.evaluate(() => {\n    // reCAPTCHA v2\n    const recaptchaEl = document.querySelector('.g-recaptcha, [data-sitekey]') as HTMLElement | null;\n    if (recaptchaEl) {\n      const sitekey = recaptchaEl.getAttribute('data-sitekey') || '';\n      return { type: 'recaptcha', sitekey };\n    }\n    // hCaptcha\n    const hcaptchaEl = document.querySelector('.h-captcha, [data-hcaptcha-sitekey]') as HTMLElement | null;\n    if (hcaptchaEl) {\n      const sitekey = hcaptchaEl.getAttribute('data-sitekey') || hcaptchaEl.getAttribute('data-hcaptcha-sitekey') || '';\n      return { type: 'hcaptcha', sitekey };\n    }\n    // Cloudflare Turnstile\n    const turnstileEl = document.querySelector('.cf-turnstile') as HTMLElement | null;\n    if (turnstileEl) {\n      const sitekey = turnstileEl.getAttribute('data-sitekey') || '';\n      return { type: 'turnstile', sitekey };\n    }\n    return null;\n  });\n\n  if (!captchaInfo?.sitekey) return null;\n\n  const token = provider === 'capsolver'\n    ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey)\n    : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);\n\n  // Inject the token into the page\n  await page.evaluate((t: string, type: string) => {\n    // reCAPTCHA\n    if (type === 'recaptcha') {\n      const el = document.querySelector('textarea.g-recaptcha-response') as HTMLTextAreaElement | null;\n      if (el) { el.value = t; }\n      // Also trigger the callback if registered\n      if ((window as any).grecaptcha?.getResponse && document.querySelector('[data-callback]')) {\n        const cb = (document.querySelector('[data-callback]') as HTMLElement)?.getAttribute('data-callback');\n        if (cb && (window as any)[cb]) (window as any)[cb](t);\n      }\n    }\n    // hCaptcha\n    if (type === 'hcaptcha') {\n      const el = document.querySelector('textarea[name=\"h-captcha-response\"]') as HTMLTextAreaElement | null;\n      if (el) { el.value = t; }\n    }\n    // Turnstile\n    if (type === 'turnstile') {\n      const el = document.querySelector('[name=\"cf-turnstile-response\"]') as HTMLInputElement | null;\n      if (el) { el.value = t; }\n    }\n  }, token, captchaInfo.type);\n\n  return { token, provider };\n}\n\nasync function solveWith2Captcha(type: string, sitekey: string, pageUrl: string, apiKey: string): Promise<string> {\n  const method = type === 'hcaptcha' ? 'hcaptcha' : type === 'turnstile' ? 'turnstile' : 'userrecaptcha';\n\n  const submitRes = await fetch('http://2captcha.com/in.php', {\n    method: 'POST',\n    body: new URLSearchParams({\n      key: apiKey,\n      method,\n      googlekey: sitekey,\n      pageurl: pageUrl,\n      json: '1',\n    }),\n  });\n  const submitData = await submitRes.json() as { status: number; request: string };\n  if (submitData.status !== 1) throw new Error(`2captcha submit failed: ${submitData.request}`);\n  const taskId = submitData.request;\n\n  // Poll for result\n  for (let i = 0; i < MAX_ATTEMPTS; i++) {\n    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));\n    const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);\n    const pollData = await pollRes.json() as { status: number; request: string };\n    if (pollData.status === 1) return pollData.request;\n    if (pollData.request !== 'CAPCHA_NOT_READY') throw new Error(`2captcha error: ${pollData.request}`);\n  }\n  throw new Error('2captcha timed out after 2 minutes');\n}\n\nasync function solveWithCapSolver(type: string, sitekey: string, pageUrl: string, apiKey: string): Promise<string> {\n  const taskType = type === 'hcaptcha' ? 'HCaptchaTaskProxyless'\n    : type === 'turnstile' ? 'AntiTurnstileTaskProxyless'\n    : 'ReCaptchaV2TaskProxyless';\n\n  const createRes = await fetch('https://api.capsolver.com/createTask', {\n    method: 'POST',\n    headers: { 'Content-Type': 'application/json' },\n    body: JSON.stringify({\n      clientKey: apiKey,\n      task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey },\n    }),\n  });\n  const createData = await createRes.json() as { errorId: number; taskId?: string; errorDescription?: string };\n  if (createData.errorId !== 0) throw new Error(`CapSolver create failed: ${createData.errorDescription}`);\n  const taskId = createData.taskId!;\n\n  for (let i = 0; i < MAX_ATTEMPTS; i++) {\n    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));\n    const pollRes = await fetch('https://api.capsolver.com/getTaskResult', {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({ clientKey: apiKey, taskId }),\n    });\n    const pollData = await pollRes.json() as { errorId: number; status: string; solution?: { gRecaptchaResponse?: string; token?: string }; errorDescription?: string };\n    if (pollData.errorId !== 0) throw new Error(`CapSolver error: ${pollData.errorDescription}`);\n    if (pollData.status === 'ready') {\n      return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? '';\n    }\n  }\n  throw new Error('CapSolver timed out after 2 minutes');\n}\n"],
+  "mappings": "AAaA,MAAM,mBAAmB;AACzB,MAAM,eAAe;AAOrB,eAAsB,aACpB,MACA,SACA,QACA,WAAkC,YACL;AAE7B,QAAM,cAAc,MAAM,KAAK,SAAS,MAAM;AAE5C,UAAM,cAAc,SAAS,cAAc,8BAA8B;AACzE,QAAI,aAAa;AACf,YAAM,UAAU,YAAY,aAAa,cAAc,KAAK;AAC5D,aAAO,EAAE,MAAM,aAAa,QAAQ;AAAA,IACtC;AAEA,UAAM,aAAa,SAAS,cAAc,qCAAqC;AAC/E,QAAI,YAAY;AACd,YAAM,UAAU,WAAW,aAAa,cAAc,KAAK,WAAW,aAAa,uBAAuB,KAAK;AAC/G,aAAO,EAAE,MAAM,YAAY,QAAQ;AAAA,IACrC;AAEA,UAAM,cAAc,SAAS,cAAc,eAAe;AAC1D,QAAI,aAAa;AACf,YAAM,UAAU,YAAY,aAAa,cAAc,KAAK;AAC5D,aAAO,EAAE,MAAM,aAAa,QAAQ;AAAA,IACtC;AACA,WAAO;AAAA,EACT,CAAC;AAED,MAAI,CAAC,aAAa,QAAS,QAAO;AAElC,QAAM,QAAQ,aAAa,cACvB,MAAM,mBAAmB,YAAY,MAAM,YAAY,SAAS,SAAS,MAAM,IAC/E,MAAM,kBAAkB,YAAY,MAAM,YAAY,SAAS,SAAS,MAAM;AAGlF,QAAM,KAAK,SAAS,CAAC,GAAW,SAAiB;AAE/C,QAAI,SAAS,aAAa;AACxB,YAAM,KAAK,SAAS,cAAc,+BAA+B;AACjE,UAAI,IAAI;AAAE,WAAG,QAAQ;AAAA,MAAG;AAExB,UAAK,OAAe,YAAY,eAAe,SAAS,cAAc,iBAAiB,GAAG;AACxF,cAAM,KAAM,SAAS,cAAc,iBAAiB,GAAmB,aAAa,eAAe;AACnG,YAAI,MAAO,OAAe,EAAE,EAAG,CAAC,OAAe,EAAE,EAAE,CAAC;AAAA,MACtD;AAAA,IACF;AAEA,QAAI,SAAS,YAAY;AACvB,YAAM,KAAK,SAAS,cAAc,qCAAqC;AACvE,UAAI,IAAI;AAAE,WAAG,QAAQ;AAAA,MAAG;AAAA,IAC1B;AAEA,QAAI,SAAS,aAAa;AACxB,YAAM,KAAK,SAAS,cAAc,gCAAgC;AAClE,UAAI,IAAI;AAAE,WAAG,QAAQ;AAAA,MAAG;AAAA,IAC1B;AAAA,EACF,GAAG,OAAO,YAAY,IAAI;AAE1B,SAAO,EAAE,OAAO,SAAS;AAC3B;AAEA,eAAe,kBAAkB,MAAc,SAAiB,SAAiB,QAAiC;AAChH,QAAM,SAAS,SAAS,aAAa,aAAa,SAAS,cAAc,cAAc;AAEvF,QAAM,YAAY,MAAM,MAAM,8BAA8B;AAAA,IAC1D,QAAQ;AAAA,IACR,MAAM,IAAI,gBAAgB;AAAA,MACxB,KAAK;AAAA,MACL;AAAA,MACA,WAAW;AAAA,MACX,SAAS;AAAA,MACT,MAAM;AAAA,IACR,CAAC;AAAA,EACH,CAAC;AACD,QAAM,aAAa,MAAM,UAAU,KAAK;AACxC,MAAI,WAAW,WAAW,EAAG,OAAM,IAAI,MAAM,2BAA2B,WAAW,OAAO,EAAE;AAC5F,QAAM,SAAS,WAAW;AAG1B,WAAS,IAAI,GAAG,IAAI,cAAc,KAAK;AACrC,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,gBAAgB,CAAC;AACtD,UAAM,UAAU,MAAM,MAAM,mCAAmC,MAAM,kBAAkB,MAAM,SAAS;AACtG,UAAM,WAAW,MAAM,QAAQ,KAAK;AACpC,QAAI,SAAS,WAAW,EAAG,QAAO,SAAS;AAC3C,QAAI,SAAS,YAAY,mBAAoB,OAAM,IAAI,MAAM,mBAAmB,SAAS,OAAO,EAAE;AAAA,EACpG;AACA,QAAM,IAAI,MAAM,oCAAoC;AACtD;AAEA,eAAe,mBAAmB,MAAc,SAAiB,SAAiB,QAAiC;AACjH,QAAM,WAAW,SAAS,aAAa,0BACnC,SAAS,cAAc,+BACvB;AAEJ,QAAM,YAAY,MAAM,MAAM,wCAAwC;AAAA,IACpE,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,IAC9C,MAAM,KAAK,UAAU;AAAA,MACnB,WAAW;AAAA,MACX,MAAM,EAAE,MAAM,UAAU,YAAY,SAAS,YAAY,QAAQ;AAAA,IACnE,CAAC;AAAA,EACH,CAAC;AACD,QAAM,aAAa,MAAM,UAAU,KAAK;AACxC,MAAI,WAAW,YAAY,EAAG,OAAM,IAAI,MAAM,4BAA4B,WAAW,gBAAgB,EAAE;AACvG,QAAM,SAAS,WAAW;AAE1B,WAAS,IAAI,GAAG,IAAI,cAAc,KAAK;AACrC,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,gBAAgB,CAAC;AACtD,UAAM,UAAU,MAAM,MAAM,2CAA2C;AAAA,MACrE,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU,EAAE,WAAW,QAAQ,OAAO,CAAC;AAAA,IACpD,CAAC;AACD,UAAM,WAAW,MAAM,QAAQ,KAAK;AACpC,QAAI,SAAS,YAAY,EAAG,OAAM,IAAI,MAAM,oBAAoB,SAAS,gBAAgB,EAAE;AAC3F,QAAI,SAAS,WAAW,SAAS;AAC/B,aAAO,SAAS,UAAU,sBAAsB,SAAS,UAAU,SAAS;AAAA,IAC9E;AAAA,EACF;AACA,QAAM,IAAI,MAAM,qCAAqC;AACvD;",
+  "names": []
+}

package/dist/color-contrast.d.ts

@@ -0,0 +1,30 @@
+/**
+ * WCAG color contrast checker.
+ *
+ * Evaluates text contrast ratios against backgrounds for all visible text elements.
+ * Reports elements that fail WCAG 2.1 AA (4.5:1 for normal text, 3:1 for large text)
+ * or AAA (7:1 for normal, 4.5:1 for large text) requirements.
+ *
+ * Uses computedStyle to get actual rendered colors — accounts for CSS variables
+ * and inheritance.
+ */
+import type { Page } from 'playwright';
+export interface ContrastViolation {
+    selector: string;
+    text: string;
+    foreground: string;
+    background: string;
+    ratio: number;
+    required: number;
+    level: 'AA' | 'AAA';
+    isLargeText: boolean;
+    severity: 'fail' | 'warn';
+}
+export interface ContrastAuditResult {
+    url: string;
+    violations: ContrastViolation[];
+    passCount: number;
+    score: number;
+    auditedAt: string;
+}
+export declare function auditColorContrast(page: Page, url: string): Promise<ContrastAuditResult>;

package/dist/color-contrast.js

@@ -0,0 +1,78 @@
+async function auditColorContrast(page, url) {
+  const auditedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const violations = await page.evaluate(() => {
+    function parseRgb(color) {
+      const m = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?/);
+      return m ? [+m[1], +m[2], +m[3], m[4] !== void 0 ? +m[4] : 1] : null;
+    }
+    function luminance([r, g, b]) {
+      const toLinear = (c) => {
+        const n = c / 255;
+        return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);
+      };
+      return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);
+    }
+    function contrastRatio(fg, bg) {
+      const l1 = luminance(fg);
+      const l2 = luminance(bg);
+      const lighter = Math.max(l1, l2);
+      const darker = Math.min(l1, l2);
+      return Math.round((lighter + 0.05) / (darker + 0.05) * 100) / 100;
+    }
+    const violations2 = [];
+    const elements = Array.from(document.querySelectorAll("p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button"));
+    let checked = 0;
+    for (const el of elements) {
+      if (checked >= 100) break;
+      const text = (el.textContent ?? "").trim().slice(0, 60);
+      if (!text) continue;
+      const style = window.getComputedStyle(el);
+      if (style.display === "none" || style.visibility === "hidden") continue;
+      const fg = parseRgb(style.color);
+      const bg = parseRgb(style.backgroundColor);
+      if (!fg || !bg || bg[3] === 0) continue;
+      const fontSize = parseFloat(style.fontSize);
+      const fontWeight = style.fontWeight;
+      const isLargeText = fontSize >= 18 || fontSize >= 14 && (fontWeight === "bold" || +fontWeight >= 700);
+      const ratio = contrastRatio(fg, bg);
+      const aaRequired = isLargeText ? 3 : 4.5;
+      const aaaRequired = isLargeText ? 4.5 : 7;
+      if (ratio < aaRequired) {
+        violations2.push({
+          selector: el.tagName.toLowerCase(),
+          text: text.slice(0, 50),
+          foreground: style.color,
+          background: style.backgroundColor,
+          ratio,
+          required: aaRequired,
+          level: "AA",
+          isLargeText,
+          severity: "fail"
+        });
+      } else if (ratio < aaaRequired) {
+        violations2.push({
+          selector: el.tagName.toLowerCase(),
+          text: text.slice(0, 50),
+          foreground: style.color,
+          background: style.backgroundColor,
+          ratio,
+          required: aaaRequired,
+          level: "AAA",
+          isLargeText,
+          severity: "warn"
+        });
+      } else {
+        checked++;
+      }
+    }
+    return violations2;
+  }).catch(() => []);
+  const passCount = Math.max(0, 100 - violations.length);
+  const criticals = violations.filter((v) => v.severity === "fail").length;
+  const score = Math.max(0, 100 - criticals * 15 - violations.filter((v) => v.severity === "warn").length * 3);
+  return { url, violations, passCount, score, auditedAt };
+}
+export {
+  auditColorContrast
+};
+//# sourceMappingURL=color-contrast.js.map

package/dist/color-contrast.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/color-contrast.ts"],
+  "sourcesContent": ["/**\n * WCAG color contrast checker.\n *\n * Evaluates text contrast ratios against backgrounds for all visible text elements.\n * Reports elements that fail WCAG 2.1 AA (4.5:1 for normal text, 3:1 for large text)\n * or AAA (7:1 for normal, 4.5:1 for large text) requirements.\n *\n * Uses computedStyle to get actual rendered colors \u2014 accounts for CSS variables\n * and inheritance.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface ContrastViolation {\n  selector: string;\n  text: string;\n  foreground: string;   // rgb(r, g, b)\n  background: string;   // rgb(r, g, b)\n  ratio: number;        // actual contrast ratio\n  required: number;     // required ratio (4.5 or 3.0)\n  level: 'AA' | 'AAA';\n  isLargeText: boolean; // font-size >= 18px or bold >= 14px\n  severity: 'fail' | 'warn'; // fail = below AA, warn = below AAA\n}\n\nexport interface ContrastAuditResult {\n  url: string;\n  violations: ContrastViolation[];\n  passCount: number;\n  score: number; // 0-100\n  auditedAt: string;\n}\n\nexport async function auditColorContrast(page: Page, url: string): Promise<ContrastAuditResult> {\n  const auditedAt = new Date().toISOString();\n\n  const violations: ContrastViolation[] = await page.evaluate(() => {\n    function parseRgb(color: string): [number, number, number, number] | null {\n      const m = color.match(/rgba?\\((\\d+),\\s*(\\d+),\\s*(\\d+)(?:,\\s*([\\d.]+))?/);\n      return m ? [+m[1], +m[2], +m[3], m[4] !== undefined ? +m[4] : 1] : null;\n    }\n    function luminance([r, g, b]: [number, number, number, number]): number {\n      const toLinear = (c: number) => {\n        const n = c / 255;\n        return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);\n      };\n      return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);\n    }\n    function contrastRatio(fg: [number,number,number,number], bg: [number,number,number,number]): number {\n      const l1 = luminance(fg);\n      const l2 = luminance(bg);\n      const lighter = Math.max(l1, l2);\n      const darker = Math.min(l1, l2);\n      return Math.round(((lighter + 0.05) / (darker + 0.05)) * 100) / 100;\n    }\n\n    const violations: any[] = [];\n    const elements = Array.from(document.querySelectorAll('p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button'));\n    let checked = 0;\n\n    for (const el of elements as HTMLElement[]) {\n      if (checked >= 100) break; // limit to avoid perf hit\n      const text = (el.textContent ?? '').trim().slice(0, 60);\n      if (!text) continue;\n\n      const style = window.getComputedStyle(el);\n      if (style.display === 'none' || style.visibility === 'hidden') continue;\n\n      const fg = parseRgb(style.color);\n      const bg = parseRgb(style.backgroundColor);\n      if (!fg || !bg || bg[3] === 0) continue; // transparent bg \u2014 can't check\n\n      const fontSize = parseFloat(style.fontSize);\n      const fontWeight = style.fontWeight;\n      const isLargeText = fontSize >= 18 || (fontSize >= 14 && (fontWeight === 'bold' || +fontWeight >= 700));\n      const ratio = contrastRatio(fg, bg);\n      const aaRequired = isLargeText ? 3.0 : 4.5;\n      const aaaRequired = isLargeText ? 4.5 : 7.0;\n\n      if (ratio < aaRequired) {\n        violations.push({\n          selector: el.tagName.toLowerCase(),\n          text: text.slice(0, 50),\n          foreground: style.color,\n          background: style.backgroundColor,\n          ratio,\n          required: aaRequired,\n          level: 'AA',\n          isLargeText,\n          severity: 'fail',\n        });\n      } else if (ratio < aaaRequired) {\n        violations.push({\n          selector: el.tagName.toLowerCase(),\n          text: text.slice(0, 50),\n          foreground: style.color,\n          background: style.backgroundColor,\n          ratio,\n          required: aaaRequired,\n          level: 'AAA',\n          isLargeText,\n          severity: 'warn',\n        });\n      } else {\n        checked++;\n      }\n    }\n    return violations;\n  }).catch(() => [] as ContrastViolation[]);\n\n  const passCount = Math.max(0, 100 - violations.length);\n  const criticals = violations.filter(v => v.severity === 'fail').length;\n  const score = Math.max(0, 100 - criticals * 15 - violations.filter(v => v.severity === 'warn').length * 3);\n\n  return { url, violations, passCount, score, auditedAt };\n}\n"],
+  "mappings": "AAiCA,eAAsB,mBAAmB,MAAY,KAA2C;AAC9F,QAAM,aAAY,oBAAI,KAAK,GAAE,YAAY;AAEzC,QAAM,aAAkC,MAAM,KAAK,SAAS,MAAM;AAChE,aAAS,SAAS,OAAwD;AACxE,YAAM,IAAI,MAAM,MAAM,iDAAiD;AACvE,aAAO,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,MAAM,SAAY,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI;AAAA,IACrE;AACA,aAAS,UAAU,CAAC,GAAG,GAAG,CAAC,GAA6C;AACtE,YAAM,WAAW,CAAC,MAAc;AAC9B,cAAM,IAAI,IAAI;AACd,eAAO,KAAK,UAAU,IAAI,QAAQ,KAAK,KAAK,IAAI,SAAS,OAAO,GAAG;AAAA,MACrE;AACA,aAAO,SAAS,SAAS,CAAC,IAAI,SAAS,SAAS,CAAC,IAAI,SAAS,SAAS,CAAC;AAAA,IAC1E;AACA,aAAS,cAAc,IAAmC,IAA2C;AACnG,YAAM,KAAK,UAAU,EAAE;AACvB,YAAM,KAAK,UAAU,EAAE;AACvB,YAAM,UAAU,KAAK,IAAI,IAAI,EAAE;AAC/B,YAAM,SAAS,KAAK,IAAI,IAAI,EAAE;AAC9B,aAAO,KAAK,OAAQ,UAAU,SAAS,SAAS,QAAS,GAAG,IAAI;AAAA,IAClE;AAEA,UAAMA,cAAoB,CAAC;AAC3B,UAAM,WAAW,MAAM,KAAK,SAAS,iBAAiB,+DAA+D,CAAC;AACtH,QAAI,UAAU;AAEd,eAAW,MAAM,UAA2B;AAC1C,UAAI,WAAW,IAAK;AACpB,YAAM,QAAQ,GAAG,eAAe,IAAI,KAAK,EAAE,MAAM,GAAG,EAAE;AACtD,UAAI,CAAC,KAAM;AAEX,YAAM,QAAQ,OAAO,iBAAiB,EAAE;AACxC,UAAI,MAAM,YAAY,UAAU,MAAM,eAAe,SAAU;AAE/D,YAAM,KAAK,SAAS,MAAM,KAAK;AAC/B,YAAM,KAAK,SAAS,MAAM,eAAe;AACzC,UAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,MAAM,EAAG;AAE/B,YAAM,WAAW,WAAW,MAAM,QAAQ;AAC1C,YAAM,aAAa,MAAM;AACzB,YAAM,cAAc,YAAY,MAAO,YAAY,OAAO,eAAe,UAAU,CAAC,cAAc;AAClG,YAAM,QAAQ,cAAc,IAAI,EAAE;AAClC,YAAM,aAAa,cAAc,IAAM;AACvC,YAAM,cAAc,cAAc,MAAM;AAExC,UAAI,QAAQ,YAAY;AACtB,QAAAA,YAAW,KAAK;AAAA,UACd,UAAU,GAAG,QAAQ,YAAY;AAAA,UACjC,MAAM,KAAK,MAAM,GAAG,EAAE;AAAA,UACtB,YAAY,MAAM;AAAA,UAClB,YAAY,MAAM;AAAA,UAClB;AAAA,UACA,UAAU;AAAA,UACV,OAAO;AAAA,UACP;AAAA,UACA,UAAU;AAAA,QACZ,CAAC;AAAA,MACH,WAAW,QAAQ,aAAa;AAC9B,QAAAA,YAAW,KAAK;AAAA,UACd,UAAU,GAAG,QAAQ,YAAY;AAAA,UACjC,MAAM,KAAK,MAAM,GAAG,EAAE;AAAA,UACtB,YAAY,MAAM;AAAA,UAClB,YAAY,MAAM;AAAA,UAClB;AAAA,UACA,UAAU;AAAA,UACV,OAAO;AAAA,UACP;AAAA,UACA,UAAU;AAAA,QACZ,CAAC;AAAA,MACH,OAAO;AACL;AAAA,MACF;AAAA,IACF;AACA,WAAOA;AAAA,EACT,CAAC,EAAE,MAAM,MAAM,CAAC,CAAwB;AAExC,QAAM,YAAY,KAAK,IAAI,GAAG,MAAM,WAAW,MAAM;AACrD,QAAM,YAAY,WAAW,OAAO,OAAK,EAAE,aAAa,MAAM,EAAE;AAChE,QAAM,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,KAAK,WAAW,OAAO,OAAK,EAAE,aAAa,MAAM,EAAE,SAAS,CAAC;AAEzG,SAAO,EAAE,KAAK,YAAY,WAAW,OAAO,UAAU;AACxD;",
+  "names": ["violations"]
+}

package/dist/content-chunker.d.ts

@@ -0,0 +1,18 @@
+export type ChunkingStrategy = 'fixed' | 'overlap' | 'sentence' | 'paragraph' | 'markdown-heading' | 'semantic' | 'recursive';
+export interface ChunkOpts {
+    strategy: ChunkingStrategy;
+    chunkSize?: number;
+    overlap?: number;
+    separators?: string[];
+}
+export interface Chunk {
+    index: number;
+    text: string;
+    tokenEstimate: number;
+    startChar: number;
+    endChar: number;
+    heading?: string;
+}
+export declare function estimateTokens(text: string): number;
+export declare function chunkContent(text: string, opts: ChunkOpts): Chunk[];
+export declare function mergeSmallChunks(chunks: Chunk[], minTokens: number): Chunk[];

package/dist/content-chunker.js

@@ -0,0 +1,152 @@
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function chunkContent(text, opts) {
+  const size = opts.chunkSize ?? 2e3;
+  const charSize = size * 4;
+  const overlap = opts.overlap ?? 200;
+  const overlapChars = overlap * 4;
+  switch (opts.strategy) {
+    case "fixed":
+      return fixedChunk(text, charSize);
+    case "overlap":
+      return overlapChunk(text, charSize, overlapChars);
+    case "sentence":
+      return sentenceChunk(text, charSize);
+    case "paragraph":
+      return paragraphChunk(text, charSize);
+    case "markdown-heading":
+      return markdownHeadingChunk(text, charSize);
+    case "semantic":
+      return semanticChunk(text, charSize);
+    case "recursive":
+      return recursiveChunk(text, charSize, opts.separators ?? ["\n\n", "\n", ". ", " "]);
+    default:
+      return fixedChunk(text, charSize);
+  }
+}
+function makeChunk(text, index, startChar, heading) {
+  return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };
+}
+function fixedChunk(text, charSize) {
+  const chunks = [];
+  let i = 0;
+  while (i < text.length) {
+    const slice = text.slice(i, i + charSize);
+    chunks.push(makeChunk(slice, chunks.length, i));
+    i += charSize;
+  }
+  return chunks;
+}
+function overlapChunk(text, charSize, overlapChars) {
+  const chunks = [];
+  let i = 0;
+  while (i < text.length) {
+    const slice = text.slice(i, i + charSize);
+    chunks.push(makeChunk(slice, chunks.length, i));
+    i += charSize - overlapChars;
+    if (i <= 0) break;
+  }
+  return chunks;
+}
+function sentenceChunk(text, charSize) {
+  const sentences = text.match(/[^.!?]*[.!?]+["']?\s*/g) ?? [text];
+  return groupIntoChunks(sentences, charSize);
+}
+function paragraphChunk(text, charSize) {
+  const paras = text.split(/\n\n+/).filter((p) => p.trim());
+  return groupIntoChunks(paras, charSize);
+}
+function markdownHeadingChunk(text, charSize) {
+  const chunks = [];
+  const headingRe = /^(#{1,3}\s.+)$/m;
+  const sections = text.split(/(?=^#{1,3}\s)/m).filter((s) => s.trim());
+  let pos = 0;
+  for (const section of sections) {
+    const headingMatch = section.match(headingRe);
+    const heading = headingMatch ? headingMatch[1].replace(/^#+\s*/, "").trim() : void 0;
+    if (section.length <= charSize) {
+      chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });
+    } else {
+      const sub = paragraphChunk(section, charSize);
+      for (const s of sub) {
+        chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });
+      }
+    }
+    pos += section.length;
+  }
+  return chunks.length > 0 ? chunks : fixedChunk(text, charSize);
+}
+function semanticChunk(text, charSize) {
+  const paras = text.split(/\n\n+/).filter((p) => p.trim());
+  const merged = [];
+  let current = "";
+  for (const para of paras) {
+    if (current.length + para.length + 2 <= charSize) {
+      current = current ? current + "\n\n" + para : para;
+    } else {
+      if (current) merged.push(current);
+      current = para;
+    }
+  }
+  if (current) merged.push(current);
+  return groupIntoChunks(merged, charSize);
+}
+function recursiveChunk(text, charSize, separators) {
+  if (text.length <= charSize) return [makeChunk(text, 0, 0)];
+  const [sep, ...rest] = separators;
+  if (!sep) return fixedChunk(text, charSize);
+  const parts = text.split(sep).filter((p) => p);
+  const chunks = [];
+  let pos = 0;
+  for (const part of parts) {
+    if (part.length <= charSize) {
+      chunks.push(makeChunk(part, chunks.length, pos));
+    } else {
+      const sub = recursiveChunk(part, charSize, rest);
+      for (const s of sub) chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });
+    }
+    pos += part.length + sep.length;
+  }
+  return chunks;
+}
+function groupIntoChunks(parts, charSize) {
+  const chunks = [];
+  let current = "";
+  let startChar = 0;
+  let pos = 0;
+  for (const part of parts) {
+    if (current.length + part.length + 1 > charSize && current) {
+      chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+      startChar = pos;
+      current = part;
+    } else {
+      current = current ? current + " " + part : part;
+    }
+    pos += part.length + 1;
+  }
+  if (current.trim()) chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+  return chunks;
+}
+function mergeSmallChunks(chunks, minTokens) {
+  const minChars = minTokens * 4;
+  const result = [];
+  let i = 0;
+  while (i < chunks.length) {
+    if (chunks[i].text.length < minChars && i + 1 < chunks.length) {
+      const merged = chunks[i].text + "\n\n" + chunks[i + 1].text;
+      result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));
+      i += 2;
+    } else {
+      result.push({ ...chunks[i], index: result.length });
+      i++;
+    }
+  }
+  return result;
+}
+export {
+  chunkContent,
+  estimateTokens,
+  mergeSmallChunks
+};
+//# sourceMappingURL=content-chunker.js.map

package/dist/content-chunker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/content-chunker.ts"],
+  "sourcesContent": ["export type ChunkingStrategy = 'fixed' | 'overlap' | 'sentence' | 'paragraph' | 'markdown-heading' | 'semantic' | 'recursive';\n\nexport interface ChunkOpts {\n  strategy: ChunkingStrategy;\n  chunkSize?: number;   // token estimate target (1 token \u2248 4 chars). Default 2000.\n  overlap?: number;     // overlap tokens for 'overlap' strategy. Default 200.\n  separators?: string[]; // for 'recursive' strategy. Default ['\\\\n\\\\n', '\\\\n', '. ', ' ']\n}\n\nexport interface Chunk {\n  index: number;\n  text: string;\n  tokenEstimate: number;\n  startChar: number;\n  endChar: number;\n  heading?: string;\n}\n\nexport function estimateTokens(text: string): number {\n  return Math.ceil(text.length / 4);\n}\n\nexport function chunkContent(text: string, opts: ChunkOpts): Chunk[] {\n  const size = opts.chunkSize ?? 2000;\n  const charSize = size * 4;\n  const overlap = opts.overlap ?? 200;\n  const overlapChars = overlap * 4;\n\n  switch (opts.strategy) {\n    case 'fixed': return fixedChunk(text, charSize);\n    case 'overlap': return overlapChunk(text, charSize, overlapChars);\n    case 'sentence': return sentenceChunk(text, charSize);\n    case 'paragraph': return paragraphChunk(text, charSize);\n    case 'markdown-heading': return markdownHeadingChunk(text, charSize);\n    case 'semantic': return semanticChunk(text, charSize);\n    case 'recursive': return recursiveChunk(text, charSize, opts.separators ?? ['\\n\\n', '\\n', '. ', ' ']);\n    default: return fixedChunk(text, charSize);\n  }\n}\n\nfunction makeChunk(text: string, index: number, startChar: number, heading?: string): Chunk {\n  return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };\n}\n\nfunction fixedChunk(text: string, charSize: number): Chunk[] {\n  const chunks: Chunk[] = [];\n  let i = 0;\n  while (i < text.length) {\n    const slice = text.slice(i, i + charSize);\n    chunks.push(makeChunk(slice, chunks.length, i));\n    i += charSize;\n  }\n  return chunks;\n}\n\nfunction overlapChunk(text: string, charSize: number, overlapChars: number): Chunk[] {\n  const chunks: Chunk[] = [];\n  let i = 0;\n  while (i < text.length) {\n    const slice = text.slice(i, i + charSize);\n    chunks.push(makeChunk(slice, chunks.length, i));\n    i += charSize - overlapChars;\n    if (i <= 0) break;\n  }\n  return chunks;\n}\n\nfunction sentenceChunk(text: string, charSize: number): Chunk[] {\n  const sentences = text.match(/[^.!?]*[.!?]+[\"']?\\s*/g) ?? [text];\n  return groupIntoChunks(sentences, charSize);\n}\n\nfunction paragraphChunk(text: string, charSize: number): Chunk[] {\n  const paras = text.split(/\\n\\n+/).filter(p => p.trim());\n  return groupIntoChunks(paras, charSize);\n}\n\nfunction markdownHeadingChunk(text: string, charSize: number): Chunk[] {\n  const chunks: Chunk[] = [];\n  const headingRe = /^(#{1,3}\\s.+)$/m;\n  const sections = text.split(/(?=^#{1,3}\\s)/m).filter(s => s.trim());\n  let pos = 0;\n  for (const section of sections) {\n    const headingMatch = section.match(headingRe);\n    const heading = headingMatch ? headingMatch[1].replace(/^#+\\s*/, '').trim() : undefined;\n    if (section.length <= charSize) {\n      chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });\n    } else {\n      // Section too large \u2014 sub-chunk by paragraph\n      const sub = paragraphChunk(section, charSize);\n      for (const s of sub) {\n        chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });\n      }\n    }\n    pos += section.length;\n  }\n  return chunks.length > 0 ? chunks : fixedChunk(text, charSize);\n}\n\nfunction semanticChunk(text: string, charSize: number): Chunk[] {\n  // Like paragraph but merges short paragraphs with the next\n  const paras = text.split(/\\n\\n+/).filter(p => p.trim());\n  const merged: string[] = [];\n  let current = '';\n  for (const para of paras) {\n    if (current.length + para.length + 2 <= charSize) {\n      current = current ? current + '\\n\\n' + para : para;\n    } else {\n      if (current) merged.push(current);\n      current = para;\n    }\n  }\n  if (current) merged.push(current);\n  return groupIntoChunks(merged, charSize);\n}\n\nfunction recursiveChunk(text: string, charSize: number, separators: string[]): Chunk[] {\n  if (text.length <= charSize) return [makeChunk(text, 0, 0)];\n  const [sep, ...rest] = separators;\n  if (!sep) return fixedChunk(text, charSize);\n  const parts = text.split(sep).filter(p => p);\n  const chunks: Chunk[] = [];\n  let pos = 0;\n  for (const part of parts) {\n    if (part.length <= charSize) {\n      chunks.push(makeChunk(part, chunks.length, pos));\n    } else {\n      const sub = recursiveChunk(part, charSize, rest);\n      for (const s of sub) chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });\n    }\n    pos += part.length + sep.length;\n  }\n  return chunks;\n}\n\nfunction groupIntoChunks(parts: string[], charSize: number): Chunk[] {\n  const chunks: Chunk[] = [];\n  let current = '';\n  let startChar = 0;\n  let pos = 0;\n  for (const part of parts) {\n    if (current.length + part.length + 1 > charSize && current) {\n      chunks.push(makeChunk(current.trim(), chunks.length, startChar));\n      startChar = pos;\n      current = part;\n    } else {\n      current = current ? current + ' ' + part : part;\n    }\n    pos += part.length + 1;\n  }\n  if (current.trim()) chunks.push(makeChunk(current.trim(), chunks.length, startChar));\n  return chunks;\n}\n\nexport function mergeSmallChunks(chunks: Chunk[], minTokens: number): Chunk[] {\n  const minChars = minTokens * 4;\n  const result: Chunk[] = [];\n  let i = 0;\n  while (i < chunks.length) {\n    if (chunks[i].text.length < minChars && i + 1 < chunks.length) {\n      const merged = chunks[i].text + '\\n\\n' + chunks[i + 1].text;\n      result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));\n      i += 2;\n    } else {\n      result.push({ ...chunks[i], index: result.length });\n      i++;\n    }\n  }\n  return result;\n}\n"],
+  "mappings": "AAkBO,SAAS,eAAe,MAAsB;AACnD,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AAEO,SAAS,aAAa,MAAc,MAA0B;AACnE,QAAM,OAAO,KAAK,aAAa;AAC/B,QAAM,WAAW,OAAO;AACxB,QAAM,UAAU,KAAK,WAAW;AAChC,QAAM,eAAe,UAAU;AAE/B,UAAQ,KAAK,UAAU;AAAA,IACrB,KAAK;AAAS,aAAO,WAAW,MAAM,QAAQ;AAAA,IAC9C,KAAK;AAAW,aAAO,aAAa,MAAM,UAAU,YAAY;AAAA,IAChE,KAAK;AAAY,aAAO,cAAc,MAAM,QAAQ;AAAA,IACpD,KAAK;AAAa,aAAO,eAAe,MAAM,QAAQ;AAAA,IACtD,KAAK;AAAoB,aAAO,qBAAqB,MAAM,QAAQ;AAAA,IACnE,KAAK;AAAY,aAAO,cAAc,MAAM,QAAQ;AAAA,IACpD,KAAK;AAAa,aAAO,eAAe,MAAM,UAAU,KAAK,cAAc,CAAC,QAAQ,MAAM,MAAM,GAAG,CAAC;AAAA,IACpG;AAAS,aAAO,WAAW,MAAM,QAAQ;AAAA,EAC3C;AACF;AAEA,SAAS,UAAU,MAAc,OAAe,WAAmB,SAAyB;AAC1F,SAAO,EAAE,OAAO,MAAM,eAAe,eAAe,IAAI,GAAG,WAAW,SAAS,YAAY,KAAK,QAAQ,QAAQ;AAClH;AAEA,SAAS,WAAW,MAAc,UAA2B;AAC3D,QAAM,SAAkB,CAAC;AACzB,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,QAAQ;AACtB,UAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,QAAQ;AACxC,WAAO,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAC9C,SAAK;AAAA,EACP;AACA,SAAO;AACT;AAEA,SAAS,aAAa,MAAc,UAAkB,cAA+B;AACnF,QAAM,SAAkB,CAAC;AACzB,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,QAAQ;AACtB,UAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,QAAQ;AACxC,WAAO,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAC9C,SAAK,WAAW;AAChB,QAAI,KAAK,EAAG;AAAA,EACd;AACA,SAAO;AACT;AAEA,SAAS,cAAc,MAAc,UAA2B;AAC9D,QAAM,YAAY,KAAK,MAAM,wBAAwB,KAAK,CAAC,IAAI;AAC/D,SAAO,gBAAgB,WAAW,QAAQ;AAC5C;AAEA,SAAS,eAAe,MAAc,UAA2B;AAC/D,QAAM,QAAQ,KAAK,MAAM,OAAO,EAAE,OAAO,OAAK,EAAE,KAAK,CAAC;AACtD,SAAO,gBAAgB,OAAO,QAAQ;AACxC;AAEA,SAAS,qBAAqB,MAAc,UAA2B;AACrE,QAAM,SAAkB,CAAC;AACzB,QAAM,YAAY;AAClB,QAAM,WAAW,KAAK,MAAM,gBAAgB,EAAE,OAAO,OAAK,EAAE,KAAK,CAAC;AAClE,MAAI,MAAM;AACV,aAAW,WAAW,UAAU;AAC9B,UAAM,eAAe,QAAQ,MAAM,SAAS;AAC5C,UAAM,UAAU,eAAe,aAAa,CAAC,EAAE,QAAQ,UAAU,EAAE,EAAE,KAAK,IAAI;AAC9E,QAAI,QAAQ,UAAU,UAAU;AAC9B,aAAO,KAAK,EAAE,GAAG,UAAU,QAAQ,KAAK,GAAG,OAAO,QAAQ,GAAG,GAAG,QAAQ,CAAC;AAAA,IAC3E,OAAO;AAEL,YAAM,MAAM,eAAe,SAAS,QAAQ;AAC5C,iBAAW,KAAK,KAAK;AACnB,eAAO,KAAK,EAAE,GAAG,GAAG,OAAO,OAAO,QAAQ,WAAW,MAAM,EAAE,WAAW,SAAS,MAAM,EAAE,SAAS,QAAQ,CAAC;AAAA,MAC7G;AAAA,IACF;AACA,WAAO,QAAQ;AAAA,EACjB;AACA,SAAO,OAAO,SAAS,IAAI,SAAS,WAAW,MAAM,QAAQ;AAC/D;AAEA,SAAS,cAAc,MAAc,UAA2B;AAE9D,QAAM,QAAQ,KAAK,MAAM,OAAO,EAAE,OAAO,OAAK,EAAE,KAAK,CAAC;AACtD,QAAM,SAAmB,CAAC;AAC1B,MAAI,UAAU;AACd,aAAW,QAAQ,OAAO;AACxB,QAAI,QAAQ,SAAS,KAAK,SAAS,KAAK,UAAU;AAChD,gBAAU,UAAU,UAAU,SAAS,OAAO;AAAA,IAChD,OAAO;AACL,UAAI,QAAS,QAAO,KAAK,OAAO;AAChC,gBAAU;AAAA,IACZ;AAAA,EACF;AACA,MAAI,QAAS,QAAO,KAAK,OAAO;AAChC,SAAO,gBAAgB,QAAQ,QAAQ;AACzC;AAEA,SAAS,eAAe,MAAc,UAAkB,YAA+B;AACrF,MAAI,KAAK,UAAU,SAAU,QAAO,CAAC,UAAU,MAAM,GAAG,CAAC,CAAC;AAC1D,QAAM,CAAC,KAAK,GAAG,IAAI,IAAI;AACvB,MAAI,CAAC,IAAK,QAAO,WAAW,MAAM,QAAQ;AAC1C,QAAM,QAAQ,KAAK,MAAM,GAAG,EAAE,OAAO,OAAK,CAAC;AAC3C,QAAM,SAAkB,CAAC;AACzB,MAAI,MAAM;AACV,aAAW,QAAQ,OAAO;AACxB,QAAI,KAAK,UAAU,UAAU;AAC3B,aAAO,KAAK,UAAU,MAAM,OAAO,QAAQ,GAAG,CAAC;AAAA,IACjD,OAAO;AACL,YAAM,MAAM,eAAe,MAAM,UAAU,IAAI;AAC/C,iBAAW,KAAK,IAAK,QAAO,KAAK,EAAE,GAAG,GAAG,OAAO,OAAO,QAAQ,WAAW,MAAM,EAAE,WAAW,SAAS,MAAM,EAAE,QAAQ,CAAC;AAAA,IACzH;AACA,WAAO,KAAK,SAAS,IAAI;AAAA,EAC3B;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,OAAiB,UAA2B;AACnE,QAAM,SAAkB,CAAC;AACzB,MAAI,UAAU;AACd,MAAI,YAAY;AAChB,MAAI,MAAM;AACV,aAAW,QAAQ,OAAO;AACxB,QAAI,QAAQ,SAAS,KAAK,SAAS,IAAI,YAAY,SAAS;AAC1D,aAAO,KAAK,UAAU,QAAQ,KAAK,GAAG,OAAO,QAAQ,SAAS,CAAC;AAC/D,kBAAY;AACZ,gBAAU;AAAA,IACZ,OAAO;AACL,gBAAU,UAAU,UAAU,MAAM,OAAO;AAAA,IAC7C;AACA,WAAO,KAAK,SAAS;AAAA,EACvB;AACA,MAAI,QAAQ,KAAK,EAAG,QAAO,KAAK,UAAU,QAAQ,KAAK,GAAG,OAAO,QAAQ,SAAS,CAAC;AACnF,SAAO;AACT;AAEO,SAAS,iBAAiB,QAAiB,WAA4B;AAC5E,QAAM,WAAW,YAAY;AAC7B,QAAM,SAAkB,CAAC;AACzB,MAAI,IAAI;AACR,SAAO,IAAI,OAAO,QAAQ;AACxB,QAAI,OAAO,CAAC,EAAE,KAAK,SAAS,YAAY,IAAI,IAAI,OAAO,QAAQ;AAC7D,YAAM,SAAS,OAAO,CAAC,EAAE,OAAO,SAAS,OAAO,IAAI,CAAC,EAAE;AACvD,aAAO,KAAK,UAAU,QAAQ,OAAO,QAAQ,OAAO,CAAC,EAAE,WAAW,OAAO,CAAC,EAAE,WAAW,OAAO,IAAI,CAAC,EAAE,OAAO,CAAC;AAC7G,WAAK;AAAA,IACP,OAAO;AACL,aAAO,KAAK,EAAE,GAAG,OAAO,CAAC,GAAG,OAAO,OAAO,OAAO,CAAC;AAClD;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;",
+  "names": []
+}

package/dist/content-dedup.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Content deduplication — detect near-duplicate pages during crawl.
+ *
+ * Uses MinHash-inspired approach: instead of full MinHash (needs many hash functions),
+ * we use 8 structural fingerprints extracted from the markdown to detect pages that
+ * share the same layout/template even if the data differs.
+ *
+ * Typical duplicates: /products?page=1 vs /products?page=2 (same template, different data).
+ * These waste crawl budget and generate redundant test cases.
+ */
+export interface DedupResult {
+    isDuplicate: boolean;
+    similarUrl?: string;
+    similarityScore: number;
+}
+export declare class ContentDeduplicator {
+    private seen;
+    private readonly threshold;
+    constructor(threshold?: number);
+    check(url: string, markdown: string): DedupResult;
+    size(): number;
+    reset(): void;
+}

package/dist/content-dedup.js

@@ -0,0 +1,60 @@
+function extractFingerprints(markdown) {
+  const lines = markdown.split("\n").filter((l) => l.trim().length > 0);
+  const fps = [];
+  const headings = lines.filter((l) => /^#{1,6} /.test(l));
+  const hCounts = [0, 0, 0, 0, 0, 0];
+  for (const h of headings) {
+    const lvl = h.match(/^(#{1,6}) /)?.[1].length ?? 1;
+    hCounts[lvl - 1]++;
+  }
+  fps.push("h:" + hCounts.join(","));
+  const linkCount = (markdown.match(/\[.+?\]\(.+?\)/g) ?? []).length;
+  fps.push("lk:" + Math.floor(linkCount / 5));
+  const listCount = lines.filter((l) => /^[\-\*\+] |^\d+\. /.test(l)).length;
+  fps.push("li:" + Math.floor(listCount / 3));
+  fps.push("code:" + (markdown.includes("```") ? "1" : "0"));
+  fps.push("tbl:" + (markdown.includes("|") ? "1" : "0"));
+  const paraCount = markdown.split(/\n{2,}/).filter((p) => p.trim().length > 30).length;
+  fps.push("p:" + Math.floor(paraCount / 2));
+  const firstH = headings[0]?.replace(/^#{1,6} /, "").toLowerCase().replace(/[^a-z0-9]/g, "") ?? "";
+  fps.push("fh:" + firstH.slice(0, 20));
+  const wordCount = markdown.split(/\s+/).length;
+  fps.push("wc:" + Math.floor(wordCount / 50));
+  return fps;
+}
+function similarity(a, b) {
+  if (a.length !== b.length) return 0;
+  let matches = 0;
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] === b[i]) matches++;
+  }
+  return matches / a.length;
+}
+class ContentDeduplicator {
+  seen = /* @__PURE__ */ new Map();
+  threshold;
+  constructor(threshold = 0.85) {
+    this.threshold = threshold;
+  }
+  check(url, markdown) {
+    const fps = extractFingerprints(markdown);
+    for (const [key, entry] of this.seen) {
+      const score = similarity(fps, entry.fps);
+      if (score >= this.threshold) {
+        return { isDuplicate: true, similarUrl: entry.url, similarityScore: score };
+      }
+    }
+    this.seen.set(url, { fps, url });
+    return { isDuplicate: false, similarityScore: 1 };
+  }
+  size() {
+    return this.seen.size;
+  }
+  reset() {
+    this.seen.clear();
+  }
+}
+export {
+  ContentDeduplicator
+};
+//# sourceMappingURL=content-dedup.js.map

package/dist/content-dedup.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/content-dedup.ts"],
+  "sourcesContent": ["/**\n * Content deduplication \u2014 detect near-duplicate pages during crawl.\n *\n * Uses MinHash-inspired approach: instead of full MinHash (needs many hash functions),\n * we use 8 structural fingerprints extracted from the markdown to detect pages that\n * share the same layout/template even if the data differs.\n *\n * Typical duplicates: /products?page=1 vs /products?page=2 (same template, different data).\n * These waste crawl budget and generate redundant test cases.\n */\n\n/** Extract structural fingerprints from markdown \u2014 layout-sensitive, data-insensitive */\nfunction extractFingerprints(markdown: string): string[] {\n  const lines = markdown.split('\\n').filter(l => l.trim().length > 0);\n  const fps: string[] = [];\n\n  // FP1: heading structure (H1-H6 count distribution)\n  const headings = lines.filter(l => /^#{1,6} /.test(l));\n  const hCounts = [0, 0, 0, 0, 0, 0];\n  for (const h of headings) {\n    const lvl = h.match(/^(#{1,6}) /)?.[1].length ?? 1;\n    hCounts[lvl - 1]++;\n  }\n  fps.push('h:' + hCounts.join(','));\n\n  // FP2: link count (grouped into buckets)\n  const linkCount = (markdown.match(/\\[.+?\\]\\(.+?\\)/g) ?? []).length;\n  fps.push('lk:' + Math.floor(linkCount / 5));\n\n  // FP3: list item count (grouped)\n  const listCount = lines.filter(l => /^[\\-\\*\\+] |^\\d+\\. /.test(l)).length;\n  fps.push('li:' + Math.floor(listCount / 3));\n\n  // FP4: code block presence\n  fps.push('code:' + (markdown.includes('```') ? '1' : '0'));\n\n  // FP5: table presence\n  fps.push('tbl:' + (markdown.includes('|') ? '1' : '0'));\n\n  // FP6: approximate paragraph count\n  const paraCount = markdown.split(/\\n{2,}/).filter(p => p.trim().length > 30).length;\n  fps.push('p:' + Math.floor(paraCount / 2));\n\n  // FP7: first heading text (normalized)\n  const firstH = headings[0]?.replace(/^#{1,6} /, '').toLowerCase().replace(/[^a-z0-9]/g, '') ?? '';\n  fps.push('fh:' + firstH.slice(0, 20));\n\n  // FP8: word count bucket\n  const wordCount = markdown.split(/\\s+/).length;\n  fps.push('wc:' + Math.floor(wordCount / 50));\n\n  return fps;\n}\n\nfunction similarity(a: string[], b: string[]): number {\n  if (a.length !== b.length) return 0;\n  let matches = 0;\n  for (let i = 0; i < a.length; i++) {\n    if (a[i] === b[i]) matches++;\n  }\n  return matches / a.length;\n}\n\nexport interface DedupResult {\n  isDuplicate: boolean;\n  similarUrl?: string;\n  similarityScore: number;\n}\n\nexport class ContentDeduplicator {\n  private seen = new Map<string, { fps: string[]; url: string }>();\n  private readonly threshold: number;\n\n  constructor(threshold = 0.85) {\n    this.threshold = threshold;\n  }\n\n  check(url: string, markdown: string): DedupResult {\n    const fps = extractFingerprints(markdown);\n\n    for (const [key, entry] of this.seen) {\n      const score = similarity(fps, entry.fps);\n      if (score >= this.threshold) {\n        return { isDuplicate: true, similarUrl: entry.url, similarityScore: score };\n      }\n    }\n\n    this.seen.set(url, { fps, url });\n    return { isDuplicate: false, similarityScore: 1.0 };\n  }\n\n  size(): number {\n    return this.seen.size;\n  }\n\n  reset(): void {\n    this.seen.clear();\n  }\n}\n"],
+  "mappings": "AAYA,SAAS,oBAAoB,UAA4B;AACvD,QAAM,QAAQ,SAAS,MAAM,IAAI,EAAE,OAAO,OAAK,EAAE,KAAK,EAAE,SAAS,CAAC;AAClE,QAAM,MAAgB,CAAC;AAGvB,QAAM,WAAW,MAAM,OAAO,OAAK,WAAW,KAAK,CAAC,CAAC;AACrD,QAAM,UAAU,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;AACjC,aAAW,KAAK,UAAU;AACxB,UAAM,MAAM,EAAE,MAAM,YAAY,IAAI,CAAC,EAAE,UAAU;AACjD,YAAQ,MAAM,CAAC;AAAA,EACjB;AACA,MAAI,KAAK,OAAO,QAAQ,KAAK,GAAG,CAAC;AAGjC,QAAM,aAAa,SAAS,MAAM,iBAAiB,KAAK,CAAC,GAAG;AAC5D,MAAI,KAAK,QAAQ,KAAK,MAAM,YAAY,CAAC,CAAC;AAG1C,QAAM,YAAY,MAAM,OAAO,OAAK,qBAAqB,KAAK,CAAC,CAAC,EAAE;AAClE,MAAI,KAAK,QAAQ,KAAK,MAAM,YAAY,CAAC,CAAC;AAG1C,MAAI,KAAK,WAAW,SAAS,SAAS,KAAK,IAAI,MAAM,IAAI;AAGzD,MAAI,KAAK,UAAU,SAAS,SAAS,GAAG,IAAI,MAAM,IAAI;AAGtD,QAAM,YAAY,SAAS,MAAM,QAAQ,EAAE,OAAO,OAAK,EAAE,KAAK,EAAE,SAAS,EAAE,EAAE;AAC7E,MAAI,KAAK,OAAO,KAAK,MAAM,YAAY,CAAC,CAAC;AAGzC,QAAM,SAAS,SAAS,CAAC,GAAG,QAAQ,YAAY,EAAE,EAAE,YAAY,EAAE,QAAQ,cAAc,EAAE,KAAK;AAC/F,MAAI,KAAK,QAAQ,OAAO,MAAM,GAAG,EAAE,CAAC;AAGpC,QAAM,YAAY,SAAS,MAAM,KAAK,EAAE;AACxC,MAAI,KAAK,QAAQ,KAAK,MAAM,YAAY,EAAE,CAAC;AAE3C,SAAO;AACT;AAEA,SAAS,WAAW,GAAa,GAAqB;AACpD,MAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,MAAI,UAAU;AACd,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,QAAI,EAAE,CAAC,MAAM,EAAE,CAAC,EAAG;AAAA,EACrB;AACA,SAAO,UAAU,EAAE;AACrB;AAQO,MAAM,oBAAoB;AAAA,EACvB,OAAO,oBAAI,IAA4C;AAAA,EAC9C;AAAA,EAEjB,YAAY,YAAY,MAAM;AAC5B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,KAAa,UAA+B;AAChD,UAAM,MAAM,oBAAoB,QAAQ;AAExC,eAAW,CAAC,KAAK,KAAK,KAAK,KAAK,MAAM;AACpC,YAAM,QAAQ,WAAW,KAAK,MAAM,GAAG;AACvC,UAAI,SAAS,KAAK,WAAW;AAC3B,eAAO,EAAE,aAAa,MAAM,YAAY,MAAM,KAAK,iBAAiB,MAAM;AAAA,MAC5E;AAAA,IACF;AAEA,SAAK,KAAK,IAAI,KAAK,EAAE,KAAK,IAAI,CAAC;AAC/B,WAAO,EAAE,aAAa,OAAO,iBAAiB,EAAI;AAAA,EACpD;AAAA,EAEA,OAAe;AACb,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA,EAEA,QAAc;AACZ,SAAK,KAAK,MAAM;AAAA,EAClB;AACF;",
+  "names": []
+}

package/dist/cookie-consent.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Cookie consent auto-acceptor.
+ *
+ * Detects and clicks "Accept" on common cookie consent banners before the
+ * crawler proceeds. Without this, many EU sites show a consent wall that
+ * blocks content or shows a different layout, making crawl results unreliable.
+ *
+ * Supports: Cookiebot, OneTrust, TrustArc, Usercentrics, GDPR Cookie Compliance,
+ * generic "Accept All" / "Accept Cookies" buttons.
+ *
+ * Strategy: try known selectors first, then fall back to text-based button search.
+ * Non-blocking — if no banner is found or click fails, crawl proceeds normally.
+ */
+import type { Page } from 'playwright';
+export interface ConsentResult {
+    found: boolean;
+    clicked: boolean;
+    method?: string;
+    error?: string;
+}
+export declare function acceptCookieConsent(page: Page, opts?: {
+    waitMs?: number;
+}): Promise<ConsentResult>;
+/** Check if a consent banner is still visible after attempted dismissal */
+export declare function isConsentBannerVisible(page: Page): Promise<boolean>;