@taodigital/crawler 0.0.0-stage → 1.2.1

package/dist/browser-extension.js

@@ -0,0 +1,113 @@
+import { createHash, randomBytes } from "node:crypto";
+import { createWriteStream, existsSync, mkdirSync } from "node:fs";
+import { writeFile, mkdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { pipeline } from "node:stream/promises";
+import { Readable } from "node:stream";
+const PROFILE_BASE = process.env.EXTENSION_PROFILE_DIR ?? join(tmpdir(), "zeta-ext-profiles");
+async function downloadCrx(url) {
+  const dest = join(tmpdir(), `zeta-ext-${randomBytes(6).toString("hex")}.crx`);
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Failed to download extension from ${url}: ${res.status}`);
+  const writer = createWriteStream(dest);
+  await pipeline(Readable.fromWeb(res.body), writer);
+  return dest;
+}
+async function unpackCrx(crxPath, destDir) {
+  const { readFile } = await import("node:fs/promises");
+  const buf = await readFile(crxPath);
+  const magic = buf.readUInt32LE(0);
+  if (magic !== 1131557428) {
+    await mkdir(destDir, { recursive: true });
+    await writeFile(join(destDir, "__raw.crx"), buf);
+    return;
+  }
+  const version = buf.readUInt32LE(4);
+  let zipStart;
+  if (version === 3) {
+    const headerSize = buf.readUInt32LE(8);
+    zipStart = 12 + headerSize;
+  } else if (version === 2) {
+    const pubKeyLen = buf.readUInt32LE(8);
+    const sigLen = buf.readUInt32LE(12);
+    zipStart = 16 + pubKeyLen + sigLen;
+  } else {
+    throw new Error(`Unknown CRX version ${version}`);
+  }
+  const zipBuf = buf.subarray(zipStart);
+  const tmpZip = join(tmpdir(), `zeta-ext-${randomBytes(4).toString("hex")}.zip`);
+  await writeFile(tmpZip, zipBuf);
+  await mkdir(destDir, { recursive: true });
+  const unzipper = await import("unzipper").catch(() => null);
+  if (unzipper) {
+    await unzipper.Open.file(tmpZip).then((d) => d.extract({ path: destDir }));
+  } else {
+    const { execFile } = await import("node:child_process");
+    await new Promise(
+      (res, rej) => execFile("unzip", ["-o", tmpZip, "-d", destDir], (err) => err ? rej(err) : res())
+    );
+  }
+  await rm(tmpZip, { force: true });
+}
+async function prepareExtensionProfile(extensions) {
+  const key = createHash("md5").update(extensions.map((e) => e.source).sort().join("|")).digest("hex").slice(0, 12);
+  const profileDir = join(PROFILE_BASE, key);
+  const extensionDirs = [];
+  mkdirSync(profileDir, { recursive: true });
+  for (const ext of extensions) {
+    const extHash = createHash("md5").update(ext.source).digest("hex").slice(0, 8);
+    const extDir = join(profileDir, `ext-${extHash}`);
+    if (!existsSync(extDir)) {
+      let crxPath = ext.source;
+      if (ext.source.startsWith("http://") || ext.source.startsWith("https://")) {
+        crxPath = await downloadCrx(ext.source);
+        await unpackCrx(crxPath, extDir);
+        await rm(crxPath, { force: true });
+      } else if (ext.source.endsWith(".crx")) {
+        await unpackCrx(ext.source, extDir);
+      } else {
+        extensionDirs.push(ext.source);
+        continue;
+      }
+    }
+    extensionDirs.push(extDir);
+  }
+  const userDataDir = join(profileDir, "user-data");
+  mkdirSync(userDataDir, { recursive: true });
+  const defaultsDir = join(userDataDir, "Default");
+  mkdirSync(defaultsDir, { recursive: true });
+  const prefsPath = join(defaultsDir, "Preferences");
+  if (!existsSync(prefsPath)) {
+    await writeFile(prefsPath, JSON.stringify({
+      browser: { has_seen_welcome_page: true },
+      extensions: { alerts: { initialized: true } }
+    }), "utf8");
+  }
+  const loadExtensionArg = extensionDirs.join(",");
+  const args = [
+    `--load-extension=${loadExtensionArg}`,
+    `--disable-extensions-except=${loadExtensionArg}`,
+    `--user-data-dir=${userDataDir}`,
+    "--no-first-run",
+    "--no-default-browser-check"
+  ];
+  return {
+    userDataDir,
+    args,
+    cleanup: async () => {
+      await rm(profileDir, { recursive: true, force: true });
+    }
+  };
+}
+async function adBlockerProfile() {
+  return prepareExtensionProfile([{
+    name: "uBlock Origin",
+    source: "https://github.com/gorhill/uBlock/releases/download/1.57.2/uBlock0_1.57.2.chromium.zip"
+  }]);
+}
+export {
+  adBlockerProfile,
+  prepareExtensionProfile
+};
+//# sourceMappingURL=browser-extension.js.map

package/dist/browser-extension.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/browser-extension.ts"],
+  "sourcesContent": ["/**\n * Browser extension injection support.\n *\n * Chrome extensions require a persistent profile directory \u2014 incompatible\n * with headless Docker by default. This module handles:\n *   1. Persistent profile directory per extension set\n *   2. Extension download from .crx URL or load from local path\n *   3. Launch args for Playwright / Stagehand\n *\n * Usage:\n *   const profile = await prepareExtensionProfile(['/path/to/ext.crx']);\n *   // pass profile.userDataDir and profile.args to browser launch\n */\n\nimport { createHash, randomBytes } from 'node:crypto';\nimport { createWriteStream, existsSync, mkdirSync } from 'node:fs';\nimport { writeFile, mkdir, rm } from 'node:fs/promises';\nimport { join } from 'node:path';\nimport { tmpdir } from 'node:os';\nimport { pipeline } from 'node:stream/promises';\nimport { Readable } from 'node:stream';\n\nconst PROFILE_BASE = process.env.EXTENSION_PROFILE_DIR ?? join(tmpdir(), 'zeta-ext-profiles');\n\nexport interface ExtensionSource {\n  /** Local .crx file path or https:// URL to download from */\n  source: string;\n  /** Friendly name for logging */\n  name?: string;\n}\n\nexport interface ExtensionProfile {\n  /** Chrome user-data-dir containing Preferences + extension unpacked dirs */\n  userDataDir: string;\n  /** Args to append to chromium launch \u2014 includes --load-extension and --disable-extensions-except */\n  args: string[];\n  /** Cleanup the profile directory when done */\n  cleanup: () => Promise<void>;\n}\n\n/**\n * Download a .crx from a URL to a temp file. Returns path.\n */\nasync function downloadCrx(url: string): Promise<string> {\n  const dest = join(tmpdir(), `zeta-ext-${randomBytes(6).toString('hex')}.crx`);\n  const res = await fetch(url);\n  if (!res.ok) throw new Error(`Failed to download extension from ${url}: ${res.status}`);\n  const writer = createWriteStream(dest);\n  await pipeline(Readable.fromWeb(res.body as any), writer);\n  return dest;\n}\n\n/**\n * Unpack a .crx file to a directory Chrome can load as an unpacked extension.\n * CRX3 format: 4-byte magic + 4-byte version + 4-byte header_size + proto header + zip body\n */\nasync function unpackCrx(crxPath: string, destDir: string): Promise<void> {\n  const { readFile } = await import('node:fs/promises');\n  const buf = await readFile(crxPath);\n\n  const magic = buf.readUInt32LE(0);\n  if (magic !== 0x43723234) {\n    // Not CRX \u2014 assume it's already a zip (unpacked extension)\n    await mkdir(destDir, { recursive: true });\n    await writeFile(join(destDir, '__raw.crx'), buf);\n    return;\n  }\n\n  const version = buf.readUInt32LE(4);\n  let zipStart: number;\n  if (version === 3) {\n    const headerSize = buf.readUInt32LE(8);\n    zipStart = 12 + headerSize;\n  } else if (version === 2) {\n    const pubKeyLen = buf.readUInt32LE(8);\n    const sigLen = buf.readUInt32LE(12);\n    zipStart = 16 + pubKeyLen + sigLen;\n  } else {\n    throw new Error(`Unknown CRX version ${version}`);\n  }\n\n  const zipBuf = buf.subarray(zipStart);\n  const tmpZip = join(tmpdir(), `zeta-ext-${randomBytes(4).toString('hex')}.zip`);\n  await writeFile(tmpZip, zipBuf);\n\n  await mkdir(destDir, { recursive: true });\n\n  // Extract using unzipper (available in Node without native deps)\n  // @ts-ignore \u2013 no types for unzipper\n  const unzipper = await import('unzipper').catch(() => null);\n  if (unzipper) {\n    await unzipper.Open.file(tmpZip).then((d: any) => d.extract({ path: destDir }));\n  } else {\n    // Fallback: shell unzip\n    const { execFile } = await import('node:child_process');\n    await new Promise<void>((res, rej) =>\n      execFile('unzip', ['-o', tmpZip, '-d', destDir], (err) => err ? rej(err) : res())\n    );\n  }\n  await rm(tmpZip, { force: true });\n}\n\n/**\n * Prepare a Chrome user-data-dir with the given extensions loaded.\n * Returns profile info + cleanup function.\n */\nexport async function prepareExtensionProfile(\n  extensions: ExtensionSource[]\n): Promise<ExtensionProfile> {\n  // Stable profile key based on extension sources\n  const key = createHash('md5')\n    .update(extensions.map(e => e.source).sort().join('|'))\n    .digest('hex')\n    .slice(0, 12);\n\n  const profileDir = join(PROFILE_BASE, key);\n  const extensionDirs: string[] = [];\n\n  mkdirSync(profileDir, { recursive: true });\n\n  for (const ext of extensions) {\n    const extHash = createHash('md5').update(ext.source).digest('hex').slice(0, 8);\n    const extDir = join(profileDir, `ext-${extHash}`);\n\n    if (!existsSync(extDir)) {\n      let crxPath = ext.source;\n\n      if (ext.source.startsWith('http://') || ext.source.startsWith('https://')) {\n        crxPath = await downloadCrx(ext.source);\n        await unpackCrx(crxPath, extDir);\n        await rm(crxPath, { force: true });\n      } else if (ext.source.endsWith('.crx')) {\n        await unpackCrx(ext.source, extDir);\n      } else {\n        // Assume already-unpacked extension directory \u2014 symlink or copy\n        extensionDirs.push(ext.source);\n        continue;\n      }\n    }\n\n    extensionDirs.push(extDir);\n  }\n\n  const userDataDir = join(profileDir, 'user-data');\n  mkdirSync(userDataDir, { recursive: true });\n\n  // Write a minimal Preferences file so Chrome doesn't show first-run UI\n  const defaultsDir = join(userDataDir, 'Default');\n  mkdirSync(defaultsDir, { recursive: true });\n  const prefsPath = join(defaultsDir, 'Preferences');\n  if (!existsSync(prefsPath)) {\n    await writeFile(prefsPath, JSON.stringify({\n      browser: { has_seen_welcome_page: true },\n      extensions: { alerts: { initialized: true } },\n    }), 'utf8');\n  }\n\n  const loadExtensionArg = extensionDirs.join(',');\n  const args: string[] = [\n    `--load-extension=${loadExtensionArg}`,\n    `--disable-extensions-except=${loadExtensionArg}`,\n    `--user-data-dir=${userDataDir}`,\n    '--no-first-run',\n    '--no-default-browser-check',\n  ];\n\n  return {\n    userDataDir,\n    args,\n    cleanup: async () => {\n      await rm(profileDir, { recursive: true, force: true });\n    },\n  };\n}\n\n/**\n * Ad-blocker extension profile using the uBlock Origin CRX.\n * Reduces noise from ad networks during crawls.\n */\nexport async function adBlockerProfile(): Promise<ExtensionProfile> {\n  // uBlock Origin 1.57.2 from GitHub release (MIT licensed)\n  return prepareExtensionProfile([{\n    name: 'uBlock Origin',\n    source: 'https://github.com/gorhill/uBlock/releases/download/1.57.2/uBlock0_1.57.2.chromium.zip',\n  }]);\n}\n"],
+  "mappings": "AAcA,SAAS,YAAY,mBAAmB;AACxC,SAAS,mBAAmB,YAAY,iBAAiB;AACzD,SAAS,WAAW,OAAO,UAAU;AACrC,SAAS,YAAY;AACrB,SAAS,cAAc;AACvB,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AAEzB,MAAM,eAAe,QAAQ,IAAI,yBAAyB,KAAK,OAAO,GAAG,mBAAmB;AAqB5F,eAAe,YAAY,KAA8B;AACvD,QAAM,OAAO,KAAK,OAAO,GAAG,YAAY,YAAY,CAAC,EAAE,SAAS,KAAK,CAAC,MAAM;AAC5E,QAAM,MAAM,MAAM,MAAM,GAAG;AAC3B,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,qCAAqC,GAAG,KAAK,IAAI,MAAM,EAAE;AACtF,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,SAAS,SAAS,QAAQ,IAAI,IAAW,GAAG,MAAM;AACxD,SAAO;AACT;AAMA,eAAe,UAAU,SAAiB,SAAgC;AACxE,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,kBAAkB;AACpD,QAAM,MAAM,MAAM,SAAS,OAAO;AAElC,QAAM,QAAQ,IAAI,aAAa,CAAC;AAChC,MAAI,UAAU,YAAY;AAExB,UAAM,MAAM,SAAS,EAAE,WAAW,KAAK,CAAC;AACxC,UAAM,UAAU,KAAK,SAAS,WAAW,GAAG,GAAG;AAC/C;AAAA,EACF;AAEA,QAAM,UAAU,IAAI,aAAa,CAAC;AAClC,MAAI;AACJ,MAAI,YAAY,GAAG;AACjB,UAAM,aAAa,IAAI,aAAa,CAAC;AACrC,eAAW,KAAK;AAAA,EAClB,WAAW,YAAY,GAAG;AACxB,UAAM,YAAY,IAAI,aAAa,CAAC;AACpC,UAAM,SAAS,IAAI,aAAa,EAAE;AAClC,eAAW,KAAK,YAAY;AAAA,EAC9B,OAAO;AACL,UAAM,IAAI,MAAM,uBAAuB,OAAO,EAAE;AAAA,EAClD;AAEA,QAAM,SAAS,IAAI,SAAS,QAAQ;AACpC,QAAM,SAAS,KAAK,OAAO,GAAG,YAAY,YAAY,CAAC,EAAE,SAAS,KAAK,CAAC,MAAM;AAC9E,QAAM,UAAU,QAAQ,MAAM;AAE9B,QAAM,MAAM,SAAS,EAAE,WAAW,KAAK,CAAC;AAIxC,QAAM,WAAW,MAAM,OAAO,UAAU,EAAE,MAAM,MAAM,IAAI;AAC1D,MAAI,UAAU;AACZ,UAAM,SAAS,KAAK,KAAK,MAAM,EAAE,KAAK,CAAC,MAAW,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC,CAAC;AAAA,EAChF,OAAO;AAEL,UAAM,EAAE,SAAS,IAAI,MAAM,OAAO,oBAAoB;AACtD,UAAM,IAAI;AAAA,MAAc,CAAC,KAAK,QAC5B,SAAS,SAAS,CAAC,MAAM,QAAQ,MAAM,OAAO,GAAG,CAAC,QAAQ,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC;AAAA,IAClF;AAAA,EACF;AACA,QAAM,GAAG,QAAQ,EAAE,OAAO,KAAK,CAAC;AAClC;AAMA,eAAsB,wBACpB,YAC2B;AAE3B,QAAM,MAAM,WAAW,KAAK,EACzB,OAAO,WAAW,IAAI,OAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EACrD,OAAO,KAAK,EACZ,MAAM,GAAG,EAAE;AAEd,QAAM,aAAa,KAAK,cAAc,GAAG;AACzC,QAAM,gBAA0B,CAAC;AAEjC,YAAU,YAAY,EAAE,WAAW,KAAK,CAAC;AAEzC,aAAW,OAAO,YAAY;AAC5B,UAAM,UAAU,WAAW,KAAK,EAAE,OAAO,IAAI,MAAM,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,CAAC;AAC7E,UAAM,SAAS,KAAK,YAAY,OAAO,OAAO,EAAE;AAEhD,QAAI,CAAC,WAAW,MAAM,GAAG;AACvB,UAAI,UAAU,IAAI;AAElB,UAAI,IAAI,OAAO,WAAW,SAAS,KAAK,IAAI,OAAO,WAAW,UAAU,GAAG;AACzE,kBAAU,MAAM,YAAY,IAAI,MAAM;AACtC,cAAM,UAAU,SAAS,MAAM;AAC/B,cAAM,GAAG,SAAS,EAAE,OAAO,KAAK,CAAC;AAAA,MACnC,WAAW,IAAI,OAAO,SAAS,MAAM,GAAG;AACtC,cAAM,UAAU,IAAI,QAAQ,MAAM;AAAA,MACpC,OAAO;AAEL,sBAAc,KAAK,IAAI,MAAM;AAC7B;AAAA,MACF;AAAA,IACF;AAEA,kBAAc,KAAK,MAAM;AAAA,EAC3B;AAEA,QAAM,cAAc,KAAK,YAAY,WAAW;AAChD,YAAU,aAAa,EAAE,WAAW,KAAK,CAAC;AAG1C,QAAM,cAAc,KAAK,aAAa,SAAS;AAC/C,YAAU,aAAa,EAAE,WAAW,KAAK,CAAC;AAC1C,QAAM,YAAY,KAAK,aAAa,aAAa;AACjD,MAAI,CAAC,WAAW,SAAS,GAAG;AAC1B,UAAM,UAAU,WAAW,KAAK,UAAU;AAAA,MACxC,SAAS,EAAE,uBAAuB,KAAK;AAAA,MACvC,YAAY,EAAE,QAAQ,EAAE,aAAa,KAAK,EAAE;AAAA,IAC9C,CAAC,GAAG,MAAM;AAAA,EACZ;AAEA,QAAM,mBAAmB,cAAc,KAAK,GAAG;AAC/C,QAAM,OAAiB;AAAA,IACrB,oBAAoB,gBAAgB;AAAA,IACpC,+BAA+B,gBAAgB;AAAA,IAC/C,mBAAmB,WAAW;AAAA,IAC9B;AAAA,IACA;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS,YAAY;AACnB,YAAM,GAAG,YAAY,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC;AAAA,IACvD;AAAA,EACF;AACF;AAMA,eAAsB,mBAA8C;AAElE,SAAO,wBAAwB,CAAC;AAAA,IAC9B,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC,CAAC;AACJ;",
+  "names": []
+}

package/dist/captcha-solver.d.ts

@@ -0,0 +1,16 @@
+/**
+ * CAPTCHA auto-solving via 2captcha or CapSolver APIs.
+ * Supports reCAPTCHA v2, hCaptcha, and Cloudflare Turnstile.
+ */
+export type CaptchaSolverProvider = '2captcha' | 'capsolver';
+interface SolveResult {
+    token: string;
+    provider: CaptchaSolverProvider;
+}
+/**
+ * Detect CAPTCHA type and sitekey on the page, then solve it.
+ * Returns null if no CAPTCHA found or solver not configured.
+ * Throws on solve failure.
+ */
+export declare function solveCaptcha(page: any, pageUrl: string, apiKey: string, provider?: CaptchaSolverProvider): Promise<SolveResult | null>;
+export {};

package/dist/captcha-solver.js

@@ -0,0 +1,105 @@
+const POLL_INTERVAL_MS = 3e3;
+const MAX_ATTEMPTS = 40;
+async function solveCaptcha(page, pageUrl, apiKey, provider = "2captcha") {
+  const captchaInfo = await page.evaluate(() => {
+    const recaptchaEl = document.querySelector(".g-recaptcha, [data-sitekey]");
+    if (recaptchaEl) {
+      const sitekey = recaptchaEl.getAttribute("data-sitekey") || "";
+      return { type: "recaptcha", sitekey };
+    }
+    const hcaptchaEl = document.querySelector(".h-captcha, [data-hcaptcha-sitekey]");
+    if (hcaptchaEl) {
+      const sitekey = hcaptchaEl.getAttribute("data-sitekey") || hcaptchaEl.getAttribute("data-hcaptcha-sitekey") || "";
+      return { type: "hcaptcha", sitekey };
+    }
+    const turnstileEl = document.querySelector(".cf-turnstile");
+    if (turnstileEl) {
+      const sitekey = turnstileEl.getAttribute("data-sitekey") || "";
+      return { type: "turnstile", sitekey };
+    }
+    return null;
+  });
+  if (!captchaInfo?.sitekey) return null;
+  const token = provider === "capsolver" ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey) : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);
+  await page.evaluate((t, type) => {
+    if (type === "recaptcha") {
+      const el = document.querySelector("textarea.g-recaptcha-response");
+      if (el) {
+        el.value = t;
+      }
+      if (window.grecaptcha?.getResponse && document.querySelector("[data-callback]")) {
+        const cb = document.querySelector("[data-callback]")?.getAttribute("data-callback");
+        if (cb && window[cb]) window[cb](t);
+      }
+    }
+    if (type === "hcaptcha") {
+      const el = document.querySelector('textarea[name="h-captcha-response"]');
+      if (el) {
+        el.value = t;
+      }
+    }
+    if (type === "turnstile") {
+      const el = document.querySelector('[name="cf-turnstile-response"]');
+      if (el) {
+        el.value = t;
+      }
+    }
+  }, token, captchaInfo.type);
+  return { token, provider };
+}
+async function solveWith2Captcha(type, sitekey, pageUrl, apiKey) {
+  const method = type === "hcaptcha" ? "hcaptcha" : type === "turnstile" ? "turnstile" : "userrecaptcha";
+  const submitRes = await fetch("http://2captcha.com/in.php", {
+    method: "POST",
+    body: new URLSearchParams({
+      key: apiKey,
+      method,
+      googlekey: sitekey,
+      pageurl: pageUrl,
+      json: "1"
+    })
+  });
+  const submitData = await submitRes.json();
+  if (submitData.status !== 1) throw new Error(`2captcha submit failed: ${submitData.request}`);
+  const taskId = submitData.request;
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+    const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);
+    const pollData = await pollRes.json();
+    if (pollData.status === 1) return pollData.request;
+    if (pollData.request !== "CAPCHA_NOT_READY") throw new Error(`2captcha error: ${pollData.request}`);
+  }
+  throw new Error("2captcha timed out after 2 minutes");
+}
+async function solveWithCapSolver(type, sitekey, pageUrl, apiKey) {
+  const taskType = type === "hcaptcha" ? "HCaptchaTaskProxyless" : type === "turnstile" ? "AntiTurnstileTaskProxyless" : "ReCaptchaV2TaskProxyless";
+  const createRes = await fetch("https://api.capsolver.com/createTask", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      clientKey: apiKey,
+      task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey }
+    })
+  });
+  const createData = await createRes.json();
+  if (createData.errorId !== 0) throw new Error(`CapSolver create failed: ${createData.errorDescription}`);
+  const taskId = createData.taskId;
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+    const pollRes = await fetch("https://api.capsolver.com/getTaskResult", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ clientKey: apiKey, taskId })
+    });
+    const pollData = await pollRes.json();
+    if (pollData.errorId !== 0) throw new Error(`CapSolver error: ${pollData.errorDescription}`);
+    if (pollData.status === "ready") {
+      return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? "";
+    }
+  }
+  throw new Error("CapSolver timed out after 2 minutes");
+}
+export {
+  solveCaptcha
+};
+//# sourceMappingURL=captcha-solver.js.map

package/dist/captcha-solver.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/captcha-solver.ts"],
+  "sourcesContent": ["/**\n * CAPTCHA auto-solving via 2captcha or CapSolver APIs.\n * Supports reCAPTCHA v2, hCaptcha, and Cloudflare Turnstile.\n */\n\nexport type CaptchaSolverProvider = '2captcha' | 'capsolver';\n\ninterface SolveResult {\n  token: string;\n  provider: CaptchaSolverProvider;\n}\n\n// Poll interval and max attempts\nconst POLL_INTERVAL_MS = 3_000;\nconst MAX_ATTEMPTS = 40; // 2 min max\n\n/**\n * Detect CAPTCHA type and sitekey on the page, then solve it.\n * Returns null if no CAPTCHA found or solver not configured.\n * Throws on solve failure.\n */\nexport async function solveCaptcha(\n  page: any,\n  pageUrl: string,\n  apiKey: string,\n  provider: CaptchaSolverProvider = '2captcha'\n): Promise<SolveResult | null> {\n  // Detect CAPTCHA type\n  const captchaInfo = await page.evaluate(() => {\n    // reCAPTCHA v2\n    const recaptchaEl = document.querySelector('.g-recaptcha, [data-sitekey]') as HTMLElement | null;\n    if (recaptchaEl) {\n      const sitekey = recaptchaEl.getAttribute('data-sitekey') || '';\n      return { type: 'recaptcha', sitekey };\n    }\n    // hCaptcha\n    const hcaptchaEl = document.querySelector('.h-captcha, [data-hcaptcha-sitekey]') as HTMLElement | null;\n    if (hcaptchaEl) {\n      const sitekey = hcaptchaEl.getAttribute('data-sitekey') || hcaptchaEl.getAttribute('data-hcaptcha-sitekey') || '';\n      return { type: 'hcaptcha', sitekey };\n    }\n    // Cloudflare Turnstile\n    const turnstileEl = document.querySelector('.cf-turnstile') as HTMLElement | null;\n    if (turnstileEl) {\n      const sitekey = turnstileEl.getAttribute('data-sitekey') || '';\n      return { type: 'turnstile', sitekey };\n    }\n    return null;\n  });\n\n  if (!captchaInfo?.sitekey) return null;\n\n  const token = provider === 'capsolver'\n    ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey)\n    : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);\n\n  // Inject the token into the page\n  await page.evaluate((t: string, type: string) => {\n    // reCAPTCHA\n    if (type === 'recaptcha') {\n      const el = document.querySelector('textarea.g-recaptcha-response') as HTMLTextAreaElement | null;\n      if (el) { el.value = t; }\n      // Also trigger the callback if registered\n      if ((window as any).grecaptcha?.getResponse && document.querySelector('[data-callback]')) {\n        const cb = (document.querySelector('[data-callback]') as HTMLElement)?.getAttribute('data-callback');\n        if (cb && (window as any)[cb]) (window as any)[cb](t);\n      }\n    }\n    // hCaptcha\n    if (type === 'hcaptcha') {\n      const el = document.querySelector('textarea[name=\"h-captcha-response\"]') as HTMLTextAreaElement | null;\n      if (el) { el.value = t; }\n    }\n    // Turnstile\n    if (type === 'turnstile') {\n      const el = document.querySelector('[name=\"cf-turnstile-response\"]') as HTMLInputElement | null;\n      if (el) { el.value = t; }\n    }\n  }, token, captchaInfo.type);\n\n  return { token, provider };\n}\n\nasync function solveWith2Captcha(type: string, sitekey: string, pageUrl: string, apiKey: string): Promise<string> {\n  const method = type === 'hcaptcha' ? 'hcaptcha' : type === 'turnstile' ? 'turnstile' : 'userrecaptcha';\n\n  const submitRes = await fetch('http://2captcha.com/in.php', {\n    method: 'POST',\n    body: new URLSearchParams({\n      key: apiKey,\n      method,\n      googlekey: sitekey,\n      pageurl: pageUrl,\n      json: '1',\n    }),\n  });\n  const submitData = await submitRes.json() as { status: number; request: string };\n  if (submitData.status !== 1) throw new Error(`2captcha submit failed: ${submitData.request}`);\n  const taskId = submitData.request;\n\n  // Poll for result\n  for (let i = 0; i < MAX_ATTEMPTS; i++) {\n    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));\n    const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);\n    const pollData = await pollRes.json() as { status: number; request: string };\n    if (pollData.status === 1) return pollData.request;\n    if (pollData.request !== 'CAPCHA_NOT_READY') throw new Error(`2captcha error: ${pollData.request}`);\n  }\n  throw new Error('2captcha timed out after 2 minutes');\n}\n\nasync function solveWithCapSolver(type: string, sitekey: string, pageUrl: string, apiKey: string): Promise<string> {\n  const taskType = type === 'hcaptcha' ? 'HCaptchaTaskProxyless'\n    : type === 'turnstile' ? 'AntiTurnstileTaskProxyless'\n    : 'ReCaptchaV2TaskProxyless';\n\n  const createRes = await fetch('https://api.capsolver.com/createTask', {\n    method: 'POST',\n    headers: { 'Content-Type': 'application/json' },\n    body: JSON.stringify({\n      clientKey: apiKey,\n      task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey },\n    }),\n  });\n  const createData = await createRes.json() as { errorId: number; taskId?: string; errorDescription?: string };\n  if (createData.errorId !== 0) throw new Error(`CapSolver create failed: ${createData.errorDescription}`);\n  const taskId = createData.taskId!;\n\n  for (let i = 0; i < MAX_ATTEMPTS; i++) {\n    await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));\n    const pollRes = await fetch('https://api.capsolver.com/getTaskResult', {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({ clientKey: apiKey, taskId }),\n    });\n    const pollData = await pollRes.json() as { errorId: number; status: string; solution?: { gRecaptchaResponse?: string; token?: string }; errorDescription?: string };\n    if (pollData.errorId !== 0) throw new Error(`CapSolver error: ${pollData.errorDescription}`);\n    if (pollData.status === 'ready') {\n      return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? '';\n    }\n  }\n  throw new Error('CapSolver timed out after 2 minutes');\n}\n"],
+  "mappings": "AAaA,MAAM,mBAAmB;AACzB,MAAM,eAAe;AAOrB,eAAsB,aACpB,MACA,SACA,QACA,WAAkC,YACL;AAE7B,QAAM,cAAc,MAAM,KAAK,SAAS,MAAM;AAE5C,UAAM,cAAc,SAAS,cAAc,8BAA8B;AACzE,QAAI,aAAa;AACf,YAAM,UAAU,YAAY,aAAa,cAAc,KAAK;AAC5D,aAAO,EAAE,MAAM,aAAa,QAAQ;AAAA,IACtC;AAEA,UAAM,aAAa,SAAS,cAAc,qCAAqC;AAC/E,QAAI,YAAY;AACd,YAAM,UAAU,WAAW,aAAa,cAAc,KAAK,WAAW,aAAa,uBAAuB,KAAK;AAC/G,aAAO,EAAE,MAAM,YAAY,QAAQ;AAAA,IACrC;AAEA,UAAM,cAAc,SAAS,cAAc,eAAe;AAC1D,QAAI,aAAa;AACf,YAAM,UAAU,YAAY,aAAa,cAAc,KAAK;AAC5D,aAAO,EAAE,MAAM,aAAa,QAAQ;AAAA,IACtC;AACA,WAAO;AAAA,EACT,CAAC;AAED,MAAI,CAAC,aAAa,QAAS,QAAO;AAElC,QAAM,QAAQ,aAAa,cACvB,MAAM,mBAAmB,YAAY,MAAM,YAAY,SAAS,SAAS,MAAM,IAC/E,MAAM,kBAAkB,YAAY,MAAM,YAAY,SAAS,SAAS,MAAM;AAGlF,QAAM,KAAK,SAAS,CAAC,GAAW,SAAiB;AAE/C,QAAI,SAAS,aAAa;AACxB,YAAM,KAAK,SAAS,cAAc,+BAA+B;AACjE,UAAI,IAAI;AAAE,WAAG,QAAQ;AAAA,MAAG;AAExB,UAAK,OAAe,YAAY,eAAe,SAAS,cAAc,iBAAiB,GAAG;AACxF,cAAM,KAAM,SAAS,cAAc,iBAAiB,GAAmB,aAAa,eAAe;AACnG,YAAI,MAAO,OAAe,EAAE,EAAG,CAAC,OAAe,EAAE,EAAE,CAAC;AAAA,MACtD;AAAA,IACF;AAEA,QAAI,SAAS,YAAY;AACvB,YAAM,KAAK,SAAS,cAAc,qCAAqC;AACvE,UAAI,IAAI;AAAE,WAAG,QAAQ;AAAA,MAAG;AAAA,IAC1B;AAEA,QAAI,SAAS,aAAa;AACxB,YAAM,KAAK,SAAS,cAAc,gCAAgC;AAClE,UAAI,IAAI;AAAE,WAAG,QAAQ;AAAA,MAAG;AAAA,IAC1B;AAAA,EACF,GAAG,OAAO,YAAY,IAAI;AAE1B,SAAO,EAAE,OAAO,SAAS;AAC3B;AAEA,eAAe,kBAAkB,MAAc,SAAiB,SAAiB,QAAiC;AAChH,QAAM,SAAS,SAAS,aAAa,aAAa,SAAS,cAAc,cAAc;AAEvF,QAAM,YAAY,MAAM,MAAM,8BAA8B;AAAA,IAC1D,QAAQ;AAAA,IACR,MAAM,IAAI,gBAAgB;AAAA,MACxB,KAAK;AAAA,MACL;AAAA,MACA,WAAW;AAAA,MACX,SAAS;AAAA,MACT,MAAM;AAAA,IACR,CAAC;AAAA,EACH,CAAC;AACD,QAAM,aAAa,MAAM,UAAU,KAAK;AACxC,MAAI,WAAW,WAAW,EAAG,OAAM,IAAI,MAAM,2BAA2B,WAAW,OAAO,EAAE;AAC5F,QAAM,SAAS,WAAW;AAG1B,WAAS,IAAI,GAAG,IAAI,cAAc,KAAK;AACrC,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,gBAAgB,CAAC;AACtD,UAAM,UAAU,MAAM,MAAM,mCAAmC,MAAM,kBAAkB,MAAM,SAAS;AACtG,UAAM,WAAW,MAAM,QAAQ,KAAK;AACpC,QAAI,SAAS,WAAW,EAAG,QAAO,SAAS;AAC3C,QAAI,SAAS,YAAY,mBAAoB,OAAM,IAAI,MAAM,mBAAmB,SAAS,OAAO,EAAE;AAAA,EACpG;AACA,QAAM,IAAI,MAAM,oCAAoC;AACtD;AAEA,eAAe,mBAAmB,MAAc,SAAiB,SAAiB,QAAiC;AACjH,QAAM,WAAW,SAAS,aAAa,0BACnC,SAAS,cAAc,+BACvB;AAEJ,QAAM,YAAY,MAAM,MAAM,wCAAwC;AAAA,IACpE,QAAQ;AAAA,IACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,IAC9C,MAAM,KAAK,UAAU;AAAA,MACnB,WAAW;AAAA,MACX,MAAM,EAAE,MAAM,UAAU,YAAY,SAAS,YAAY,QAAQ;AAAA,IACnE,CAAC;AAAA,EACH,CAAC;AACD,QAAM,aAAa,MAAM,UAAU,KAAK;AACxC,MAAI,WAAW,YAAY,EAAG,OAAM,IAAI,MAAM,4BAA4B,WAAW,gBAAgB,EAAE;AACvG,QAAM,SAAS,WAAW;AAE1B,WAAS,IAAI,GAAG,IAAI,cAAc,KAAK;AACrC,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,gBAAgB,CAAC;AACtD,UAAM,UAAU,MAAM,MAAM,2CAA2C;AAAA,MACrE,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU,EAAE,WAAW,QAAQ,OAAO,CAAC;AAAA,IACpD,CAAC;AACD,UAAM,WAAW,MAAM,QAAQ,KAAK;AACpC,QAAI,SAAS,YAAY,EAAG,OAAM,IAAI,MAAM,oBAAoB,SAAS,gBAAgB,EAAE;AAC3F,QAAI,SAAS,WAAW,SAAS;AAC/B,aAAO,SAAS,UAAU,sBAAsB,SAAS,UAAU,SAAS;AAAA,IAC9E;AAAA,EACF;AACA,QAAM,IAAI,MAAM,qCAAqC;AACvD;",
+  "names": []
+}

package/dist/color-contrast.d.ts

@@ -0,0 +1,30 @@
+/**
+ * WCAG color contrast checker.
+ *
+ * Evaluates text contrast ratios against backgrounds for all visible text elements.
+ * Reports elements that fail WCAG 2.1 AA (4.5:1 for normal text, 3:1 for large text)
+ * or AAA (7:1 for normal, 4.5:1 for large text) requirements.
+ *
+ * Uses computedStyle to get actual rendered colors — accounts for CSS variables
+ * and inheritance.
+ */
+import type { Page } from 'playwright';
+export interface ContrastViolation {
+    selector: string;
+    text: string;
+    foreground: string;
+    background: string;
+    ratio: number;
+    required: number;
+    level: 'AA' | 'AAA';
+    isLargeText: boolean;
+    severity: 'fail' | 'warn';
+}
+export interface ContrastAuditResult {
+    url: string;
+    violations: ContrastViolation[];
+    passCount: number;
+    score: number;
+    auditedAt: string;
+}
+export declare function auditColorContrast(page: Page, url: string): Promise<ContrastAuditResult>;

package/dist/color-contrast.js

@@ -0,0 +1,78 @@
+async function auditColorContrast(page, url) {
+  const auditedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const violations = await page.evaluate(() => {
+    function parseRgb(color) {
+      const m = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?/);
+      return m ? [+m[1], +m[2], +m[3], m[4] !== void 0 ? +m[4] : 1] : null;
+    }
+    function luminance([r, g, b]) {
+      const toLinear = (c) => {
+        const n = c / 255;
+        return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);
+      };
+      return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);
+    }
+    function contrastRatio(fg, bg) {
+      const l1 = luminance(fg);
+      const l2 = luminance(bg);
+      const lighter = Math.max(l1, l2);
+      const darker = Math.min(l1, l2);
+      return Math.round((lighter + 0.05) / (darker + 0.05) * 100) / 100;
+    }
+    const violations2 = [];
+    const elements = Array.from(document.querySelectorAll("p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button"));
+    let checked = 0;
+    for (const el of elements) {
+      if (checked >= 100) break;
+      const text = (el.textContent ?? "").trim().slice(0, 60);
+      if (!text) continue;
+      const style = window.getComputedStyle(el);
+      if (style.display === "none" || style.visibility === "hidden") continue;
+      const fg = parseRgb(style.color);
+      const bg = parseRgb(style.backgroundColor);
+      if (!fg || !bg || bg[3] === 0) continue;
+      const fontSize = parseFloat(style.fontSize);
+      const fontWeight = style.fontWeight;
+      const isLargeText = fontSize >= 18 || fontSize >= 14 && (fontWeight === "bold" || +fontWeight >= 700);
+      const ratio = contrastRatio(fg, bg);
+      const aaRequired = isLargeText ? 3 : 4.5;
+      const aaaRequired = isLargeText ? 4.5 : 7;
+      if (ratio < aaRequired) {
+        violations2.push({
+          selector: el.tagName.toLowerCase(),
+          text: text.slice(0, 50),
+          foreground: style.color,
+          background: style.backgroundColor,
+          ratio,
+          required: aaRequired,
+          level: "AA",
+          isLargeText,
+          severity: "fail"
+        });
+      } else if (ratio < aaaRequired) {
+        violations2.push({
+          selector: el.tagName.toLowerCase(),
+          text: text.slice(0, 50),
+          foreground: style.color,
+          background: style.backgroundColor,
+          ratio,
+          required: aaaRequired,
+          level: "AAA",
+          isLargeText,
+          severity: "warn"
+        });
+      } else {
+        checked++;
+      }
+    }
+    return violations2;
+  }).catch(() => []);
+  const passCount = Math.max(0, 100 - violations.length);
+  const criticals = violations.filter((v) => v.severity === "fail").length;
+  const score = Math.max(0, 100 - criticals * 15 - violations.filter((v) => v.severity === "warn").length * 3);
+  return { url, violations, passCount, score, auditedAt };
+}
+export {
+  auditColorContrast
+};
+//# sourceMappingURL=color-contrast.js.map

package/dist/color-contrast.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/color-contrast.ts"],
+  "sourcesContent": ["/**\n * WCAG color contrast checker.\n *\n * Evaluates text contrast ratios against backgrounds for all visible text elements.\n * Reports elements that fail WCAG 2.1 AA (4.5:1 for normal text, 3:1 for large text)\n * or AAA (7:1 for normal, 4.5:1 for large text) requirements.\n *\n * Uses computedStyle to get actual rendered colors \u2014 accounts for CSS variables\n * and inheritance.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface ContrastViolation {\n  selector: string;\n  text: string;\n  foreground: string;   // rgb(r, g, b)\n  background: string;   // rgb(r, g, b)\n  ratio: number;        // actual contrast ratio\n  required: number;     // required ratio (4.5 or 3.0)\n  level: 'AA' | 'AAA';\n  isLargeText: boolean; // font-size >= 18px or bold >= 14px\n  severity: 'fail' | 'warn'; // fail = below AA, warn = below AAA\n}\n\nexport interface ContrastAuditResult {\n  url: string;\n  violations: ContrastViolation[];\n  passCount: number;\n  score: number; // 0-100\n  auditedAt: string;\n}\n\nexport async function auditColorContrast(page: Page, url: string): Promise<ContrastAuditResult> {\n  const auditedAt = new Date().toISOString();\n\n  const violations: ContrastViolation[] = await page.evaluate(() => {\n    function parseRgb(color: string): [number, number, number, number] | null {\n      const m = color.match(/rgba?\\((\\d+),\\s*(\\d+),\\s*(\\d+)(?:,\\s*([\\d.]+))?/);\n      return m ? [+m[1], +m[2], +m[3], m[4] !== undefined ? +m[4] : 1] : null;\n    }\n    function luminance([r, g, b]: [number, number, number, number]): number {\n      const toLinear = (c: number) => {\n        const n = c / 255;\n        return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);\n      };\n      return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);\n    }\n    function contrastRatio(fg: [number,number,number,number], bg: [number,number,number,number]): number {\n      const l1 = luminance(fg);\n      const l2 = luminance(bg);\n      const lighter = Math.max(l1, l2);\n      const darker = Math.min(l1, l2);\n      return Math.round(((lighter + 0.05) / (darker + 0.05)) * 100) / 100;\n    }\n\n    const violations: any[] = [];\n    const elements = Array.from(document.querySelectorAll('p, h1, h2, h3, h4, h5, h6, a, span, li, td, th, label, button'));\n    let checked = 0;\n\n    for (const el of elements as HTMLElement[]) {\n      if (checked >= 100) break; // limit to avoid perf hit\n      const text = (el.textContent ?? '').trim().slice(0, 60);\n      if (!text) continue;\n\n      const style = window.getComputedStyle(el);\n      if (style.display === 'none' || style.visibility === 'hidden') continue;\n\n      const fg = parseRgb(style.color);\n      const bg = parseRgb(style.backgroundColor);\n      if (!fg || !bg || bg[3] === 0) continue; // transparent bg \u2014 can't check\n\n      const fontSize = parseFloat(style.fontSize);\n      const fontWeight = style.fontWeight;\n      const isLargeText = fontSize >= 18 || (fontSize >= 14 && (fontWeight === 'bold' || +fontWeight >= 700));\n      const ratio = contrastRatio(fg, bg);\n      const aaRequired = isLargeText ? 3.0 : 4.5;\n      const aaaRequired = isLargeText ? 4.5 : 7.0;\n\n      if (ratio < aaRequired) {\n        violations.push({\n          selector: el.tagName.toLowerCase(),\n          text: text.slice(0, 50),\n          foreground: style.color,\n          background: style.backgroundColor,\n          ratio,\n          required: aaRequired,\n          level: 'AA',\n          isLargeText,\n          severity: 'fail',\n        });\n      } else if (ratio < aaaRequired) {\n        violations.push({\n          selector: el.tagName.toLowerCase(),\n          text: text.slice(0, 50),\n          foreground: style.color,\n          background: style.backgroundColor,\n          ratio,\n          required: aaaRequired,\n          level: 'AAA',\n          isLargeText,\n          severity: 'warn',\n        });\n      } else {\n        checked++;\n      }\n    }\n    return violations;\n  }).catch(() => [] as ContrastViolation[]);\n\n  const passCount = Math.max(0, 100 - violations.length);\n  const criticals = violations.filter(v => v.severity === 'fail').length;\n  const score = Math.max(0, 100 - criticals * 15 - violations.filter(v => v.severity === 'warn').length * 3);\n\n  return { url, violations, passCount, score, auditedAt };\n}\n"],
+  "mappings": "AAiCA,eAAsB,mBAAmB,MAAY,KAA2C;AAC9F,QAAM,aAAY,oBAAI,KAAK,GAAE,YAAY;AAEzC,QAAM,aAAkC,MAAM,KAAK,SAAS,MAAM;AAChE,aAAS,SAAS,OAAwD;AACxE,YAAM,IAAI,MAAM,MAAM,iDAAiD;AACvE,aAAO,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,MAAM,SAAY,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI;AAAA,IACrE;AACA,aAAS,UAAU,CAAC,GAAG,GAAG,CAAC,GAA6C;AACtE,YAAM,WAAW,CAAC,MAAc;AAC9B,cAAM,IAAI,IAAI;AACd,eAAO,KAAK,UAAU,IAAI,QAAQ,KAAK,KAAK,IAAI,SAAS,OAAO,GAAG;AAAA,MACrE;AACA,aAAO,SAAS,SAAS,CAAC,IAAI,SAAS,SAAS,CAAC,IAAI,SAAS,SAAS,CAAC;AAAA,IAC1E;AACA,aAAS,cAAc,IAAmC,IAA2C;AACnG,YAAM,KAAK,UAAU,EAAE;AACvB,YAAM,KAAK,UAAU,EAAE;AACvB,YAAM,UAAU,KAAK,IAAI,IAAI,EAAE;AAC/B,YAAM,SAAS,KAAK,IAAI,IAAI,EAAE;AAC9B,aAAO,KAAK,OAAQ,UAAU,SAAS,SAAS,QAAS,GAAG,IAAI;AAAA,IAClE;AAEA,UAAMA,cAAoB,CAAC;AAC3B,UAAM,WAAW,MAAM,KAAK,SAAS,iBAAiB,+DAA+D,CAAC;AACtH,QAAI,UAAU;AAEd,eAAW,MAAM,UAA2B;AAC1C,UAAI,WAAW,IAAK;AACpB,YAAM,QAAQ,GAAG,eAAe,IAAI,KAAK,EAAE,MAAM,GAAG,EAAE;AACtD,UAAI,CAAC,KAAM;AAEX,YAAM,QAAQ,OAAO,iBAAiB,EAAE;AACxC,UAAI,MAAM,YAAY,UAAU,MAAM,eAAe,SAAU;AAE/D,YAAM,KAAK,SAAS,MAAM,KAAK;AAC/B,YAAM,KAAK,SAAS,MAAM,eAAe;AACzC,UAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,MAAM,EAAG;AAE/B,YAAM,WAAW,WAAW,MAAM,QAAQ;AAC1C,YAAM,aAAa,MAAM;AACzB,YAAM,cAAc,YAAY,MAAO,YAAY,OAAO,eAAe,UAAU,CAAC,cAAc;AAClG,YAAM,QAAQ,cAAc,IAAI,EAAE;AAClC,YAAM,aAAa,cAAc,IAAM;AACvC,YAAM,cAAc,cAAc,MAAM;AAExC,UAAI,QAAQ,YAAY;AACtB,QAAAA,YAAW,KAAK;AAAA,UACd,UAAU,GAAG,QAAQ,YAAY;AAAA,UACjC,MAAM,KAAK,MAAM,GAAG,EAAE;AAAA,UACtB,YAAY,MAAM;AAAA,UAClB,YAAY,MAAM;AAAA,UAClB;AAAA,UACA,UAAU;AAAA,UACV,OAAO;AAAA,UACP;AAAA,UACA,UAAU;AAAA,QACZ,CAAC;AAAA,MACH,WAAW,QAAQ,aAAa;AAC9B,QAAAA,YAAW,KAAK;AAAA,UACd,UAAU,GAAG,QAAQ,YAAY;AAAA,UACjC,MAAM,KAAK,MAAM,GAAG,EAAE;AAAA,UACtB,YAAY,MAAM;AAAA,UAClB,YAAY,MAAM;AAAA,UAClB;AAAA,UACA,UAAU;AAAA,UACV,OAAO;AAAA,UACP;AAAA,UACA,UAAU;AAAA,QACZ,CAAC;AAAA,MACH,OAAO;AACL;AAAA,MACF;AAAA,IACF;AACA,WAAOA;AAAA,EACT,CAAC,EAAE,MAAM,MAAM,CAAC,CAAwB;AAExC,QAAM,YAAY,KAAK,IAAI,GAAG,MAAM,WAAW,MAAM;AACrD,QAAM,YAAY,WAAW,OAAO,OAAK,EAAE,aAAa,MAAM,EAAE;AAChE,QAAM,QAAQ,KAAK,IAAI,GAAG,MAAM,YAAY,KAAK,WAAW,OAAO,OAAK,EAAE,aAAa,MAAM,EAAE,SAAS,CAAC;AAEzG,SAAO,EAAE,KAAK,YAAY,WAAW,OAAO,UAAU;AACxD;",
+  "names": ["violations"]
+}

package/dist/content-chunker.d.ts

@@ -0,0 +1,18 @@
+export type ChunkingStrategy = 'fixed' | 'overlap' | 'sentence' | 'paragraph' | 'markdown-heading' | 'semantic' | 'recursive';
+export interface ChunkOpts {
+    strategy: ChunkingStrategy;
+    chunkSize?: number;
+    overlap?: number;
+    separators?: string[];
+}
+export interface Chunk {
+    index: number;
+    text: string;
+    tokenEstimate: number;
+    startChar: number;
+    endChar: number;
+    heading?: string;
+}
+export declare function estimateTokens(text: string): number;
+export declare function chunkContent(text: string, opts: ChunkOpts): Chunk[];
+export declare function mergeSmallChunks(chunks: Chunk[], minTokens: number): Chunk[];

package/dist/content-chunker.js

@@ -0,0 +1,152 @@
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function chunkContent(text, opts) {
+  const size = opts.chunkSize ?? (parseInt(process.env.CRAWLER_CONTENT_CHUNK_SIZE ?? "") || 2e3);
+  const charSize = size * 4;
+  const overlap = opts.overlap ?? (parseInt(process.env.CRAWLER_CONTENT_CHUNK_OVERLAP ?? "") || 200);
+  const overlapChars = overlap * 4;
+  switch (opts.strategy) {
+    case "fixed":
+      return fixedChunk(text, charSize);
+    case "overlap":
+      return overlapChunk(text, charSize, overlapChars);
+    case "sentence":
+      return sentenceChunk(text, charSize);
+    case "paragraph":
+      return paragraphChunk(text, charSize);
+    case "markdown-heading":
+      return markdownHeadingChunk(text, charSize);
+    case "semantic":
+      return semanticChunk(text, charSize);
+    case "recursive":
+      return recursiveChunk(text, charSize, opts.separators ?? ["\n\n", "\n", ". ", " "]);
+    default:
+      return fixedChunk(text, charSize);
+  }
+}
+function makeChunk(text, index, startChar, heading) {
+  return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };
+}
+function fixedChunk(text, charSize) {
+  const chunks = [];
+  let i = 0;
+  while (i < text.length) {
+    const slice = text.slice(i, i + charSize);
+    chunks.push(makeChunk(slice, chunks.length, i));
+    i += charSize;
+  }
+  return chunks;
+}
+function overlapChunk(text, charSize, overlapChars) {
+  const chunks = [];
+  let i = 0;
+  while (i < text.length) {
+    const slice = text.slice(i, i + charSize);
+    chunks.push(makeChunk(slice, chunks.length, i));
+    i += charSize - overlapChars;
+    if (i <= 0) break;
+  }
+  return chunks;
+}
+function sentenceChunk(text, charSize) {
+  const sentences = text.match(/[^.!?]*[.!?]+["']?\s*/g) ?? [text];
+  return groupIntoChunks(sentences, charSize);
+}
+function paragraphChunk(text, charSize) {
+  const paras = text.split(/\n\n+/).filter((p) => p.trim());
+  return groupIntoChunks(paras, charSize);
+}
+function markdownHeadingChunk(text, charSize) {
+  const chunks = [];
+  const headingRe = /^(#{1,3}\s.+)$/m;
+  const sections = text.split(/(?=^#{1,3}\s)/m).filter((s) => s.trim());
+  let pos = 0;
+  for (const section of sections) {
+    const headingMatch = section.match(headingRe);
+    const heading = headingMatch ? headingMatch[1].replace(/^#+\s*/, "").trim() : void 0;
+    if (section.length <= charSize) {
+      chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });
+    } else {
+      const sub = paragraphChunk(section, charSize);
+      for (const s of sub) {
+        chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });
+      }
+    }
+    pos += section.length;
+  }
+  return chunks.length > 0 ? chunks : fixedChunk(text, charSize);
+}
+function semanticChunk(text, charSize) {
+  const paras = text.split(/\n\n+/).filter((p) => p.trim());
+  const merged = [];
+  let current = "";
+  for (const para of paras) {
+    if (current.length + para.length + 2 <= charSize) {
+      current = current ? current + "\n\n" + para : para;
+    } else {
+      if (current) merged.push(current);
+      current = para;
+    }
+  }
+  if (current) merged.push(current);
+  return groupIntoChunks(merged, charSize);
+}
+function recursiveChunk(text, charSize, separators) {
+  if (text.length <= charSize) return [makeChunk(text, 0, 0)];
+  const [sep, ...rest] = separators;
+  if (!sep) return fixedChunk(text, charSize);
+  const parts = text.split(sep).filter((p) => p);
+  const chunks = [];
+  let pos = 0;
+  for (const part of parts) {
+    if (part.length <= charSize) {
+      chunks.push(makeChunk(part, chunks.length, pos));
+    } else {
+      const sub = recursiveChunk(part, charSize, rest);
+      for (const s of sub) chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });
+    }
+    pos += part.length + sep.length;
+  }
+  return chunks;
+}
+function groupIntoChunks(parts, charSize) {
+  const chunks = [];
+  let current = "";
+  let startChar = 0;
+  let pos = 0;
+  for (const part of parts) {
+    if (current.length + part.length + 1 > charSize && current) {
+      chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+      startChar = pos;
+      current = part;
+    } else {
+      current = current ? current + " " + part : part;
+    }
+    pos += part.length + 1;
+  }
+  if (current.trim()) chunks.push(makeChunk(current.trim(), chunks.length, startChar));
+  return chunks;
+}
+function mergeSmallChunks(chunks, minTokens) {
+  const minChars = minTokens * 4;
+  const result = [];
+  let i = 0;
+  while (i < chunks.length) {
+    if (chunks[i].text.length < minChars && i + 1 < chunks.length) {
+      const merged = chunks[i].text + "\n\n" + chunks[i + 1].text;
+      result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));
+      i += 2;
+    } else {
+      result.push({ ...chunks[i], index: result.length });
+      i++;
+    }
+  }
+  return result;
+}
+export {
+  chunkContent,
+  estimateTokens,
+  mergeSmallChunks
+};
+//# sourceMappingURL=content-chunker.js.map

package/dist/content-chunker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/content-chunker.ts"],
+  "sourcesContent": ["export type ChunkingStrategy = 'fixed' | 'overlap' | 'sentence' | 'paragraph' | 'markdown-heading' | 'semantic' | 'recursive';\n\nexport interface ChunkOpts {\n  strategy: ChunkingStrategy;\n  chunkSize?: number;   // token estimate target (1 token \u2248 4 chars). Default 2000.\n  overlap?: number;     // overlap tokens for 'overlap' strategy. Default 200.\n  separators?: string[]; // for 'recursive' strategy. Default ['\\\\n\\\\n', '\\\\n', '. ', ' ']\n}\n\nexport interface Chunk {\n  index: number;\n  text: string;\n  tokenEstimate: number;\n  startChar: number;\n  endChar: number;\n  heading?: string;\n}\n\nexport function estimateTokens(text: string): number {\n  return Math.ceil(text.length / 4);\n}\n\nexport function chunkContent(text: string, opts: ChunkOpts): Chunk[] {\n  const size = opts.chunkSize ?? (parseInt(process.env.CRAWLER_CONTENT_CHUNK_SIZE ?? '') || 2000);\n  const charSize = size * 4;\n  const overlap = opts.overlap ?? (parseInt(process.env.CRAWLER_CONTENT_CHUNK_OVERLAP ?? '') || 200);\n  const overlapChars = overlap * 4;\n\n  switch (opts.strategy) {\n    case 'fixed': return fixedChunk(text, charSize);\n    case 'overlap': return overlapChunk(text, charSize, overlapChars);\n    case 'sentence': return sentenceChunk(text, charSize);\n    case 'paragraph': return paragraphChunk(text, charSize);\n    case 'markdown-heading': return markdownHeadingChunk(text, charSize);\n    case 'semantic': return semanticChunk(text, charSize);\n    case 'recursive': return recursiveChunk(text, charSize, opts.separators ?? ['\\n\\n', '\\n', '. ', ' ']);\n    default: return fixedChunk(text, charSize);\n  }\n}\n\nfunction makeChunk(text: string, index: number, startChar: number, heading?: string): Chunk {\n  return { index, text, tokenEstimate: estimateTokens(text), startChar, endChar: startChar + text.length, heading };\n}\n\nfunction fixedChunk(text: string, charSize: number): Chunk[] {\n  const chunks: Chunk[] = [];\n  let i = 0;\n  while (i < text.length) {\n    const slice = text.slice(i, i + charSize);\n    chunks.push(makeChunk(slice, chunks.length, i));\n    i += charSize;\n  }\n  return chunks;\n}\n\nfunction overlapChunk(text: string, charSize: number, overlapChars: number): Chunk[] {\n  const chunks: Chunk[] = [];\n  let i = 0;\n  while (i < text.length) {\n    const slice = text.slice(i, i + charSize);\n    chunks.push(makeChunk(slice, chunks.length, i));\n    i += charSize - overlapChars;\n    if (i <= 0) break;\n  }\n  return chunks;\n}\n\nfunction sentenceChunk(text: string, charSize: number): Chunk[] {\n  const sentences = text.match(/[^.!?]*[.!?]+[\"']?\\s*/g) ?? [text];\n  return groupIntoChunks(sentences, charSize);\n}\n\nfunction paragraphChunk(text: string, charSize: number): Chunk[] {\n  const paras = text.split(/\\n\\n+/).filter(p => p.trim());\n  return groupIntoChunks(paras, charSize);\n}\n\nfunction markdownHeadingChunk(text: string, charSize: number): Chunk[] {\n  const chunks: Chunk[] = [];\n  const headingRe = /^(#{1,3}\\s.+)$/m;\n  const sections = text.split(/(?=^#{1,3}\\s)/m).filter(s => s.trim());\n  let pos = 0;\n  for (const section of sections) {\n    const headingMatch = section.match(headingRe);\n    const heading = headingMatch ? headingMatch[1].replace(/^#+\\s*/, '').trim() : undefined;\n    if (section.length <= charSize) {\n      chunks.push({ ...makeChunk(section.trim(), chunks.length, pos), heading });\n    } else {\n      // Section too large \u2014 sub-chunk by paragraph\n      const sub = paragraphChunk(section, charSize);\n      for (const s of sub) {\n        chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar, heading });\n      }\n    }\n    pos += section.length;\n  }\n  return chunks.length > 0 ? chunks : fixedChunk(text, charSize);\n}\n\nfunction semanticChunk(text: string, charSize: number): Chunk[] {\n  // Like paragraph but merges short paragraphs with the next\n  const paras = text.split(/\\n\\n+/).filter(p => p.trim());\n  const merged: string[] = [];\n  let current = '';\n  for (const para of paras) {\n    if (current.length + para.length + 2 <= charSize) {\n      current = current ? current + '\\n\\n' + para : para;\n    } else {\n      if (current) merged.push(current);\n      current = para;\n    }\n  }\n  if (current) merged.push(current);\n  return groupIntoChunks(merged, charSize);\n}\n\nfunction recursiveChunk(text: string, charSize: number, separators: string[]): Chunk[] {\n  if (text.length <= charSize) return [makeChunk(text, 0, 0)];\n  const [sep, ...rest] = separators;\n  if (!sep) return fixedChunk(text, charSize);\n  const parts = text.split(sep).filter(p => p);\n  const chunks: Chunk[] = [];\n  let pos = 0;\n  for (const part of parts) {\n    if (part.length <= charSize) {\n      chunks.push(makeChunk(part, chunks.length, pos));\n    } else {\n      const sub = recursiveChunk(part, charSize, rest);\n      for (const s of sub) chunks.push({ ...s, index: chunks.length, startChar: pos + s.startChar, endChar: pos + s.endChar });\n    }\n    pos += part.length + sep.length;\n  }\n  return chunks;\n}\n\nfunction groupIntoChunks(parts: string[], charSize: number): Chunk[] {\n  const chunks: Chunk[] = [];\n  let current = '';\n  let startChar = 0;\n  let pos = 0;\n  for (const part of parts) {\n    if (current.length + part.length + 1 > charSize && current) {\n      chunks.push(makeChunk(current.trim(), chunks.length, startChar));\n      startChar = pos;\n      current = part;\n    } else {\n      current = current ? current + ' ' + part : part;\n    }\n    pos += part.length + 1;\n  }\n  if (current.trim()) chunks.push(makeChunk(current.trim(), chunks.length, startChar));\n  return chunks;\n}\n\nexport function mergeSmallChunks(chunks: Chunk[], minTokens: number): Chunk[] {\n  const minChars = minTokens * 4;\n  const result: Chunk[] = [];\n  let i = 0;\n  while (i < chunks.length) {\n    if (chunks[i].text.length < minChars && i + 1 < chunks.length) {\n      const merged = chunks[i].text + '\\n\\n' + chunks[i + 1].text;\n      result.push(makeChunk(merged, result.length, chunks[i].startChar, chunks[i].heading ?? chunks[i + 1].heading));\n      i += 2;\n    } else {\n      result.push({ ...chunks[i], index: result.length });\n      i++;\n    }\n  }\n  return result;\n}\n"],
+  "mappings": "AAkBO,SAAS,eAAe,MAAsB;AACnD,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AAEO,SAAS,aAAa,MAAc,MAA0B;AACnE,QAAM,OAAO,KAAK,cAAc,SAAS,QAAQ,IAAI,8BAA8B,EAAE,KAAK;AAC1F,QAAM,WAAW,OAAO;AACxB,QAAM,UAAU,KAAK,YAAY,SAAS,QAAQ,IAAI,iCAAiC,EAAE,KAAK;AAC9F,QAAM,eAAe,UAAU;AAE/B,UAAQ,KAAK,UAAU;AAAA,IACrB,KAAK;AAAS,aAAO,WAAW,MAAM,QAAQ;AAAA,IAC9C,KAAK;AAAW,aAAO,aAAa,MAAM,UAAU,YAAY;AAAA,IAChE,KAAK;AAAY,aAAO,cAAc,MAAM,QAAQ;AAAA,IACpD,KAAK;AAAa,aAAO,eAAe,MAAM,QAAQ;AAAA,IACtD,KAAK;AAAoB,aAAO,qBAAqB,MAAM,QAAQ;AAAA,IACnE,KAAK;AAAY,aAAO,cAAc,MAAM,QAAQ;AAAA,IACpD,KAAK;AAAa,aAAO,eAAe,MAAM,UAAU,KAAK,cAAc,CAAC,QAAQ,MAAM,MAAM,GAAG,CAAC;AAAA,IACpG;AAAS,aAAO,WAAW,MAAM,QAAQ;AAAA,EAC3C;AACF;AAEA,SAAS,UAAU,MAAc,OAAe,WAAmB,SAAyB;AAC1F,SAAO,EAAE,OAAO,MAAM,eAAe,eAAe,IAAI,GAAG,WAAW,SAAS,YAAY,KAAK,QAAQ,QAAQ;AAClH;AAEA,SAAS,WAAW,MAAc,UAA2B;AAC3D,QAAM,SAAkB,CAAC;AACzB,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,QAAQ;AACtB,UAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,QAAQ;AACxC,WAAO,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAC9C,SAAK;AAAA,EACP;AACA,SAAO;AACT;AAEA,SAAS,aAAa,MAAc,UAAkB,cAA+B;AACnF,QAAM,SAAkB,CAAC;AACzB,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,QAAQ;AACtB,UAAM,QAAQ,KAAK,MAAM,GAAG,IAAI,QAAQ;AACxC,WAAO,KAAK,UAAU,OAAO,OAAO,QAAQ,CAAC,CAAC;AAC9C,SAAK,WAAW;AAChB,QAAI,KAAK,EAAG;AAAA,EACd;AACA,SAAO;AACT;AAEA,SAAS,cAAc,MAAc,UAA2B;AAC9D,QAAM,YAAY,KAAK,MAAM,wBAAwB,KAAK,CAAC,IAAI;AAC/D,SAAO,gBAAgB,WAAW,QAAQ;AAC5C;AAEA,SAAS,eAAe,MAAc,UAA2B;AAC/D,QAAM,QAAQ,KAAK,MAAM,OAAO,EAAE,OAAO,OAAK,EAAE,KAAK,CAAC;AACtD,SAAO,gBAAgB,OAAO,QAAQ;AACxC;AAEA,SAAS,qBAAqB,MAAc,UAA2B;AACrE,QAAM,SAAkB,CAAC;AACzB,QAAM,YAAY;AAClB,QAAM,WAAW,KAAK,MAAM,gBAAgB,EAAE,OAAO,OAAK,EAAE,KAAK,CAAC;AAClE,MAAI,MAAM;AACV,aAAW,WAAW,UAAU;AAC9B,UAAM,eAAe,QAAQ,MAAM,SAAS;AAC5C,UAAM,UAAU,eAAe,aAAa,CAAC,EAAE,QAAQ,UAAU,EAAE,EAAE,KAAK,IAAI;AAC9E,QAAI,QAAQ,UAAU,UAAU;AAC9B,aAAO,KAAK,EAAE,GAAG,UAAU,QAAQ,KAAK,GAAG,OAAO,QAAQ,GAAG,GAAG,QAAQ,CAAC;AAAA,IAC3E,OAAO;AAEL,YAAM,MAAM,eAAe,SAAS,QAAQ;AAC5C,iBAAW,KAAK,KAAK;AACnB,eAAO,KAAK,EAAE,GAAG,GAAG,OAAO,OAAO,QAAQ,WAAW,MAAM,EAAE,WAAW,SAAS,MAAM,EAAE,SAAS,QAAQ,CAAC;AAAA,MAC7G;AAAA,IACF;AACA,WAAO,QAAQ;AAAA,EACjB;AACA,SAAO,OAAO,SAAS,IAAI,SAAS,WAAW,MAAM,QAAQ;AAC/D;AAEA,SAAS,cAAc,MAAc,UAA2B;AAE9D,QAAM,QAAQ,KAAK,MAAM,OAAO,EAAE,OAAO,OAAK,EAAE,KAAK,CAAC;AACtD,QAAM,SAAmB,CAAC;AAC1B,MAAI,UAAU;AACd,aAAW,QAAQ,OAAO;AACxB,QAAI,QAAQ,SAAS,KAAK,SAAS,KAAK,UAAU;AAChD,gBAAU,UAAU,UAAU,SAAS,OAAO;AAAA,IAChD,OAAO;AACL,UAAI,QAAS,QAAO,KAAK,OAAO;AAChC,gBAAU;AAAA,IACZ;AAAA,EACF;AACA,MAAI,QAAS,QAAO,KAAK,OAAO;AAChC,SAAO,gBAAgB,QAAQ,QAAQ;AACzC;AAEA,SAAS,eAAe,MAAc,UAAkB,YAA+B;AACrF,MAAI,KAAK,UAAU,SAAU,QAAO,CAAC,UAAU,MAAM,GAAG,CAAC,CAAC;AAC1D,QAAM,CAAC,KAAK,GAAG,IAAI,IAAI;AACvB,MAAI,CAAC,IAAK,QAAO,WAAW,MAAM,QAAQ;AAC1C,QAAM,QAAQ,KAAK,MAAM,GAAG,EAAE,OAAO,OAAK,CAAC;AAC3C,QAAM,SAAkB,CAAC;AACzB,MAAI,MAAM;AACV,aAAW,QAAQ,OAAO;AACxB,QAAI,KAAK,UAAU,UAAU;AAC3B,aAAO,KAAK,UAAU,MAAM,OAAO,QAAQ,GAAG,CAAC;AAAA,IACjD,OAAO;AACL,YAAM,MAAM,eAAe,MAAM,UAAU,IAAI;AAC/C,iBAAW,KAAK,IAAK,QAAO,KAAK,EAAE,GAAG,GAAG,OAAO,OAAO,QAAQ,WAAW,MAAM,EAAE,WAAW,SAAS,MAAM,EAAE,QAAQ,CAAC;AAAA,IACzH;AACA,WAAO,KAAK,SAAS,IAAI;AAAA,EAC3B;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,OAAiB,UAA2B;AACnE,QAAM,SAAkB,CAAC;AACzB,MAAI,UAAU;AACd,MAAI,YAAY;AAChB,MAAI,MAAM;AACV,aAAW,QAAQ,OAAO;AACxB,QAAI,QAAQ,SAAS,KAAK,SAAS,IAAI,YAAY,SAAS;AAC1D,aAAO,KAAK,UAAU,QAAQ,KAAK,GAAG,OAAO,QAAQ,SAAS,CAAC;AAC/D,kBAAY;AACZ,gBAAU;AAAA,IACZ,OAAO;AACL,gBAAU,UAAU,UAAU,MAAM,OAAO;AAAA,IAC7C;AACA,WAAO,KAAK,SAAS;AAAA,EACvB;AACA,MAAI,QAAQ,KAAK,EAAG,QAAO,KAAK,UAAU,QAAQ,KAAK,GAAG,OAAO,QAAQ,SAAS,CAAC;AACnF,SAAO;AACT;AAEO,SAAS,iBAAiB,QAAiB,WAA4B;AAC5E,QAAM,WAAW,YAAY;AAC7B,QAAM,SAAkB,CAAC;AACzB,MAAI,IAAI;AACR,SAAO,IAAI,OAAO,QAAQ;AACxB,QAAI,OAAO,CAAC,EAAE,KAAK,SAAS,YAAY,IAAI,IAAI,OAAO,QAAQ;AAC7D,YAAM,SAAS,OAAO,CAAC,EAAE,OAAO,SAAS,OAAO,IAAI,CAAC,EAAE;AACvD,aAAO,KAAK,UAAU,QAAQ,OAAO,QAAQ,OAAO,CAAC,EAAE,WAAW,OAAO,CAAC,EAAE,WAAW,OAAO,IAAI,CAAC,EAAE,OAAO,CAAC;AAC7G,WAAK;AAAA,IACP,OAAO;AACL,aAAO,KAAK,EAAE,GAAG,OAAO,CAAC,GAAG,OAAO,OAAO,OAAO,CAAC;AAClD;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;",
+  "names": []
+}

package/dist/content-dedup.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Content deduplication — detect near-duplicate pages during crawl.
+ *
+ * Uses MinHash-inspired approach: instead of full MinHash (needs many hash functions),
+ * we use 8 structural fingerprints extracted from the markdown to detect pages that
+ * share the same layout/template even if the data differs.
+ *
+ * Typical duplicates: /products?page=1 vs /products?page=2 (same template, different data).
+ * These waste crawl budget and generate redundant test cases.
+ */
+export interface DedupResult {
+    isDuplicate: boolean;
+    similarUrl?: string;
+    similarityScore: number;
+}
+export declare class ContentDeduplicator {
+    private seen;
+    private readonly threshold;
+    constructor(threshold?: number);
+    check(url: string, markdown: string): DedupResult;
+    size(): number;
+    reset(): void;
+}