@detiq/crawler 1.1.9 → 1.1.10

package/dist/browser-extension.js

@@ -1,151 +1,113 @@
-/**
- * Browser extension injection support.
- *
- * Chrome extensions require a persistent profile directory — incompatible
- * with headless Docker by default. This module handles:
- *   1. Persistent profile directory per extension set
- *   2. Extension download from .crx URL or load from local path
- *   3. Launch args for Playwright / Stagehand
- *
- * Usage:
- *   const profile = await prepareExtensionProfile(['/path/to/ext.crx']);
- *   // pass profile.userDataDir and profile.args to browser launch
- */
-import { createHash, randomBytes } from 'node:crypto';
-import { createWriteStream, existsSync, mkdirSync } from 'node:fs';
-import { writeFile, mkdir, rm } from 'node:fs/promises';
-import { join } from 'node:path';
-import { tmpdir } from 'node:os';
-import { pipeline } from 'node:stream/promises';
-import { Readable } from 'node:stream';
-const PROFILE_BASE = process.env.EXTENSION_PROFILE_DIR ?? join(tmpdir(), 'zeta-ext-profiles');
-/**
- * Download a .crx from a URL to a temp file. Returns path.
- */
+import { createHash, randomBytes } from "node:crypto";
+import { createWriteStream, existsSync, mkdirSync } from "node:fs";
+import { writeFile, mkdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { pipeline } from "node:stream/promises";
+import { Readable } from "node:stream";
+const PROFILE_BASE = process.env.EXTENSION_PROFILE_DIR ?? join(tmpdir(), "zeta-ext-profiles");
 async function downloadCrx(url) {
-    const dest = join(tmpdir(), `zeta-ext-${randomBytes(6).toString('hex')}.crx`);
-    const res = await fetch(url);
-    if (!res.ok)
-        throw new Error(`Failed to download extension from ${url}: ${res.status}`);
-    const writer = createWriteStream(dest);
-    await pipeline(Readable.fromWeb(res.body), writer);
-    return dest;
+  const dest = join(tmpdir(), `zeta-ext-${randomBytes(6).toString("hex")}.crx`);
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Failed to download extension from ${url}: ${res.status}`);
+  const writer = createWriteStream(dest);
+  await pipeline(Readable.fromWeb(res.body), writer);
+  return dest;
 }
-/**
- * Unpack a .crx file to a directory Chrome can load as an unpacked extension.
- * CRX3 format: 4-byte magic + 4-byte version + 4-byte header_size + proto header + zip body
- */
 async function unpackCrx(crxPath, destDir) {
-    const { readFile } = await import('node:fs/promises');
-    const buf = await readFile(crxPath);
-    const magic = buf.readUInt32LE(0);
-    if (magic !== 0x43723234) {
-        // Not CRX — assume it's already a zip (unpacked extension)
-        await mkdir(destDir, { recursive: true });
-        await writeFile(join(destDir, '__raw.crx'), buf);
-        return;
-    }
-    const version = buf.readUInt32LE(4);
-    let zipStart;
-    if (version === 3) {
-        const headerSize = buf.readUInt32LE(8);
-        zipStart = 12 + headerSize;
-    }
-    else if (version === 2) {
-        const pubKeyLen = buf.readUInt32LE(8);
-        const sigLen = buf.readUInt32LE(12);
-        zipStart = 16 + pubKeyLen + sigLen;
-    }
-    else {
-        throw new Error(`Unknown CRX version ${version}`);
-    }
-    const zipBuf = buf.subarray(zipStart);
-    const tmpZip = join(tmpdir(), `zeta-ext-${randomBytes(4).toString('hex')}.zip`);
-    await writeFile(tmpZip, zipBuf);
+  const { readFile } = await import("node:fs/promises");
+  const buf = await readFile(crxPath);
+  const magic = buf.readUInt32LE(0);
+  if (magic !== 1131557428) {
     await mkdir(destDir, { recursive: true });
-    // Extract using unzipper (available in Node without native deps)
-    // @ts-ignore – no types for unzipper
-    const unzipper = await import('unzipper').catch(() => null);
-    if (unzipper) {
-        await unzipper.Open.file(tmpZip).then((d) => d.extract({ path: destDir }));
-    }
-    else {
-        // Fallback: shell unzip
-        const { execFile } = await import('node:child_process');
-        await new Promise((res, rej) => execFile('unzip', ['-o', tmpZip, '-d', destDir], (err) => err ? rej(err) : res()));
-    }
-    await rm(tmpZip, { force: true });
+    await writeFile(join(destDir, "__raw.crx"), buf);
+    return;
+  }
+  const version = buf.readUInt32LE(4);
+  let zipStart;
+  if (version === 3) {
+    const headerSize = buf.readUInt32LE(8);
+    zipStart = 12 + headerSize;
+  } else if (version === 2) {
+    const pubKeyLen = buf.readUInt32LE(8);
+    const sigLen = buf.readUInt32LE(12);
+    zipStart = 16 + pubKeyLen + sigLen;
+  } else {
+    throw new Error(`Unknown CRX version ${version}`);
+  }
+  const zipBuf = buf.subarray(zipStart);
+  const tmpZip = join(tmpdir(), `zeta-ext-${randomBytes(4).toString("hex")}.zip`);
+  await writeFile(tmpZip, zipBuf);
+  await mkdir(destDir, { recursive: true });
+  const unzipper = await import("unzipper").catch(() => null);
+  if (unzipper) {
+    await unzipper.Open.file(tmpZip).then((d) => d.extract({ path: destDir }));
+  } else {
+    const { execFile } = await import("node:child_process");
+    await new Promise(
+      (res, rej) => execFile("unzip", ["-o", tmpZip, "-d", destDir], (err) => err ? rej(err) : res())
+    );
+  }
+  await rm(tmpZip, { force: true });
 }
-/**
- * Prepare a Chrome user-data-dir with the given extensions loaded.
- * Returns profile info + cleanup function.
- */
-export async function prepareExtensionProfile(extensions) {
-    // Stable profile key based on extension sources
-    const key = createHash('md5')
-        .update(extensions.map(e => e.source).sort().join('|'))
-        .digest('hex')
-        .slice(0, 12);
-    const profileDir = join(PROFILE_BASE, key);
-    const extensionDirs = [];
-    mkdirSync(profileDir, { recursive: true });
-    for (const ext of extensions) {
-        const extHash = createHash('md5').update(ext.source).digest('hex').slice(0, 8);
-        const extDir = join(profileDir, `ext-${extHash}`);
-        if (!existsSync(extDir)) {
-            let crxPath = ext.source;
-            if (ext.source.startsWith('http://') || ext.source.startsWith('https://')) {
-                crxPath = await downloadCrx(ext.source);
-                await unpackCrx(crxPath, extDir);
-                await rm(crxPath, { force: true });
-            }
-            else if (ext.source.endsWith('.crx')) {
-                await unpackCrx(ext.source, extDir);
-            }
-            else {
-                // Assume already-unpacked extension directory — symlink or copy
-                extensionDirs.push(ext.source);
-                continue;
-            }
-        }
-        extensionDirs.push(extDir);
+async function prepareExtensionProfile(extensions) {
+  const key = createHash("md5").update(extensions.map((e) => e.source).sort().join("|")).digest("hex").slice(0, 12);
+  const profileDir = join(PROFILE_BASE, key);
+  const extensionDirs = [];
+  mkdirSync(profileDir, { recursive: true });
+  for (const ext of extensions) {
+    const extHash = createHash("md5").update(ext.source).digest("hex").slice(0, 8);
+    const extDir = join(profileDir, `ext-${extHash}`);
+    if (!existsSync(extDir)) {
+      let crxPath = ext.source;
+      if (ext.source.startsWith("http://") || ext.source.startsWith("https://")) {
+        crxPath = await downloadCrx(ext.source);
+        await unpackCrx(crxPath, extDir);
+        await rm(crxPath, { force: true });
+      } else if (ext.source.endsWith(".crx")) {
+        await unpackCrx(ext.source, extDir);
+      } else {
+        extensionDirs.push(ext.source);
+        continue;
+      }
     }
-    const userDataDir = join(profileDir, 'user-data');
-    mkdirSync(userDataDir, { recursive: true });
-    // Write a minimal Preferences file so Chrome doesn't show first-run UI
-    const defaultsDir = join(userDataDir, 'Default');
-    mkdirSync(defaultsDir, { recursive: true });
-    const prefsPath = join(defaultsDir, 'Preferences');
-    if (!existsSync(prefsPath)) {
-        await writeFile(prefsPath, JSON.stringify({
-            browser: { has_seen_welcome_page: true },
-            extensions: { alerts: { initialized: true } },
-        }), 'utf8');
+    extensionDirs.push(extDir);
+  }
+  const userDataDir = join(profileDir, "user-data");
+  mkdirSync(userDataDir, { recursive: true });
+  const defaultsDir = join(userDataDir, "Default");
+  mkdirSync(defaultsDir, { recursive: true });
+  const prefsPath = join(defaultsDir, "Preferences");
+  if (!existsSync(prefsPath)) {
+    await writeFile(prefsPath, JSON.stringify({
+      browser: { has_seen_welcome_page: true },
+      extensions: { alerts: { initialized: true } }
+    }), "utf8");
+  }
+  const loadExtensionArg = extensionDirs.join(",");
+  const args = [
+    `--load-extension=${loadExtensionArg}`,
+    `--disable-extensions-except=${loadExtensionArg}`,
+    `--user-data-dir=${userDataDir}`,
+    "--no-first-run",
+    "--no-default-browser-check"
+  ];
+  return {
+    userDataDir,
+    args,
+    cleanup: async () => {
+      await rm(profileDir, { recursive: true, force: true });
     }
-    const loadExtensionArg = extensionDirs.join(',');
-    const args = [
-        `--load-extension=${loadExtensionArg}`,
-        `--disable-extensions-except=${loadExtensionArg}`,
-        `--user-data-dir=${userDataDir}`,
-        '--no-first-run',
-        '--no-default-browser-check',
-    ];
-    return {
-        userDataDir,
-        args,
-        cleanup: async () => {
-            await rm(profileDir, { recursive: true, force: true });
-        },
-    };
+  };
 }
-/**
- * Ad-blocker extension profile using the uBlock Origin CRX.
- * Reduces noise from ad networks during crawls.
- */
-export async function adBlockerProfile() {
-    // uBlock Origin 1.57.2 from GitHub release (MIT licensed)
-    return prepareExtensionProfile([{
-            name: 'uBlock Origin',
-            source: 'https://github.com/gorhill/uBlock/releases/download/1.57.2/uBlock0_1.57.2.chromium.zip',
-        }]);
+async function adBlockerProfile() {
+  return prepareExtensionProfile([{
+    name: "uBlock Origin",
+    source: "https://github.com/gorhill/uBlock/releases/download/1.57.2/uBlock0_1.57.2.chromium.zip"
+  }]);
 }
+export {
+  adBlockerProfile,
+  prepareExtensionProfile
+};
+//# sourceMappingURL=browser-extension.js.map

package/dist/captcha-solver.js

@@ -1,132 +1,105 @@
-/**
- * CAPTCHA auto-solving via 2captcha or CapSolver APIs.
- * Supports reCAPTCHA v2, hCaptcha, and Cloudflare Turnstile.
- */
-// Poll interval and max attempts
-const POLL_INTERVAL_MS = 3_000;
-const MAX_ATTEMPTS = 40; // 2 min max
-/**
- * Detect CAPTCHA type and sitekey on the page, then solve it.
- * Returns null if no CAPTCHA found or solver not configured.
- * Throws on solve failure.
- */
-export async function solveCaptcha(page, pageUrl, apiKey, provider = '2captcha') {
-    // Detect CAPTCHA type
-    const captchaInfo = await page.evaluate(() => {
-        // reCAPTCHA v2
-        const recaptchaEl = document.querySelector('.g-recaptcha, [data-sitekey]');
-        if (recaptchaEl) {
-            const sitekey = recaptchaEl.getAttribute('data-sitekey') || '';
-            return { type: 'recaptcha', sitekey };
-        }
-        // hCaptcha
-        const hcaptchaEl = document.querySelector('.h-captcha, [data-hcaptcha-sitekey]');
-        if (hcaptchaEl) {
-            const sitekey = hcaptchaEl.getAttribute('data-sitekey') || hcaptchaEl.getAttribute('data-hcaptcha-sitekey') || '';
-            return { type: 'hcaptcha', sitekey };
-        }
-        // Cloudflare Turnstile
-        const turnstileEl = document.querySelector('.cf-turnstile');
-        if (turnstileEl) {
-            const sitekey = turnstileEl.getAttribute('data-sitekey') || '';
-            return { type: 'turnstile', sitekey };
-        }
-        return null;
-    });
-    if (!captchaInfo?.sitekey)
-        return null;
-    const token = provider === 'capsolver'
-        ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey)
-        : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);
-    // Inject the token into the page
-    await page.evaluate((t, type) => {
-        // reCAPTCHA
-        if (type === 'recaptcha') {
-            const el = document.querySelector('textarea.g-recaptcha-response');
-            if (el) {
-                el.value = t;
-            }
-            // Also trigger the callback if registered
-            if (window.grecaptcha?.getResponse && document.querySelector('[data-callback]')) {
-                const cb = document.querySelector('[data-callback]')?.getAttribute('data-callback');
-                if (cb && window[cb])
-                    window[cb](t);
-            }
-        }
-        // hCaptcha
-        if (type === 'hcaptcha') {
-            const el = document.querySelector('textarea[name="h-captcha-response"]');
-            if (el) {
-                el.value = t;
-            }
-        }
-        // Turnstile
-        if (type === 'turnstile') {
-            const el = document.querySelector('[name="cf-turnstile-response"]');
-            if (el) {
-                el.value = t;
-            }
-        }
-    }, token, captchaInfo.type);
-    return { token, provider };
+const POLL_INTERVAL_MS = 3e3;
+const MAX_ATTEMPTS = 40;
+async function solveCaptcha(page, pageUrl, apiKey, provider = "2captcha") {
+  const captchaInfo = await page.evaluate(() => {
+    const recaptchaEl = document.querySelector(".g-recaptcha, [data-sitekey]");
+    if (recaptchaEl) {
+      const sitekey = recaptchaEl.getAttribute("data-sitekey") || "";
+      return { type: "recaptcha", sitekey };
+    }
+    const hcaptchaEl = document.querySelector(".h-captcha, [data-hcaptcha-sitekey]");
+    if (hcaptchaEl) {
+      const sitekey = hcaptchaEl.getAttribute("data-sitekey") || hcaptchaEl.getAttribute("data-hcaptcha-sitekey") || "";
+      return { type: "hcaptcha", sitekey };
+    }
+    const turnstileEl = document.querySelector(".cf-turnstile");
+    if (turnstileEl) {
+      const sitekey = turnstileEl.getAttribute("data-sitekey") || "";
+      return { type: "turnstile", sitekey };
+    }
+    return null;
+  });
+  if (!captchaInfo?.sitekey) return null;
+  const token = provider === "capsolver" ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey) : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);
+  await page.evaluate((t, type) => {
+    if (type === "recaptcha") {
+      const el = document.querySelector("textarea.g-recaptcha-response");
+      if (el) {
+        el.value = t;
+      }
+      if (window.grecaptcha?.getResponse && document.querySelector("[data-callback]")) {
+        const cb = document.querySelector("[data-callback]")?.getAttribute("data-callback");
+        if (cb && window[cb]) window[cb](t);
+      }
+    }
+    if (type === "hcaptcha") {
+      const el = document.querySelector('textarea[name="h-captcha-response"]');
+      if (el) {
+        el.value = t;
+      }
+    }
+    if (type === "turnstile") {
+      const el = document.querySelector('[name="cf-turnstile-response"]');
+      if (el) {
+        el.value = t;
+      }
+    }
+  }, token, captchaInfo.type);
+  return { token, provider };
 }
 async function solveWith2Captcha(type, sitekey, pageUrl, apiKey) {
-    const method = type === 'hcaptcha' ? 'hcaptcha' : type === 'turnstile' ? 'turnstile' : 'userrecaptcha';
-    const submitRes = await fetch('http://2captcha.com/in.php', {
-        method: 'POST',
-        body: new URLSearchParams({
-            key: apiKey,
-            method,
-            googlekey: sitekey,
-            pageurl: pageUrl,
-            json: '1',
-        }),
-    });
-    const submitData = await submitRes.json();
-    if (submitData.status !== 1)
-        throw new Error(`2captcha submit failed: ${submitData.request}`);
-    const taskId = submitData.request;
-    // Poll for result
-    for (let i = 0; i < MAX_ATTEMPTS; i++) {
-        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
-        const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);
-        const pollData = await pollRes.json();
-        if (pollData.status === 1)
-            return pollData.request;
-        if (pollData.request !== 'CAPCHA_NOT_READY')
-            throw new Error(`2captcha error: ${pollData.request}`);
-    }
-    throw new Error('2captcha timed out after 2 minutes');
+  const method = type === "hcaptcha" ? "hcaptcha" : type === "turnstile" ? "turnstile" : "userrecaptcha";
+  const submitRes = await fetch("http://2captcha.com/in.php", {
+    method: "POST",
+    body: new URLSearchParams({
+      key: apiKey,
+      method,
+      googlekey: sitekey,
+      pageurl: pageUrl,
+      json: "1"
+    })
+  });
+  const submitData = await submitRes.json();
+  if (submitData.status !== 1) throw new Error(`2captcha submit failed: ${submitData.request}`);
+  const taskId = submitData.request;
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+    const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);
+    const pollData = await pollRes.json();
+    if (pollData.status === 1) return pollData.request;
+    if (pollData.request !== "CAPCHA_NOT_READY") throw new Error(`2captcha error: ${pollData.request}`);
+  }
+  throw new Error("2captcha timed out after 2 minutes");
 }
 async function solveWithCapSolver(type, sitekey, pageUrl, apiKey) {
-    const taskType = type === 'hcaptcha' ? 'HCaptchaTaskProxyless'
-        : type === 'turnstile' ? 'AntiTurnstileTaskProxyless'
-            : 'ReCaptchaV2TaskProxyless';
-    const createRes = await fetch('https://api.capsolver.com/createTask', {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({
-            clientKey: apiKey,
-            task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey },
-        }),
+  const taskType = type === "hcaptcha" ? "HCaptchaTaskProxyless" : type === "turnstile" ? "AntiTurnstileTaskProxyless" : "ReCaptchaV2TaskProxyless";
+  const createRes = await fetch("https://api.capsolver.com/createTask", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      clientKey: apiKey,
+      task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey }
+    })
+  });
+  const createData = await createRes.json();
+  if (createData.errorId !== 0) throw new Error(`CapSolver create failed: ${createData.errorDescription}`);
+  const taskId = createData.taskId;
+  for (let i = 0; i < MAX_ATTEMPTS; i++) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+    const pollRes = await fetch("https://api.capsolver.com/getTaskResult", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ clientKey: apiKey, taskId })
     });
-    const createData = await createRes.json();
-    if (createData.errorId !== 0)
-        throw new Error(`CapSolver create failed: ${createData.errorDescription}`);
-    const taskId = createData.taskId;
-    for (let i = 0; i < MAX_ATTEMPTS; i++) {
-        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
-        const pollRes = await fetch('https://api.capsolver.com/getTaskResult', {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({ clientKey: apiKey, taskId }),
-        });
-        const pollData = await pollRes.json();
-        if (pollData.errorId !== 0)
-            throw new Error(`CapSolver error: ${pollData.errorDescription}`);
-        if (pollData.status === 'ready') {
-            return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? '';
-        }
+    const pollData = await pollRes.json();
+    if (pollData.errorId !== 0) throw new Error(`CapSolver error: ${pollData.errorDescription}`);
+    if (pollData.status === "ready") {
+      return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? "";
     }
-    throw new Error('CapSolver timed out after 2 minutes');
+  }
+  throw new Error("CapSolver timed out after 2 minutes");
 }
+export {
+  solveCaptcha
+};
+//# sourceMappingURL=captcha-solver.js.map