@detiq/crawler 1.1.8 → 1.1.9

package/dist/browser-extension.js

@@ -1,113 +1,151 @@
-import { createHash, randomBytes } from "node:crypto";
-import { createWriteStream, existsSync, mkdirSync } from "node:fs";
-import { writeFile, mkdir, rm } from "node:fs/promises";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import { pipeline } from "node:stream/promises";
-import { Readable } from "node:stream";
-const PROFILE_BASE = process.env.EXTENSION_PROFILE_DIR ?? join(tmpdir(), "zeta-ext-profiles");
+/**
+ * Browser extension injection support.
+ *
+ * Chrome extensions require a persistent profile directory — incompatible
+ * with headless Docker by default. This module handles:
+ *   1. Persistent profile directory per extension set
+ *   2. Extension download from .crx URL or load from local path
+ *   3. Launch args for Playwright / Stagehand
+ *
+ * Usage:
+ *   const profile = await prepareExtensionProfile(['/path/to/ext.crx']);
+ *   // pass profile.userDataDir and profile.args to browser launch
+ */
+import { createHash, randomBytes } from 'node:crypto';
+import { createWriteStream, existsSync, mkdirSync } from 'node:fs';
+import { writeFile, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { pipeline } from 'node:stream/promises';
+import { Readable } from 'node:stream';
+const PROFILE_BASE = process.env.EXTENSION_PROFILE_DIR ?? join(tmpdir(), 'zeta-ext-profiles');
+/**
+ * Download a .crx from a URL to a temp file. Returns path.
+ */
 async function downloadCrx(url) {
-  const dest = join(tmpdir(), `zeta-ext-${randomBytes(6).toString("hex")}.crx`);
-  const res = await fetch(url);
-  if (!res.ok) throw new Error(`Failed to download extension from ${url}: ${res.status}`);
-  const writer = createWriteStream(dest);
-  await pipeline(Readable.fromWeb(res.body), writer);
-  return dest;
+    const dest = join(tmpdir(), `zeta-ext-${randomBytes(6).toString('hex')}.crx`);
+    const res = await fetch(url);
+    if (!res.ok)
+        throw new Error(`Failed to download extension from ${url}: ${res.status}`);
+    const writer = createWriteStream(dest);
+    await pipeline(Readable.fromWeb(res.body), writer);
+    return dest;
 }
+/**
+ * Unpack a .crx file to a directory Chrome can load as an unpacked extension.
+ * CRX3 format: 4-byte magic + 4-byte version + 4-byte header_size + proto header + zip body
+ */
 async function unpackCrx(crxPath, destDir) {
-  const { readFile } = await import("node:fs/promises");
-  const buf = await readFile(crxPath);
-  const magic = buf.readUInt32LE(0);
-  if (magic !== 1131557428) {
+    const { readFile } = await import('node:fs/promises');
+    const buf = await readFile(crxPath);
+    const magic = buf.readUInt32LE(0);
+    if (magic !== 0x43723234) {
+        // Not CRX — assume it's already a zip (unpacked extension)
+        await mkdir(destDir, { recursive: true });
+        await writeFile(join(destDir, '__raw.crx'), buf);
+        return;
+    }
+    const version = buf.readUInt32LE(4);
+    let zipStart;
+    if (version === 3) {
+        const headerSize = buf.readUInt32LE(8);
+        zipStart = 12 + headerSize;
+    }
+    else if (version === 2) {
+        const pubKeyLen = buf.readUInt32LE(8);
+        const sigLen = buf.readUInt32LE(12);
+        zipStart = 16 + pubKeyLen + sigLen;
+    }
+    else {
+        throw new Error(`Unknown CRX version ${version}`);
+    }
+    const zipBuf = buf.subarray(zipStart);
+    const tmpZip = join(tmpdir(), `zeta-ext-${randomBytes(4).toString('hex')}.zip`);
+    await writeFile(tmpZip, zipBuf);
     await mkdir(destDir, { recursive: true });
-    await writeFile(join(destDir, "__raw.crx"), buf);
-    return;
-  }
-  const version = buf.readUInt32LE(4);
-  let zipStart;
-  if (version === 3) {
-    const headerSize = buf.readUInt32LE(8);
-    zipStart = 12 + headerSize;
-  } else if (version === 2) {
-    const pubKeyLen = buf.readUInt32LE(8);
-    const sigLen = buf.readUInt32LE(12);
-    zipStart = 16 + pubKeyLen + sigLen;
-  } else {
-    throw new Error(`Unknown CRX version ${version}`);
-  }
-  const zipBuf = buf.subarray(zipStart);
-  const tmpZip = join(tmpdir(), `zeta-ext-${randomBytes(4).toString("hex")}.zip`);
-  await writeFile(tmpZip, zipBuf);
-  await mkdir(destDir, { recursive: true });
-  const unzipper = await import("unzipper").catch(() => null);
-  if (unzipper) {
-    await unzipper.Open.file(tmpZip).then((d) => d.extract({ path: destDir }));
-  } else {
-    const { execFile } = await import("node:child_process");
-    await new Promise(
-      (res, rej) => execFile("unzip", ["-o", tmpZip, "-d", destDir], (err) => err ? rej(err) : res())
-    );
-  }
-  await rm(tmpZip, { force: true });
+    // Extract using unzipper (available in Node without native deps)
+    // @ts-ignore – no types for unzipper
+    const unzipper = await import('unzipper').catch(() => null);
+    if (unzipper) {
+        await unzipper.Open.file(tmpZip).then((d) => d.extract({ path: destDir }));
+    }
+    else {
+        // Fallback: shell unzip
+        const { execFile } = await import('node:child_process');
+        await new Promise((res, rej) => execFile('unzip', ['-o', tmpZip, '-d', destDir], (err) => err ? rej(err) : res()));
+    }
+    await rm(tmpZip, { force: true });
 }
-async function prepareExtensionProfile(extensions) {
-  const key = createHash("md5").update(extensions.map((e) => e.source).sort().join("|")).digest("hex").slice(0, 12);
-  const profileDir = join(PROFILE_BASE, key);
-  const extensionDirs = [];
-  mkdirSync(profileDir, { recursive: true });
-  for (const ext of extensions) {
-    const extHash = createHash("md5").update(ext.source).digest("hex").slice(0, 8);
-    const extDir = join(profileDir, `ext-${extHash}`);
-    if (!existsSync(extDir)) {
-      let crxPath = ext.source;
-      if (ext.source.startsWith("http://") || ext.source.startsWith("https://")) {
-        crxPath = await downloadCrx(ext.source);
-        await unpackCrx(crxPath, extDir);
-        await rm(crxPath, { force: true });
-      } else if (ext.source.endsWith(".crx")) {
-        await unpackCrx(ext.source, extDir);
-      } else {
-        extensionDirs.push(ext.source);
-        continue;
-      }
+/**
+ * Prepare a Chrome user-data-dir with the given extensions loaded.
+ * Returns profile info + cleanup function.
+ */
+export async function prepareExtensionProfile(extensions) {
+    // Stable profile key based on extension sources
+    const key = createHash('md5')
+        .update(extensions.map(e => e.source).sort().join('|'))
+        .digest('hex')
+        .slice(0, 12);
+    const profileDir = join(PROFILE_BASE, key);
+    const extensionDirs = [];
+    mkdirSync(profileDir, { recursive: true });
+    for (const ext of extensions) {
+        const extHash = createHash('md5').update(ext.source).digest('hex').slice(0, 8);
+        const extDir = join(profileDir, `ext-${extHash}`);
+        if (!existsSync(extDir)) {
+            let crxPath = ext.source;
+            if (ext.source.startsWith('http://') || ext.source.startsWith('https://')) {
+                crxPath = await downloadCrx(ext.source);
+                await unpackCrx(crxPath, extDir);
+                await rm(crxPath, { force: true });
+            }
+            else if (ext.source.endsWith('.crx')) {
+                await unpackCrx(ext.source, extDir);
+            }
+            else {
+                // Assume already-unpacked extension directory — symlink or copy
+                extensionDirs.push(ext.source);
+                continue;
+            }
+        }
+        extensionDirs.push(extDir);
     }
-    extensionDirs.push(extDir);
-  }
-  const userDataDir = join(profileDir, "user-data");
-  mkdirSync(userDataDir, { recursive: true });
-  const defaultsDir = join(userDataDir, "Default");
-  mkdirSync(defaultsDir, { recursive: true });
-  const prefsPath = join(defaultsDir, "Preferences");
-  if (!existsSync(prefsPath)) {
-    await writeFile(prefsPath, JSON.stringify({
-      browser: { has_seen_welcome_page: true },
-      extensions: { alerts: { initialized: true } }
-    }), "utf8");
-  }
-  const loadExtensionArg = extensionDirs.join(",");
-  const args = [
-    `--load-extension=${loadExtensionArg}`,
-    `--disable-extensions-except=${loadExtensionArg}`,
-    `--user-data-dir=${userDataDir}`,
-    "--no-first-run",
-    "--no-default-browser-check"
-  ];
-  return {
-    userDataDir,
-    args,
-    cleanup: async () => {
-      await rm(profileDir, { recursive: true, force: true });
+    const userDataDir = join(profileDir, 'user-data');
+    mkdirSync(userDataDir, { recursive: true });
+    // Write a minimal Preferences file so Chrome doesn't show first-run UI
+    const defaultsDir = join(userDataDir, 'Default');
+    mkdirSync(defaultsDir, { recursive: true });
+    const prefsPath = join(defaultsDir, 'Preferences');
+    if (!existsSync(prefsPath)) {
+        await writeFile(prefsPath, JSON.stringify({
+            browser: { has_seen_welcome_page: true },
+            extensions: { alerts: { initialized: true } },
+        }), 'utf8');
     }
-  };
+    const loadExtensionArg = extensionDirs.join(',');
+    const args = [
+        `--load-extension=${loadExtensionArg}`,
+        `--disable-extensions-except=${loadExtensionArg}`,
+        `--user-data-dir=${userDataDir}`,
+        '--no-first-run',
+        '--no-default-browser-check',
+    ];
+    return {
+        userDataDir,
+        args,
+        cleanup: async () => {
+            await rm(profileDir, { recursive: true, force: true });
+        },
+    };
 }
-async function adBlockerProfile() {
-  return prepareExtensionProfile([{
-    name: "uBlock Origin",
-    source: "https://github.com/gorhill/uBlock/releases/download/1.57.2/uBlock0_1.57.2.chromium.zip"
-  }]);
+/**
+ * Ad-blocker extension profile using the uBlock Origin CRX.
+ * Reduces noise from ad networks during crawls.
+ */
+export async function adBlockerProfile() {
+    // uBlock Origin 1.57.2 from GitHub release (MIT licensed)
+    return prepareExtensionProfile([{
+            name: 'uBlock Origin',
+            source: 'https://github.com/gorhill/uBlock/releases/download/1.57.2/uBlock0_1.57.2.chromium.zip',
+        }]);
 }
-export {
-  adBlockerProfile,
-  prepareExtensionProfile
-};
-//# sourceMappingURL=browser-extension.js.map

package/dist/captcha-solver.js

@@ -1,105 +1,132 @@
-const POLL_INTERVAL_MS = 3e3;
-const MAX_ATTEMPTS = 40;
-async function solveCaptcha(page, pageUrl, apiKey, provider = "2captcha") {
-  const captchaInfo = await page.evaluate(() => {
-    const recaptchaEl = document.querySelector(".g-recaptcha, [data-sitekey]");
-    if (recaptchaEl) {
-      const sitekey = recaptchaEl.getAttribute("data-sitekey") || "";
-      return { type: "recaptcha", sitekey };
-    }
-    const hcaptchaEl = document.querySelector(".h-captcha, [data-hcaptcha-sitekey]");
-    if (hcaptchaEl) {
-      const sitekey = hcaptchaEl.getAttribute("data-sitekey") || hcaptchaEl.getAttribute("data-hcaptcha-sitekey") || "";
-      return { type: "hcaptcha", sitekey };
-    }
-    const turnstileEl = document.querySelector(".cf-turnstile");
-    if (turnstileEl) {
-      const sitekey = turnstileEl.getAttribute("data-sitekey") || "";
-      return { type: "turnstile", sitekey };
-    }
-    return null;
-  });
-  if (!captchaInfo?.sitekey) return null;
-  const token = provider === "capsolver" ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey) : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);
-  await page.evaluate((t, type) => {
-    if (type === "recaptcha") {
-      const el = document.querySelector("textarea.g-recaptcha-response");
-      if (el) {
-        el.value = t;
-      }
-      if (window.grecaptcha?.getResponse && document.querySelector("[data-callback]")) {
-        const cb = document.querySelector("[data-callback]")?.getAttribute("data-callback");
-        if (cb && window[cb]) window[cb](t);
-      }
-    }
-    if (type === "hcaptcha") {
-      const el = document.querySelector('textarea[name="h-captcha-response"]');
-      if (el) {
-        el.value = t;
-      }
-    }
-    if (type === "turnstile") {
-      const el = document.querySelector('[name="cf-turnstile-response"]');
-      if (el) {
-        el.value = t;
-      }
-    }
-  }, token, captchaInfo.type);
-  return { token, provider };
+/**
+ * CAPTCHA auto-solving via 2captcha or CapSolver APIs.
+ * Supports reCAPTCHA v2, hCaptcha, and Cloudflare Turnstile.
+ */
+// Poll interval and max attempts
+const POLL_INTERVAL_MS = 3_000;
+const MAX_ATTEMPTS = 40; // 2 min max
+/**
+ * Detect CAPTCHA type and sitekey on the page, then solve it.
+ * Returns null if no CAPTCHA found or solver not configured.
+ * Throws on solve failure.
+ */
+export async function solveCaptcha(page, pageUrl, apiKey, provider = '2captcha') {
+    // Detect CAPTCHA type
+    const captchaInfo = await page.evaluate(() => {
+        // reCAPTCHA v2
+        const recaptchaEl = document.querySelector('.g-recaptcha, [data-sitekey]');
+        if (recaptchaEl) {
+            const sitekey = recaptchaEl.getAttribute('data-sitekey') || '';
+            return { type: 'recaptcha', sitekey };
+        }
+        // hCaptcha
+        const hcaptchaEl = document.querySelector('.h-captcha, [data-hcaptcha-sitekey]');
+        if (hcaptchaEl) {
+            const sitekey = hcaptchaEl.getAttribute('data-sitekey') || hcaptchaEl.getAttribute('data-hcaptcha-sitekey') || '';
+            return { type: 'hcaptcha', sitekey };
+        }
+        // Cloudflare Turnstile
+        const turnstileEl = document.querySelector('.cf-turnstile');
+        if (turnstileEl) {
+            const sitekey = turnstileEl.getAttribute('data-sitekey') || '';
+            return { type: 'turnstile', sitekey };
+        }
+        return null;
+    });
+    if (!captchaInfo?.sitekey)
+        return null;
+    const token = provider === 'capsolver'
+        ? await solveWithCapSolver(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey)
+        : await solveWith2Captcha(captchaInfo.type, captchaInfo.sitekey, pageUrl, apiKey);
+    // Inject the token into the page
+    await page.evaluate((t, type) => {
+        // reCAPTCHA
+        if (type === 'recaptcha') {
+            const el = document.querySelector('textarea.g-recaptcha-response');
+            if (el) {
+                el.value = t;
+            }
+            // Also trigger the callback if registered
+            if (window.grecaptcha?.getResponse && document.querySelector('[data-callback]')) {
+                const cb = document.querySelector('[data-callback]')?.getAttribute('data-callback');
+                if (cb && window[cb])
+                    window[cb](t);
+            }
+        }
+        // hCaptcha
+        if (type === 'hcaptcha') {
+            const el = document.querySelector('textarea[name="h-captcha-response"]');
+            if (el) {
+                el.value = t;
+            }
+        }
+        // Turnstile
+        if (type === 'turnstile') {
+            const el = document.querySelector('[name="cf-turnstile-response"]');
+            if (el) {
+                el.value = t;
+            }
+        }
+    }, token, captchaInfo.type);
+    return { token, provider };
 }
 async function solveWith2Captcha(type, sitekey, pageUrl, apiKey) {
-  const method = type === "hcaptcha" ? "hcaptcha" : type === "turnstile" ? "turnstile" : "userrecaptcha";
-  const submitRes = await fetch("http://2captcha.com/in.php", {
-    method: "POST",
-    body: new URLSearchParams({
-      key: apiKey,
-      method,
-      googlekey: sitekey,
-      pageurl: pageUrl,
-      json: "1"
-    })
-  });
-  const submitData = await submitRes.json();
-  if (submitData.status !== 1) throw new Error(`2captcha submit failed: ${submitData.request}`);
-  const taskId = submitData.request;
-  for (let i = 0; i < MAX_ATTEMPTS; i++) {
-    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-    const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);
-    const pollData = await pollRes.json();
-    if (pollData.status === 1) return pollData.request;
-    if (pollData.request !== "CAPCHA_NOT_READY") throw new Error(`2captcha error: ${pollData.request}`);
-  }
-  throw new Error("2captcha timed out after 2 minutes");
+    const method = type === 'hcaptcha' ? 'hcaptcha' : type === 'turnstile' ? 'turnstile' : 'userrecaptcha';
+    const submitRes = await fetch('http://2captcha.com/in.php', {
+        method: 'POST',
+        body: new URLSearchParams({
+            key: apiKey,
+            method,
+            googlekey: sitekey,
+            pageurl: pageUrl,
+            json: '1',
+        }),
+    });
+    const submitData = await submitRes.json();
+    if (submitData.status !== 1)
+        throw new Error(`2captcha submit failed: ${submitData.request}`);
+    const taskId = submitData.request;
+    // Poll for result
+    for (let i = 0; i < MAX_ATTEMPTS; i++) {
+        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+        const pollRes = await fetch(`http://2captcha.com/res.php?key=${apiKey}&action=get&id=${taskId}&json=1`);
+        const pollData = await pollRes.json();
+        if (pollData.status === 1)
+            return pollData.request;
+        if (pollData.request !== 'CAPCHA_NOT_READY')
+            throw new Error(`2captcha error: ${pollData.request}`);
+    }
+    throw new Error('2captcha timed out after 2 minutes');
 }
 async function solveWithCapSolver(type, sitekey, pageUrl, apiKey) {
-  const taskType = type === "hcaptcha" ? "HCaptchaTaskProxyless" : type === "turnstile" ? "AntiTurnstileTaskProxyless" : "ReCaptchaV2TaskProxyless";
-  const createRes = await fetch("https://api.capsolver.com/createTask", {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({
-      clientKey: apiKey,
-      task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey }
-    })
-  });
-  const createData = await createRes.json();
-  if (createData.errorId !== 0) throw new Error(`CapSolver create failed: ${createData.errorDescription}`);
-  const taskId = createData.taskId;
-  for (let i = 0; i < MAX_ATTEMPTS; i++) {
-    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-    const pollRes = await fetch("https://api.capsolver.com/getTaskResult", {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ clientKey: apiKey, taskId })
+    const taskType = type === 'hcaptcha' ? 'HCaptchaTaskProxyless'
+        : type === 'turnstile' ? 'AntiTurnstileTaskProxyless'
+            : 'ReCaptchaV2TaskProxyless';
+    const createRes = await fetch('https://api.capsolver.com/createTask', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            clientKey: apiKey,
+            task: { type: taskType, websiteURL: pageUrl, websiteKey: sitekey },
+        }),
     });
-    const pollData = await pollRes.json();
-    if (pollData.errorId !== 0) throw new Error(`CapSolver error: ${pollData.errorDescription}`);
-    if (pollData.status === "ready") {
-      return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? "";
+    const createData = await createRes.json();
+    if (createData.errorId !== 0)
+        throw new Error(`CapSolver create failed: ${createData.errorDescription}`);
+    const taskId = createData.taskId;
+    for (let i = 0; i < MAX_ATTEMPTS; i++) {
+        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+        const pollRes = await fetch('https://api.capsolver.com/getTaskResult', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ clientKey: apiKey, taskId }),
+        });
+        const pollData = await pollRes.json();
+        if (pollData.errorId !== 0)
+            throw new Error(`CapSolver error: ${pollData.errorDescription}`);
+        if (pollData.status === 'ready') {
+            return pollData.solution?.gRecaptchaResponse ?? pollData.solution?.token ?? '';
+        }
     }
-  }
-  throw new Error("CapSolver timed out after 2 minutes");
+    throw new Error('CapSolver timed out after 2 minutes');
 }
-export {
-  solveCaptcha
-};
-//# sourceMappingURL=captcha-solver.js.map