@vakra-dev/reader 0.0.3 → 0.1.0

package/dist/cli/index.js

@@ -17,162 +17,6 @@ import { ConnectionToHeroCore } from "@ulixee/hero";
 // src/scraper.ts
 import pLimit from "p-limit";
 
-// src/cloudflare/detector.ts
-var CLOUDFLARE_CHALLENGE_SELECTORS = [
-  "#challenge-running",
-  "#challenge-stage",
-  "#challenge-form",
-  ".cf-browser-verification",
-  "#cf-wrapper",
-  "#cf-hcaptcha-container",
-  "#turnstile-wrapper"
-];
-var CLOUDFLARE_TEXT_PATTERNS = [
-  "checking if the site connection is secure",
-  "this process is automatic. your browser will redirect",
-  "ray id:",
-  "performance & security by cloudflare"
-];
-var CLOUDFLARE_INFRA_PATTERNS = [
-  "/cdn-cgi/",
-  "cloudflare",
-  "__cf_bm",
-  "cf-ray"
-];
-var CLOUDFLARE_BLOCKED_PATTERNS = [
-  "sorry, you have been blocked",
-  "ray id:"
-];
-async function detectChallenge(hero) {
-  const signals = [];
-  let type = "none";
-  let hasCloudflareInfra = false;
-  let hasChallengeIndicator = false;
-  try {
-    if (!hero.document) {
-      return {
-        isChallenge: false,
-        type: "none",
-        confidence: 0,
-        signals: ["No document available"]
-      };
-    }
-    const html = await hero.document.documentElement.outerHTML;
-    const htmlLower = html.toLowerCase();
-    for (const pattern of CLOUDFLARE_INFRA_PATTERNS) {
-      if (htmlLower.includes(pattern)) {
-        hasCloudflareInfra = true;
-        signals.push(`Cloudflare infra: "${pattern}"`);
-        break;
-      }
-    }
-    if (!hasCloudflareInfra) {
-      return {
-        isChallenge: false,
-        type: "none",
-        confidence: 0,
-        signals: ["No Cloudflare infrastructure detected"]
-      };
-    }
-    for (const selector of CLOUDFLARE_CHALLENGE_SELECTORS) {
-      try {
-        const element = await hero.document.querySelector(selector);
-        if (element) {
-          hasChallengeIndicator = true;
-          signals.push(`Challenge element: ${selector}`);
-          type = "js_challenge";
-        }
-      } catch {
-      }
-    }
-    for (const pattern of CLOUDFLARE_TEXT_PATTERNS) {
-      if (htmlLower.includes(pattern)) {
-        hasChallengeIndicator = true;
-        signals.push(`Challenge text: "${pattern}"`);
-        type = type === "none" ? "js_challenge" : type;
-      }
-    }
-    if (htmlLower.includes("waiting for") && htmlLower.includes("to respond")) {
-      hasChallengeIndicator = true;
-      signals.push('Challenge text: "waiting for...to respond"');
-      type = type === "none" ? "js_challenge" : type;
-    }
-    const hasBlocked = CLOUDFLARE_BLOCKED_PATTERNS.every((p) => htmlLower.includes(p));
-    if (hasBlocked) {
-      hasChallengeIndicator = true;
-      signals.push("Cloudflare block page detected");
-      type = "blocked";
-    }
-    const isChallenge = hasCloudflareInfra && hasChallengeIndicator;
-    const confidence = isChallenge ? 100 : 0;
-    return {
-      isChallenge,
-      type: isChallenge ? type : "none",
-      confidence,
-      signals
-    };
-  } catch (error) {
-    return {
-      isChallenge: false,
-      type: "none",
-      confidence: 0,
-      signals: [`Error during detection: ${error.message}`]
-    };
-  }
-}
-
-// src/cloudflare/handler.ts
-async function waitForChallengeResolution(hero, options) {
-  const { maxWaitMs = 45e3, pollIntervalMs = 500, verbose = false, initialUrl } = options;
-  const startTime = Date.now();
-  const log = (msg) => verbose && console.log(`   ${msg}`);
-  while (Date.now() - startTime < maxWaitMs) {
-    const elapsed = Date.now() - startTime;
-    try {
-      const currentUrl = await hero.url;
-      if (currentUrl !== initialUrl) {
-        log(`\u2713 URL changed: ${initialUrl} \u2192 ${currentUrl}`);
-        log(`  Waiting for new page to load...`);
-        try {
-          await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
-          log(`  DOMContentLoaded`);
-        } catch {
-          log(`  DOMContentLoaded timeout, continuing...`);
-        }
-        await hero.waitForPaintingStable().catch(() => {
-        });
-        log(`  Page stabilized`);
-        return { resolved: true, method: "url_redirect", waitedMs: elapsed };
-      }
-    } catch {
-    }
-    const detection = await detectChallenge(hero);
-    if (!detection.isChallenge) {
-      log(`\u2713 Challenge signals cleared (confidence dropped to ${detection.confidence})`);
-      log(`  Waiting for page to load...`);
-      try {
-        await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
-        log(`  DOMContentLoaded`);
-      } catch {
-        log(`  DOMContentLoaded timeout, continuing...`);
-      }
-      await hero.waitForPaintingStable().catch(() => {
-      });
-      log(`  Page stabilized`);
-      return { resolved: true, method: "signals_cleared", waitedMs: elapsed };
-    }
-    log(
-      `\u23F3 ${(elapsed / 1e3).toFixed(1)}s - Still challenge (confidence: ${detection.confidence})`
-    );
-    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
-  }
-  return {
-    resolved: false,
-    method: "timeout",
-    waitedMs: Date.now() - startTime
-  };
-}
-
 // src/formatters/markdown.ts
 import TurndownService from "turndown";
 var turndownService = new TurndownService({
@@ -1054,96 +898,667 @@ var DEFAULT_OPTIONS = {
   showChrome: false
 };
 
-// src/scraper.ts
-var Scraper = class {
-  options;
-  pool;
-  logger = createLogger("scraper");
-  robotsCache = /* @__PURE__ */ new Map();
-  constructor(options) {
-    this.options = {
-      ...DEFAULT_OPTIONS,
-      ...options
-    };
-    if (!options.pool) {
-      throw new Error("Browser pool must be provided. Use ReaderClient for automatic pool management.");
+// src/engines/types.ts
+var ENGINE_CONFIGS = {
+  http: {
+    name: "http",
+    timeout: 3e3,
+    maxTimeout: 1e4,
+    quality: 100,
+    features: {
+      javascript: false,
+      cloudflare: false,
+      tlsFingerprint: false,
+      waitFor: false,
+      screenshots: false
+    }
+  },
+  tlsclient: {
+    name: "tlsclient",
+    timeout: 5e3,
+    maxTimeout: 15e3,
+    quality: 80,
+    features: {
+      javascript: false,
+      cloudflare: false,
+      tlsFingerprint: true,
+      waitFor: false,
+      screenshots: false
+    }
+  },
+  hero: {
+    name: "hero",
+    timeout: 3e4,
+    maxTimeout: 6e4,
+    quality: 50,
+    features: {
+      javascript: true,
+      cloudflare: true,
+      tlsFingerprint: true,
+      waitFor: true,
+      screenshots: true
     }
-    this.pool = options.pool;
   }
-  /**
-   * Get robots.txt rules for a URL, cached per domain
-   */
-  async getRobotsRules(url) {
-    const origin = new URL(url).origin;
-    if (!this.robotsCache.has(origin)) {
-      const rules = await fetchRobotsTxt(origin);
-      this.robotsCache.set(origin, rules);
+};
+var DEFAULT_ENGINE_ORDER = ["http", "tlsclient", "hero"];
+
+// src/engines/errors.ts
+var EngineError = class extends Error {
+  engine;
+  retryable;
+  constructor(engine, message, options) {
+    super(`[${engine}] ${message}`);
+    this.name = "EngineError";
+    this.engine = engine;
+    this.retryable = options?.retryable ?? true;
+    this.cause = options?.cause;
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, this.constructor);
     }
-    return this.robotsCache.get(origin) ?? null;
   }
-  /**
-   * Scrape all URLs
-   *
-   * @returns Scrape result with pages and metadata
-   */
-  async scrape() {
-    const startTime = Date.now();
-    const results = await this.scrapeWithConcurrency();
-    return this.buildScrapeResult(results, startTime);
+};
+var ChallengeDetectedError = class extends EngineError {
+  challengeType;
+  constructor(engine, challengeType) {
+    super(engine, `Challenge detected: ${challengeType || "unknown"}`, { retryable: true });
+    this.name = "ChallengeDetectedError";
+    this.challengeType = challengeType || "unknown";
   }
-  /**
-   * Scrape URLs with concurrency control
-   */
-  async scrapeWithConcurrency() {
-    const limit = pLimit(this.options.batchConcurrency || 1);
-    const tasks = this.options.urls.map(
-      (url, index) => limit(() => this.scrapeSingleUrlWithRetry(url, index))
-    );
-    const batchPromise = Promise.all(tasks);
-    if (this.options.batchTimeoutMs && this.options.batchTimeoutMs > 0) {
-      const timeoutPromise = new Promise((_, reject) => {
-        setTimeout(() => {
-          reject(new Error(`Batch operation timed out after ${this.options.batchTimeoutMs}ms`));
-        }, this.options.batchTimeoutMs);
+};
+var InsufficientContentError = class extends EngineError {
+  contentLength;
+  threshold;
+  constructor(engine, contentLength, threshold = 100) {
+    super(engine, `Insufficient content: ${contentLength} chars (threshold: ${threshold})`, { retryable: true });
+    this.name = "InsufficientContentError";
+    this.contentLength = contentLength;
+    this.threshold = threshold;
+  }
+};
+var HttpError = class extends EngineError {
+  statusCode;
+  constructor(engine, statusCode, statusText) {
+    const retryable = statusCode >= 500 || statusCode === 429;
+    super(engine, `HTTP ${statusCode}${statusText ? `: ${statusText}` : ""}`, { retryable });
+    this.name = "HttpError";
+    this.statusCode = statusCode;
+  }
+};
+var EngineTimeoutError = class extends EngineError {
+  timeoutMs;
+  constructor(engine, timeoutMs) {
+    super(engine, `Timeout after ${timeoutMs}ms`, { retryable: true });
+    this.name = "EngineTimeoutError";
+    this.timeoutMs = timeoutMs;
+  }
+};
+var EngineUnavailableError = class extends EngineError {
+  constructor(engine, reason) {
+    super(engine, reason || "Engine not available", { retryable: false });
+    this.name = "EngineUnavailableError";
+  }
+};
+var AllEnginesFailedError = class extends Error {
+  attemptedEngines;
+  errors;
+  constructor(attemptedEngines, errors) {
+    const summary = attemptedEngines.map((e) => `${e}: ${errors.get(e)?.message || "unknown"}`).join("; ");
+    super(`All engines failed: ${summary}`);
+    this.name = "AllEnginesFailedError";
+    this.attemptedEngines = attemptedEngines;
+    this.errors = errors;
+  }
+};
+
+// src/engines/http/index.ts
+var DEFAULT_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.9",
+  "Accept-Encoding": "gzip, deflate, br",
+  "Cache-Control": "no-cache",
+  Pragma: "no-cache",
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "none",
+  "Sec-Fetch-User": "?1",
+  "Upgrade-Insecure-Requests": "1"
+};
+var CHALLENGE_PATTERNS = [
+  // Cloudflare
+  "cf-browser-verification",
+  "cf_chl_opt",
+  "challenge-platform",
+  "cf-spinner",
+  "Just a moment",
+  "Checking your browser",
+  "checking if the site connection is secure",
+  "Enable JavaScript and cookies",
+  "Attention Required",
+  "_cf_chl_tk",
+  "Verifying you are human",
+  "cf-turnstile",
+  "/cdn-cgi/challenge-platform/",
+  // Generic bot detection
+  "Please Wait...",
+  "DDoS protection by",
+  "Access denied",
+  "bot detection",
+  "are you a robot",
+  "complete the security check"
+];
+var CLOUDFLARE_INFRA_PATTERNS = ["/cdn-cgi/", "cloudflare", "__cf_bm", "cf-ray"];
+var MIN_CONTENT_LENGTH = 100;
+var HttpEngine = class {
+  config = ENGINE_CONFIGS.http;
+  async scrape(meta) {
+    const startTime = Date.now();
+    const { url, options, logger: logger4, abortSignal } = meta;
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), this.config.maxTimeout);
+      if (abortSignal) {
+        abortSignal.addEventListener("abort", () => controller.abort(), { once: true });
+      }
+      logger4?.debug(`[http] Fetching ${url}`);
+      const response = await fetch(url, {
+        method: "GET",
+        headers: {
+          ...DEFAULT_HEADERS,
+          ...options.headers || {}
+        },
+        redirect: "follow",
+        signal: controller.signal
       });
-      return Promise.race([batchPromise, timeoutPromise]);
+      clearTimeout(timeoutId);
+      const duration = Date.now() - startTime;
+      const html = await response.text();
+      logger4?.debug(`[http] Got response: ${response.status} (${html.length} chars) in ${duration}ms`);
+      if (response.status >= 400) {
+        throw new HttpError("http", response.status, response.statusText);
+      }
+      const challengeType = this.detectChallenge(html);
+      if (challengeType) {
+        logger4?.debug(`[http] Challenge detected: ${challengeType}`);
+        throw new ChallengeDetectedError("http", challengeType);
+      }
+      const textContent = this.extractText(html);
+      if (textContent.length < MIN_CONTENT_LENGTH) {
+        logger4?.debug(`[http] Insufficient content: ${textContent.length} chars`);
+        throw new InsufficientContentError("http", textContent.length, MIN_CONTENT_LENGTH);
+      }
+      return {
+        html,
+        url: response.url,
+        statusCode: response.status,
+        contentType: response.headers.get("content-type") || void 0,
+        headers: this.headersToRecord(response.headers),
+        engine: "http",
+        duration
+      };
+    } catch (error) {
+      if (error instanceof ChallengeDetectedError || error instanceof InsufficientContentError || error instanceof HttpError) {
+        throw error;
+      }
+      if (error instanceof Error) {
+        if (error.name === "AbortError") {
+          throw new EngineTimeoutError("http", this.config.maxTimeout);
+        }
+        throw new EngineError("http", error.message, { cause: error });
+      }
+      throw new EngineError("http", String(error));
     }
-    return batchPromise;
   }
   /**
-   * Scrape a single URL with retry logic
+   * Detect challenge patterns in HTML
+   * @returns Challenge type or null if no challenge detected
    */
-  async scrapeSingleUrlWithRetry(url, index) {
-    const maxRetries = this.options.maxRetries || 2;
-    let lastError;
-    for (let attempt = 0; attempt <= maxRetries; attempt++) {
-      try {
-        const result = await this.scrapeSingleUrl(url, index);
-        if (result) {
-          return { result };
-        }
-        lastError = `Failed to scrape ${url}: No content returned`;
-      } catch (error) {
-        lastError = error.message;
-        if (attempt < maxRetries) {
-          const delay = Math.pow(2, attempt) * 1e3;
-          this.logger.warn(`Retry ${attempt + 1}/${maxRetries} for ${url} in ${delay}ms`);
-          await new Promise((resolve) => setTimeout(resolve, delay));
+  detectChallenge(html) {
+    const htmlLower = html.toLowerCase();
+    const hasCloudflare = CLOUDFLARE_INFRA_PATTERNS.some((p) => htmlLower.includes(p.toLowerCase()));
+    for (const pattern of CHALLENGE_PATTERNS) {
+      if (htmlLower.includes(pattern.toLowerCase())) {
+        if (hasCloudflare || pattern.includes("cf")) {
+          return "cloudflare";
         }
+        return "bot-detection";
       }
     }
-    this.logger.error(`Failed to scrape ${url} after ${maxRetries + 1} attempts: ${lastError}`);
-    return { result: null, error: lastError };
+    return null;
+  }
+  /**
+   * Convert Headers to Record<string, string>
+   */
+  headersToRecord(headers) {
+    const record = {};
+    headers.forEach((value, key) => {
+      record[key] = value;
+    });
+    return record;
+  }
+  /**
+   * Extract visible text from HTML (rough extraction)
+   */
+  extractText(html) {
+    return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  }
+  isAvailable() {
+    return true;
+  }
+};
+var httpEngine = new HttpEngine();
+
+// src/engines/tlsclient/index.ts
+import { gotScraping } from "got-scraping";
+var JS_REQUIRED_PATTERNS = [
+  // Cloudflare JS challenge
+  "cf-browser-verification",
+  "challenge-platform",
+  "_cf_chl_tk",
+  "/cdn-cgi/challenge-platform/",
+  // Generic JS requirements
+  "Enable JavaScript",
+  "JavaScript is required",
+  "Please enable JavaScript",
+  "requires JavaScript",
+  "noscript"
+];
+var BLOCKED_PATTERNS = [
+  "Access denied",
+  "Sorry, you have been blocked",
+  "bot detected",
+  "suspicious activity",
+  "too many requests"
+];
+var MIN_CONTENT_LENGTH2 = 100;
+var TlsClientEngine = class {
+  config = ENGINE_CONFIGS.tlsclient;
+  available = true;
+  constructor() {
+    try {
+      if (!gotScraping) {
+        this.available = false;
+      }
+    } catch {
+      this.available = false;
+    }
+  }
+  async scrape(meta) {
+    if (!this.available) {
+      throw new EngineUnavailableError("tlsclient", "got-scraping not available");
+    }
+    const startTime = Date.now();
+    const { url, options, logger: logger4, abortSignal } = meta;
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), this.config.maxTimeout);
+      if (abortSignal) {
+        abortSignal.addEventListener("abort", () => controller.abort(), { once: true });
+      }
+      logger4?.debug(`[tlsclient] Fetching ${url}`);
+      const response = await gotScraping({
+        url,
+        timeout: {
+          request: this.config.maxTimeout
+        },
+        headers: options.headers,
+        followRedirect: true
+        // got-scraping handles browser fingerprinting automatically
+        // It uses header generators and proper TLS settings
+      });
+      clearTimeout(timeoutId);
+      const duration = Date.now() - startTime;
+      const html = response.body;
+      logger4?.debug(`[tlsclient] Got response: ${response.statusCode} (${html.length} chars) in ${duration}ms`);
+      if (response.statusCode >= 400) {
+        throw new HttpError("tlsclient", response.statusCode, response.statusMessage);
+      }
+      const challengeType = this.detectJsRequired(html);
+      if (challengeType) {
+        logger4?.debug(`[tlsclient] JS required: ${challengeType}`);
+        throw new ChallengeDetectedError("tlsclient", challengeType);
+      }
+      const blockedReason = this.detectBlocked(html);
+      if (blockedReason) {
+        logger4?.debug(`[tlsclient] Blocked: ${blockedReason}`);
+        throw new ChallengeDetectedError("tlsclient", `blocked: ${blockedReason}`);
+      }
+      const textContent = this.extractText(html);
+      if (textContent.length < MIN_CONTENT_LENGTH2) {
+        logger4?.debug(`[tlsclient] Insufficient content: ${textContent.length} chars`);
+        throw new InsufficientContentError("tlsclient", textContent.length, MIN_CONTENT_LENGTH2);
+      }
+      return {
+        html,
+        url: response.url,
+        statusCode: response.statusCode,
+        contentType: response.headers["content-type"],
+        headers: response.headers,
+        engine: "tlsclient",
+        duration
+      };
+    } catch (error) {
+      if (error instanceof ChallengeDetectedError || error instanceof InsufficientContentError || error instanceof HttpError || error instanceof EngineUnavailableError) {
+        throw error;
+      }
+      if (error instanceof Error) {
+        if (error.name === "TimeoutError" || error.message.includes("timeout")) {
+          throw new EngineTimeoutError("tlsclient", this.config.maxTimeout);
+        }
+        if (error.name === "AbortError") {
+          throw new EngineTimeoutError("tlsclient", this.config.maxTimeout);
+        }
+        throw new EngineError("tlsclient", error.message, { cause: error });
+      }
+      throw new EngineError("tlsclient", String(error));
+    }
+  }
+  /**
+   * Detect patterns that require JS execution
+   */
+  detectJsRequired(html) {
+    const htmlLower = html.toLowerCase();
+    for (const pattern of JS_REQUIRED_PATTERNS) {
+      if (htmlLower.includes(pattern.toLowerCase())) {
+        if (pattern.includes("cf") || pattern.includes("cloudflare")) {
+          return "cloudflare-js";
+        }
+        return "js-required";
+      }
+    }
+    return null;
+  }
+  /**
+   * Detect blocked/denied patterns
+   */
+  detectBlocked(html) {
+    const htmlLower = html.toLowerCase();
+    for (const pattern of BLOCKED_PATTERNS) {
+      if (htmlLower.includes(pattern.toLowerCase())) {
+        return pattern;
+      }
+    }
+    return null;
+  }
+  /**
+   * Extract visible text from HTML
+   */
+  extractText(html) {
+    return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  }
+  isAvailable() {
+    return this.available;
+  }
+};
+var tlsClientEngine = new TlsClientEngine();
+
+// src/cloudflare/detector.ts
+var CLOUDFLARE_CHALLENGE_SELECTORS = [
+  "#challenge-running",
+  "#challenge-stage",
+  "#challenge-form",
+  ".cf-browser-verification",
+  "#cf-wrapper",
+  "#cf-hcaptcha-container",
+  "#turnstile-wrapper"
+];
+var CLOUDFLARE_TEXT_PATTERNS = [
+  "checking if the site connection is secure",
+  "this process is automatic. your browser will redirect",
+  "ray id:",
+  "performance & security by cloudflare"
+];
+var CLOUDFLARE_INFRA_PATTERNS2 = [
+  "/cdn-cgi/",
+  "cloudflare",
+  "__cf_bm",
+  "cf-ray"
+];
+var CLOUDFLARE_BLOCKED_PATTERNS = [
+  "sorry, you have been blocked",
+  "ray id:"
+];
+async function detectChallenge(hero) {
+  const signals = [];
+  let type = "none";
+  let hasCloudflareInfra = false;
+  let hasChallengeIndicator = false;
+  try {
+    if (!hero.document) {
+      return {
+        isChallenge: false,
+        type: "none",
+        confidence: 0,
+        signals: ["No document available"]
+      };
+    }
+    const html = await hero.document.documentElement.outerHTML;
+    const htmlLower = html.toLowerCase();
+    for (const pattern of CLOUDFLARE_INFRA_PATTERNS2) {
+      if (htmlLower.includes(pattern)) {
+        hasCloudflareInfra = true;
+        signals.push(`Cloudflare infra: "${pattern}"`);
+        break;
+      }
+    }
+    if (!hasCloudflareInfra) {
+      return {
+        isChallenge: false,
+        type: "none",
+        confidence: 0,
+        signals: ["No Cloudflare infrastructure detected"]
+      };
+    }
+    for (const selector of CLOUDFLARE_CHALLENGE_SELECTORS) {
+      try {
+        const element = await hero.document.querySelector(selector);
+        if (element) {
+          hasChallengeIndicator = true;
+          signals.push(`Challenge element: ${selector}`);
+          type = "js_challenge";
+        }
+      } catch {
+      }
+    }
+    for (const pattern of CLOUDFLARE_TEXT_PATTERNS) {
+      if (htmlLower.includes(pattern)) {
+        hasChallengeIndicator = true;
+        signals.push(`Challenge text: "${pattern}"`);
+        type = type === "none" ? "js_challenge" : type;
+      }
+    }
+    if (htmlLower.includes("waiting for") && htmlLower.includes("to respond")) {
+      hasChallengeIndicator = true;
+      signals.push('Challenge text: "waiting for...to respond"');
+      type = type === "none" ? "js_challenge" : type;
+    }
+    const hasBlocked = CLOUDFLARE_BLOCKED_PATTERNS.every((p) => htmlLower.includes(p));
+    if (hasBlocked) {
+      hasChallengeIndicator = true;
+      signals.push("Cloudflare block page detected");
+      type = "blocked";
+    }
+    const isChallenge = hasCloudflareInfra && hasChallengeIndicator;
+    const confidence = isChallenge ? 100 : 0;
+    return {
+      isChallenge,
+      type: isChallenge ? type : "none",
+      confidence,
+      signals
+    };
+  } catch (error) {
+    return {
+      isChallenge: false,
+      type: "none",
+      confidence: 0,
+      signals: [`Error during detection: ${error.message}`]
+    };
+  }
+}
+
+// src/cloudflare/handler.ts
+async function waitForChallengeResolution(hero, options) {
+  const { maxWaitMs = 45e3, pollIntervalMs = 500, verbose = false, initialUrl } = options;
+  const startTime = Date.now();
+  const log = (msg) => verbose && console.log(`   ${msg}`);
+  while (Date.now() - startTime < maxWaitMs) {
+    const elapsed = Date.now() - startTime;
+    try {
+      const currentUrl = await hero.url;
+      if (currentUrl !== initialUrl) {
+        log(`\u2713 URL changed: ${initialUrl} \u2192 ${currentUrl}`);
+        log(`  Waiting for new page to load...`);
+        try {
+          await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
+          log(`  DOMContentLoaded`);
+        } catch {
+          log(`  DOMContentLoaded timeout, continuing...`);
+        }
+        await hero.waitForPaintingStable().catch(() => {
+        });
+        log(`  Page stabilized`);
+        return { resolved: true, method: "url_redirect", waitedMs: elapsed };
+      }
+    } catch {
+    }
+    const detection = await detectChallenge(hero);
+    if (!detection.isChallenge) {
+      log(`\u2713 Challenge signals cleared (confidence dropped to ${detection.confidence})`);
+      log(`  Waiting for page to load...`);
+      try {
+        await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
+        log(`  DOMContentLoaded`);
+      } catch {
+        log(`  DOMContentLoaded timeout, continuing...`);
+      }
+      await hero.waitForPaintingStable().catch(() => {
+      });
+      log(`  Page stabilized`);
+      return { resolved: true, method: "signals_cleared", waitedMs: elapsed };
+    }
+    log(
+      `\u23F3 ${(elapsed / 1e3).toFixed(1)}s - Still challenge (confidence: ${detection.confidence})`
+    );
+    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+  }
+  return {
+    resolved: false,
+    method: "timeout",
+    waitedMs: Date.now() - startTime
+  };
+}
+
+// src/engines/hero/index.ts
+var MIN_CONTENT_LENGTH3 = 100;
+var HeroEngine = class {
+  config = ENGINE_CONFIGS.hero;
+  async scrape(meta) {
+    const startTime = Date.now();
+    const { url, options, logger: logger4, abortSignal } = meta;
+    const pool = options.pool;
+    if (!pool) {
+      throw new EngineUnavailableError("hero", "Browser pool not available");
+    }
+    if (abortSignal?.aborted) {
+      throw new EngineTimeoutError("hero", 0);
+    }
+    logger4?.debug(`[hero] Starting browser scrape of ${url}`);
+    try {
+      const result = await pool.withBrowser(async (hero) => {
+        let aborted = false;
+        if (abortSignal) {
+          abortSignal.addEventListener("abort", () => {
+            aborted = true;
+          }, { once: true });
+        }
+        const timeoutMs = options.timeoutMs || this.config.maxTimeout;
+        await hero.goto(url, { timeoutMs });
+        if (aborted) {
+          throw new EngineTimeoutError("hero", Date.now() - startTime);
+        }
+        try {
+          await hero.waitForLoad("DomContentLoaded", { timeoutMs });
+        } catch {
+        }
+        await hero.waitForPaintingStable();
+        if (aborted) {
+          throw new EngineTimeoutError("hero", Date.now() - startTime);
+        }
+        const initialUrl = await hero.url;
+        const detection = await detectChallenge(hero);
+        if (detection.isChallenge) {
+          logger4?.debug(`[hero] Challenge detected: ${detection.type}`);
+          if (detection.type === "blocked") {
+            throw new ChallengeDetectedError("hero", "blocked");
+          }
+          const resolution = await waitForChallengeResolution(hero, {
+            maxWaitMs: 45e3,
+            pollIntervalMs: 500,
+            verbose: options.verbose,
+            initialUrl
+          });
+          if (!resolution.resolved) {
+            throw new ChallengeDetectedError("hero", `unresolved: ${detection.type}`);
+          }
+          logger4?.debug(`[hero] Challenge resolved via ${resolution.method} in ${resolution.waitedMs}ms`);
+        }
+        if (aborted) {
+          throw new EngineTimeoutError("hero", Date.now() - startTime);
+        }
+        await this.waitForFinalPage(hero, url, logger4);
+        if (aborted) {
+          throw new EngineTimeoutError("hero", Date.now() - startTime);
+        }
+        if (options.waitForSelector) {
+          try {
+            await hero.waitForElement(hero.document.querySelector(options.waitForSelector), {
+              timeoutMs
+            });
+          } catch {
+            logger4?.debug(`[hero] Selector not found: ${options.waitForSelector}`);
+          }
+        }
+        const html = await hero.document.documentElement.outerHTML;
+        const finalUrl = await hero.url;
+        const textContent = this.extractText(html);
+        if (textContent.length < MIN_CONTENT_LENGTH3) {
+          logger4?.debug(`[hero] Insufficient content: ${textContent.length} chars`);
+          throw new InsufficientContentError("hero", textContent.length, MIN_CONTENT_LENGTH3);
+        }
+        const duration = Date.now() - startTime;
+        logger4?.debug(`[hero] Success: ${html.length} chars in ${duration}ms`);
+        return {
+          html,
+          url: finalUrl,
+          statusCode: 200,
+          // Hero doesn't expose status code directly
+          engine: "hero",
+          duration
+        };
+      });
+      return result;
+    } catch (error) {
+      if (error instanceof ChallengeDetectedError || error instanceof InsufficientContentError || error instanceof EngineTimeoutError || error instanceof EngineUnavailableError) {
+        throw error;
+      }
+      if (error instanceof Error) {
+        if (error.name === "TimeoutError" || error.message.includes("timeout")) {
+          throw new EngineTimeoutError("hero", this.config.maxTimeout);
+        }
+        if (error.message.includes("Navigation") || error.message.includes("ERR_")) {
+          throw new EngineError("hero", `Navigation failed: ${error.message}`, { cause: error });
+        }
+        throw new EngineError("hero", error.message, { cause: error });
+      }
+      throw new EngineError("hero", String(error));
+    }
   }
   /**
    * Wait for the final page to load after any Cloudflare redirects
-   * Cloudflare often does silent redirects even when bypassed, we need to ensure
-   * we're on the actual content page before scraping.
    */
-  async waitForFinalPage(hero, originalUrl, verbose) {
+  async waitForFinalPage(hero, originalUrl, logger4) {
     const maxWaitMs = 15e3;
     const startTime = Date.now();
-    const log = (msg) => verbose && this.logger.info(msg);
     try {
       await hero.waitForLoad("AllContentLoaded", { timeoutMs: maxWaitMs });
     } catch {
@@ -1152,7 +1567,7 @@ var Scraper = class {
     const normalizeUrl2 = (url) => url.replace(/\/+$/, "");
     const urlChanged = normalizeUrl2(currentUrl) !== normalizeUrl2(originalUrl);
     if (urlChanged || currentUrl.includes("__cf_chl")) {
-      log(`Cloudflare redirect detected: ${originalUrl} \u2192 ${currentUrl}`);
+      logger4?.debug(`[hero] Cloudflare redirect detected: ${originalUrl} \u2192 ${currentUrl}`);
       let lastUrl = currentUrl;
       let stableCount = 0;
       while (Date.now() - startTime < maxWaitMs) {
@@ -1167,7 +1582,7 @@ var Scraper = class {
           } else {
             stableCount = 0;
             lastUrl = currentUrl;
-            log(`URL changed to: ${currentUrl}`);
+            logger4?.debug(`[hero] URL changed to: ${currentUrl}`);
           }
         } catch {
         }
@@ -1181,7 +1596,223 @@ var Scraper = class {
     await new Promise((resolve) => setTimeout(resolve, 2e3));
   }
   /**
-   * Scrape a single URL
+   * Extract visible text from HTML
+   */
+  extractText(html) {
+    return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  }
+  isAvailable() {
+    return true;
+  }
+};
+var heroEngine = new HeroEngine();
+
+// src/engines/orchestrator.ts
+var ENGINE_REGISTRY = {
+  http: httpEngine,
+  tlsclient: tlsClientEngine,
+  hero: heroEngine
+};
+var EngineOrchestrator = class {
+  options;
+  engines;
+  engineOrder;
+  constructor(options = {}) {
+    this.options = options;
+    this.engineOrder = this.resolveEngineOrder();
+    this.engines = this.engineOrder.map((name) => ENGINE_REGISTRY[name]).filter((engine) => engine.isAvailable());
+  }
+  /**
+   * Resolve the engine order based on options
+   */
+  resolveEngineOrder() {
+    if (this.options.forceEngine) {
+      return [this.options.forceEngine];
+    }
+    let order = this.options.engines || [...DEFAULT_ENGINE_ORDER];
+    if (this.options.skipEngines) {
+      order = order.filter((e) => !this.options.skipEngines.includes(e));
+    }
+    return order;
+  }
+  /**
+   * Get available engines
+   */
+  getAvailableEngines() {
+    return this.engines.map((e) => e.config.name);
+  }
+  /**
+   * Scrape a URL using the engine cascade
+   *
+   * @param meta - Engine metadata (url, options, logger, abortSignal)
+   * @returns Scrape result with engine metadata
+   * @throws AllEnginesFailedError if all engines fail
+   */
+  async scrape(meta) {
+    const attemptedEngines = [];
+    const engineErrors = /* @__PURE__ */ new Map();
+    const logger4 = meta.logger || this.options.logger;
+    const verbose = this.options.verbose || meta.options.verbose;
+    if (this.engines.length === 0) {
+      throw new AllEnginesFailedError([], engineErrors);
+    }
+    const log = (msg) => {
+      if (verbose) {
+        logger4?.info(msg);
+      } else {
+        logger4?.debug(msg);
+      }
+    };
+    log(`[orchestrator] Starting scrape of ${meta.url} with engines: ${this.engineOrder.join(" \u2192 ")}`);
+    for (const engine of this.engines) {
+      const engineName = engine.config.name;
+      attemptedEngines.push(engineName);
+      try {
+        log(`[orchestrator] Trying ${engineName} engine...`);
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), engine.config.maxTimeout);
+        if (meta.abortSignal) {
+          meta.abortSignal.addEventListener("abort", () => controller.abort(), { once: true });
+        }
+        try {
+          const result = await engine.scrape({
+            ...meta,
+            abortSignal: controller.signal
+          });
+          clearTimeout(timeoutId);
+          log(`[orchestrator] \u2713 ${engineName} succeeded in ${result.duration}ms`);
+          return {
+            ...result,
+            attemptedEngines,
+            engineErrors
+          };
+        } finally {
+          clearTimeout(timeoutId);
+        }
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        engineErrors.set(engineName, err);
+        if (error instanceof ChallengeDetectedError) {
+          log(`[orchestrator] ${engineName} detected challenge: ${error.challengeType}`);
+        } else if (error instanceof InsufficientContentError) {
+          log(`[orchestrator] ${engineName} insufficient content: ${error.contentLength} chars`);
+        } else if (error instanceof HttpError) {
+          log(`[orchestrator] ${engineName} HTTP error: ${error.statusCode}`);
+        } else if (error instanceof EngineTimeoutError) {
+          log(`[orchestrator] ${engineName} timed out after ${error.timeoutMs}ms`);
+        } else if (error instanceof EngineUnavailableError) {
+          log(`[orchestrator] ${engineName} unavailable: ${err.message}`);
+        } else {
+          log(`[orchestrator] ${engineName} failed: ${err.message}`);
+        }
+        if (!this.shouldRetry(error)) {
+          log(`[orchestrator] Non-retryable error, stopping cascade`);
+          break;
+        }
+        log(`[orchestrator] Falling back to next engine...`);
+      }
+    }
+    log(`[orchestrator] All engines failed for ${meta.url}`);
+    throw new AllEnginesFailedError(attemptedEngines, engineErrors);
+  }
+  /**
+   * Determine if we should retry with next engine
+   */
+  shouldRetry(error) {
+    if (error instanceof ChallengeDetectedError || error instanceof InsufficientContentError || error instanceof EngineTimeoutError) {
+      return true;
+    }
+    if (error instanceof HttpError) {
+      return error.statusCode === 403 || error.statusCode === 429 || error.statusCode >= 500;
+    }
+    if (error instanceof EngineUnavailableError) {
+      return true;
+    }
+    if (error instanceof EngineError) {
+      return error.retryable;
+    }
+    return true;
+  }
+};
+
+// src/scraper.ts
+var Scraper = class {
+  options;
+  logger = createLogger("scraper");
+  robotsCache = /* @__PURE__ */ new Map();
+  constructor(options) {
+    this.options = {
+      ...DEFAULT_OPTIONS,
+      ...options
+    };
+  }
+  /**
+   * Get robots.txt rules for a URL, cached per domain
+   */
+  async getRobotsRules(url) {
+    const origin = new URL(url).origin;
+    if (!this.robotsCache.has(origin)) {
+      const rules = await fetchRobotsTxt(origin);
+      this.robotsCache.set(origin, rules);
+    }
+    return this.robotsCache.get(origin) ?? null;
+  }
+  /**
+   * Scrape all URLs
+   *
+   * @returns Scrape result with pages and metadata
+   */
+  async scrape() {
+    const startTime = Date.now();
+    const results = await this.scrapeWithConcurrency();
+    return this.buildScrapeResult(results, startTime);
+  }
+  /**
+   * Scrape URLs with concurrency control
+   */
+  async scrapeWithConcurrency() {
+    const limit = pLimit(this.options.batchConcurrency || 1);
+    const tasks = this.options.urls.map(
+      (url, index) => limit(() => this.scrapeSingleUrlWithRetry(url, index))
+    );
+    const batchPromise = Promise.all(tasks);
+    if (this.options.batchTimeoutMs && this.options.batchTimeoutMs > 0) {
+      const timeoutPromise = new Promise((_, reject) => {
+        setTimeout(() => {
+          reject(new Error(`Batch operation timed out after ${this.options.batchTimeoutMs}ms`));
+        }, this.options.batchTimeoutMs);
+      });
+      return Promise.race([batchPromise, timeoutPromise]);
+    }
+    return batchPromise;
+  }
+  /**
+   * Scrape a single URL with retry logic
+   */
+  async scrapeSingleUrlWithRetry(url, index) {
+    const maxRetries = this.options.maxRetries || 2;
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        const result = await this.scrapeSingleUrl(url, index);
+        if (result) {
+          return { result };
+        }
+        lastError = `Failed to scrape ${url}: No content returned`;
+      } catch (error) {
+        lastError = error.message;
+        if (attempt < maxRetries) {
+          const delay = Math.pow(2, attempt) * 1e3;
+          this.logger.warn(`Retry ${attempt + 1}/${maxRetries} for ${url} in ${delay}ms`);
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+      }
+    }
+    this.logger.error(`Failed to scrape ${url} after ${maxRetries + 1} attempts: ${lastError}`);
+    return { result: null, error: lastError };
+  }
+  /**
+   * Scrape a single URL using the engine orchestrator
    */
   async scrapeSingleUrl(url, index) {
     const startTime = Date.now();
@@ -1190,98 +1821,84 @@ var Scraper = class {
       throw new Error(`URL blocked by robots.txt: ${url}`);
     }
     try {
-      return await this.pool.withBrowser(async (hero) => {
-        await hero.goto(url, { timeoutMs: this.options.timeoutMs });
-        try {
-          await hero.waitForLoad("DomContentLoaded", { timeoutMs: this.options.timeoutMs });
-        } catch {
-        }
-        await hero.waitForPaintingStable();
-        const initialUrl = await hero.url;
-        const detection = await detectChallenge(hero);
-        if (detection.isChallenge) {
-          if (this.options.verbose) {
-            this.logger.info(`Challenge detected on ${url}: ${detection.type}`);
-          }
-          const result2 = await waitForChallengeResolution(hero, {
-            maxWaitMs: 45e3,
-            pollIntervalMs: 500,
-            verbose: this.options.verbose,
-            initialUrl
-          });
-          if (!result2.resolved) {
-            throw new Error(`Challenge not resolved: ${detection.type}`);
-          }
-          if (this.options.verbose) {
-            this.logger.info(`Challenge resolved via ${result2.method} in ${result2.waitedMs}ms`);
-          }
-        }
-        await this.waitForFinalPage(hero, url, this.options.verbose);
-        if (this.options.waitForSelector) {
-          try {
-            await hero.waitForElement(hero.document.querySelector(this.options.waitForSelector), {
-              timeoutMs: this.options.timeoutMs
-            });
-          } catch (error) {
-            this.logger.warn(`Selector not found: ${this.options.waitForSelector}`);
-          }
-        }
-        const html = await hero.document.documentElement.outerHTML;
-        const cleanedHtml = cleanContent(html, url, {
-          removeAds: this.options.removeAds,
-          removeBase64Images: this.options.removeBase64Images,
-          onlyMainContent: this.options.onlyMainContent,
-          includeTags: this.options.includeTags,
-          excludeTags: this.options.excludeTags
+      const orchestrator = new EngineOrchestrator({
+        engines: this.options.engines,
+        skipEngines: this.options.skipEngines,
+        forceEngine: this.options.forceEngine,
+        logger: this.logger,
+        verbose: this.options.verbose
+      });
+      const engineResult = await orchestrator.scrape({
+        url,
+        options: this.options,
+        logger: this.logger
+      });
+      if (this.options.verbose) {
+        this.logger.info(
+          `[scraper] ${url} scraped with ${engineResult.engine} engine in ${engineResult.duration}ms (attempted: ${engineResult.attemptedEngines.join(" \u2192 ")})`
+        );
+      }
+      const cleanedHtml = cleanContent(engineResult.html, engineResult.url, {
+        removeAds: this.options.removeAds,
+        removeBase64Images: this.options.removeBase64Images,
+        onlyMainContent: this.options.onlyMainContent,
+        includeTags: this.options.includeTags,
+        excludeTags: this.options.excludeTags
+      });
+      const websiteMetadata = extractMetadata(cleanedHtml, engineResult.url);
+      const duration = Date.now() - startTime;
+      const markdown = this.options.formats.includes("markdown") ? htmlToMarkdown(cleanedHtml) : void 0;
+      const htmlOutput = this.options.formats.includes("html") ? cleanedHtml : void 0;
+      if (this.options.onProgress) {
+        this.options.onProgress({
+          completed: index + 1,
+          total: this.options.urls.length,
+          currentUrl: url
         });
-        const websiteMetadata = extractMetadata(cleanedHtml, url);
-        const duration = Date.now() - startTime;
-        const markdown = this.options.formats.includes("markdown") ? htmlToMarkdown(cleanedHtml) : void 0;
-        const htmlOutput = this.options.formats.includes("html") ? cleanedHtml : void 0;
-        if (this.options.onProgress) {
-          this.options.onProgress({
-            completed: index + 1,
-            total: this.options.urls.length,
-            currentUrl: url
-          });
-        }
-        let proxyMetadata;
-        if (this.options.proxy) {
-          const proxy = this.options.proxy;
-          if (proxy.url) {
-            try {
-              const proxyUrl = new URL(proxy.url);
-              proxyMetadata = {
-                host: proxyUrl.hostname,
-                port: parseInt(proxyUrl.port, 10) || 80,
-                country: proxy.country
-              };
-            } catch {
-            }
-          } else if (proxy.host && proxy.port) {
+      }
+      let proxyMetadata;
+      if (this.options.proxy) {
+        const proxy = this.options.proxy;
+        if (proxy.url) {
+          try {
+            const proxyUrl = new URL(proxy.url);
             proxyMetadata = {
-              host: proxy.host,
-              port: proxy.port,
+              host: proxyUrl.hostname,
+              port: parseInt(proxyUrl.port, 10) || 80,
               country: proxy.country
             };
+          } catch {
           }
+        } else if (proxy.host && proxy.port) {
+          proxyMetadata = {
+            host: proxy.host,
+            port: proxy.port,
+            country: proxy.country
+          };
         }
-        const result = {
-          markdown,
-          html: htmlOutput,
-          metadata: {
-            baseUrl: url,
-            totalPages: 1,
-            scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
-            duration,
-            website: websiteMetadata,
-            proxy: proxyMetadata
-          }
-        };
-        return result;
-      });
+      }
+      const result = {
+        markdown,
+        html: htmlOutput,
+        metadata: {
+          baseUrl: url,
+          totalPages: 1,
+          scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          duration,
+          website: websiteMetadata,
+          proxy: proxyMetadata
+        }
+      };
+      return result;
     } catch (error) {
-      this.logger.error(`Failed to scrape ${url}: ${error.message}`);
+      if (error instanceof AllEnginesFailedError) {
+        const engineSummary = error.attemptedEngines.map((e) => `${e}: ${error.errors.get(e)?.message || "unknown"}`).join("; ");
+        this.logger.error(`Failed to scrape ${url}: All engines failed - ${engineSummary}`);
+      } else if (error instanceof Error) {
+        this.logger.error(`Failed to scrape ${url}: ${error.message}`);
+      } else {
+        this.logger.error(`Failed to scrape ${url}: ${String(error)}`);
+      }
       if (this.options.onProgress) {
         this.options.onProgress({
           completed: index + 1,
@@ -2604,7 +3221,7 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
   "-f, --format <formats>",
   "Content formats to include (comma-separated: markdown,html)",
   "markdown"
-).option("-o, --output <file>", "Output file (stdout if omitted)").option("-c, --concurrency <n>", "Parallel requests", "1").option("-t, --timeout <ms>", "Request timeout in milliseconds", "30000").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--batch-timeout <ms>", "Total timeout for entire batch operation", "300000").option("--show-chrome", "Show browser window for debugging").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").option("--no-main-content", "Disable main content extraction (include full page)").option("--include-tags <selectors>", "CSS selectors for elements to include (comma-separated)").option("--exclude-tags <selectors>", "CSS selectors for elements to exclude (comma-separated)").action(async (urls, options) => {
+).option("-o, --output <file>", "Output file (stdout if omitted)").option("-c, --concurrency <n>", "Parallel requests", "1").option("-t, --timeout <ms>", "Request timeout in milliseconds", "30000").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--batch-timeout <ms>", "Total timeout for entire batch operation", "300000").option("--show-chrome", "Show browser window for debugging").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").option("--no-main-content", "Disable main content extraction (include full page)").option("--include-tags <selectors>", "CSS selectors for elements to include (comma-separated)").option("--exclude-tags <selectors>", "CSS selectors for elements to exclude (comma-separated)").option("--engine <name>", "Force a specific engine (http, tlsclient, hero)").option("--skip-engine <names>", "Skip specific engines (comma-separated: http,tlsclient,hero)").action(async (urls, options) => {
   const port = parseInt(options.port, 10);
   const useStandalone = options.standalone || false;
   let useDaemon = false;
@@ -2636,6 +3253,7 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
     }
     const includeTags = options.includeTags ? options.includeTags.split(",").map((s) => s.trim()) : void 0;
     const excludeTags = options.excludeTags ? options.excludeTags.split(",").map((s) => s.trim()) : void 0;
+    const skipEngines = options.skipEngine ? options.skipEngine.split(",").map((s) => s.trim()) : void 0;
     const scrapeOptions = {
       urls,
       formats,
@@ -2651,6 +3269,9 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
       // --no-main-content sets this to false
       includeTags,
       excludeTags,
+      // Engine options
+      forceEngine: options.engine,
+      skipEngines,
       onProgress: options.verbose ? ({ completed, total, currentUrl }) => {
         console.error(`[${completed}/${total}] ${currentUrl}`);
       } : void 0