crucible 0.1.2

data/lib/crucible/stealth/evasions/navigator_vendor.js

@@ -0,0 +1,18 @@
+/**
+ * Evasion: navigator.vendor
+ * Override navigator.vendor to return the correct value.
+ */
+(function(opts) {
+  'use strict';
+
+  const utils = window._stealthUtils;
+  if (!utils) return;
+
+  const vendor = opts.vendor || 'Google Inc.';
+
+  utils.replaceGetterWithProxy(
+    Object.getPrototypeOf(navigator),
+    'vendor',
+    utils.makeHandler().getterValue(vendor)
+  );
+})({ vendor: null }); // Will be replaced by Ruby

data/lib/crucible/stealth/evasions/navigator_webdriver.js

@@ -0,0 +1,16 @@
+/**
+ * Evasion: navigator.webdriver
+ * Deletes the navigator.webdriver property that reveals automation.
+ */
+(function() {
+  'use strict';
+
+  if (navigator.webdriver === false) {
+    // Post Chrome 89.0.4339.0 and already good
+  } else if (navigator.webdriver === undefined) {
+    // Pre Chrome 89.0.4339.0 and already good
+  } else {
+    // Needs patching
+    delete Object.getPrototypeOf(navigator).webdriver;
+  }
+})();

data/lib/crucible/stealth/evasions/webgl_vendor.js

@@ -0,0 +1,43 @@
+/**
+ * Evasion: webgl.vendor
+ * Fix WebGL Vendor/Renderer being set to Google in headless mode.
+ * Default values match a typical Apple Retina MBP.
+ */
+(function(opts) {
+  'use strict';
+
+  const utils = window._stealthUtils;
+  if (!utils) return;
+
+  const vendor = opts.vendor || 'Intel Inc.';
+  const renderer = opts.renderer || 'Intel Iris OpenGL Engine';
+
+  const getParameterProxyHandler = {
+    apply: function(target, ctx, args) {
+      const param = (args || [])[0];
+      const result = utils.cache.Reflect.apply(target, ctx, args);
+      // UNMASKED_VENDOR_WEBGL
+      if (param === 37445) {
+        return vendor;
+      }
+      // UNMASKED_RENDERER_WEBGL
+      if (param === 37446) {
+        return renderer;
+      }
+      return result;
+    }
+  };
+
+  // There's more than one WebGL rendering context
+  const addProxy = (obj, propName) => {
+    utils.replaceWithProxy(obj, propName, getParameterProxyHandler);
+  };
+
+  // Patch both WebGL contexts
+  if (typeof WebGLRenderingContext !== 'undefined') {
+    addProxy(WebGLRenderingContext.prototype, 'getParameter');
+  }
+  if (typeof WebGL2RenderingContext !== 'undefined') {
+    addProxy(WebGL2RenderingContext.prototype, 'getParameter');
+  }
+})({ vendor: null, renderer: null }); // Will be replaced by Ruby

data/lib/crucible/stealth/evasions/window_outerdimensions.js

@@ -0,0 +1,18 @@
+/**
+ * Evasion: window.outerdimensions
+ * Fix missing window.outerWidth/window.outerHeight in headless mode.
+ */
+(function() {
+  'use strict';
+
+  try {
+    if (window.outerWidth && window.outerHeight) {
+      return; // Nothing to do here
+    }
+    const windowFrame = 85; // Approximate window frame size (OS/WM dependent)
+    window.outerWidth = window.innerWidth;
+    window.outerHeight = window.innerHeight + windowFrame;
+  } catch (err) {
+    // Silently fail
+  }
+})();

data/lib/crucible/stealth/utils.js

@@ -0,0 +1,266 @@
+/**
+ * Stealth utilities for modifying native browser APIs without leaving traces.
+ * Ported from puppeteer-extra-plugin-stealth.
+ */
+(function() {
+  'use strict';
+
+  if (window._stealthUtils) return;
+
+  const utils = {};
+
+  utils.init = () => {
+    utils.preloadCache();
+  };
+
+  /**
+   * Preload a cache of function copies and data.
+   */
+  utils.preloadCache = () => {
+    if (utils.cache) return;
+    utils.cache = {
+      Reflect: {
+        get: Reflect.get.bind(Reflect),
+        apply: Reflect.apply.bind(Reflect)
+      },
+      nativeToStringStr: Function.toString + ''
+    };
+  };
+
+  /**
+   * Generate a native toString result.
+   */
+  utils.makeNativeString = (name = '') => {
+    return utils.cache.nativeToStringStr.replace('toString', name || '');
+  };
+
+  /**
+   * Wrap a JS Proxy Handler and strip its presence from error stacks.
+   */
+  utils.stripProxyFromErrors = (handler = {}) => {
+    const newHandler = {
+      setPrototypeOf: function(target, proto) {
+        if (proto === null)
+          throw new TypeError('Cannot convert object to primitive value');
+        if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) {
+          throw new TypeError('Cyclic __proto__ value');
+        }
+        return Reflect.setPrototypeOf(target, proto);
+      }
+    };
+
+    const traps = Object.getOwnPropertyNames(handler);
+    traps.forEach(trap => {
+      newHandler[trap] = function() {
+        try {
+          return handler[trap].apply(this, arguments || []);
+        } catch (err) {
+          if (!err || !err.stack || !err.stack.includes('at ')) {
+            throw err;
+          }
+
+          const stripWithBlacklist = (stack, stripFirstLine = true) => {
+            const blacklist = [
+              `at Reflect.${trap} `,
+              `at Object.${trap} `,
+              `at Object.newHandler.<computed> [as ${trap}] `
+            ];
+            return err.stack
+              .split('\n')
+              .filter((line, index) => !(index === 1 && stripFirstLine))
+              .filter(line => !blacklist.some(bl => line.trim().startsWith(bl)))
+              .join('\n');
+          };
+
+          const stripWithAnchor = (stack, anchor) => {
+            const stackArr = stack.split('\n');
+            anchor = anchor || `at Object.newHandler.<computed> [as ${trap}] `;
+            const anchorIndex = stackArr.findIndex(line =>
+              line.trim().startsWith(anchor)
+            );
+            if (anchorIndex === -1) return false;
+            stackArr.splice(1, anchorIndex);
+            return stackArr.join('\n');
+          };
+
+          err.stack = err.stack.replace(
+            'at Object.toString (',
+            'at Function.toString ('
+          );
+          if ((err.stack || '').includes('at Function.toString (')) {
+            err.stack = stripWithBlacklist(err.stack, false);
+            throw err;
+          }
+
+          err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack);
+          throw err;
+        }
+      };
+    });
+    return newHandler;
+  };
+
+  /**
+   * Replace the property of an object.
+   */
+  utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => {
+    return Object.defineProperty(obj, propName, {
+      ...(Object.getOwnPropertyDescriptor(obj, propName) || {}),
+      ...descriptorOverrides
+    });
+  };
+
+  /**
+   * Patch toString of an object.
+   */
+  utils.patchToString = (obj, str = '') => {
+    const handler = {
+      apply: function(target, ctx) {
+        if (ctx === Function.prototype.toString) {
+          return utils.makeNativeString('toString');
+        }
+        if (ctx === obj) {
+          return str || utils.makeNativeString(obj.name);
+        }
+        const hasSameProto = Object.getPrototypeOf(
+          Function.prototype.toString
+        ).isPrototypeOf(ctx.toString);
+        if (!hasSameProto) {
+          return ctx.toString();
+        }
+        return target.call(ctx);
+      }
+    };
+
+    const toStringProxy = new Proxy(
+      Function.prototype.toString,
+      utils.stripProxyFromErrors(handler)
+    );
+    utils.replaceProperty(Function.prototype, 'toString', {
+      value: toStringProxy
+    });
+  };
+
+  /**
+   * Redirect toString from proxyObj to originalObj.
+   */
+  utils.redirectToString = (proxyObj, originalObj) => {
+    const handler = {
+      apply: function(target, ctx) {
+        if (ctx === Function.prototype.toString) {
+          return utils.makeNativeString('toString');
+        }
+        if (ctx === proxyObj) {
+          const fallback = () =>
+            originalObj && originalObj.name
+              ? utils.makeNativeString(originalObj.name)
+              : utils.makeNativeString(proxyObj.name);
+          return originalObj + '' || fallback();
+        }
+        if (typeof ctx === 'undefined' || ctx === null) {
+          return target.call(ctx);
+        }
+        const hasSameProto = Object.getPrototypeOf(
+          Function.prototype.toString
+        ).isPrototypeOf(ctx.toString);
+        if (!hasSameProto) {
+          return ctx.toString();
+        }
+        return target.call(ctx);
+      }
+    };
+
+    const toStringProxy = new Proxy(
+      Function.prototype.toString,
+      utils.stripProxyFromErrors(handler)
+    );
+    utils.replaceProperty(Function.prototype, 'toString', {
+      value: toStringProxy
+    });
+  };
+
+  /**
+   * Replace a property with a JS Proxy.
+   */
+  utils.replaceWithProxy = (obj, propName, handler) => {
+    const originalObj = obj[propName];
+    const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler));
+    utils.replaceProperty(obj, propName, { value: proxyObj });
+    utils.redirectToString(proxyObj, originalObj);
+    return true;
+  };
+
+  /**
+   * Replace a getter with a JS Proxy.
+   */
+  utils.replaceGetterWithProxy = (obj, propName, handler) => {
+    const fn = Object.getOwnPropertyDescriptor(obj, propName).get;
+    const fnStr = fn.toString();
+    const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler));
+    utils.replaceProperty(obj, propName, { get: proxyObj });
+    utils.patchToString(proxyObj, fnStr);
+    return true;
+  };
+
+  /**
+   * Mock a non-existing property with a JS Proxy.
+   */
+  utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => {
+    const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler));
+    utils.replaceProperty(obj, propName, { value: proxyObj });
+    utils.patchToString(proxyObj);
+    return true;
+  };
+
+  /**
+   * Create a new JS Proxy with stealth tweaks.
+   */
+  utils.createProxy = (pseudoTarget, handler) => {
+    const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler));
+    utils.patchToString(proxyObj);
+    return proxyObj;
+  };
+
+  /**
+   * Make all nested functions of an object native.
+   */
+  utils.patchToStringNested = (obj = {}) => {
+    return utils.execRecursively(obj, ['function'], utils.patchToString);
+  };
+
+  /**
+   * Traverse nested properties recursively.
+   */
+  utils.execRecursively = (obj = {}, typeFilter = [], fn) => {
+    function recurse(obj) {
+      for (const key in obj) {
+        if (obj[key] === undefined) continue;
+        if (obj[key] && typeof obj[key] === 'object') {
+          recurse(obj[key]);
+        } else {
+          if (obj[key] && typeFilter.includes(typeof obj[key])) {
+            fn.call(this, obj[key]);
+          }
+        }
+      }
+    }
+    recurse(obj);
+    return obj;
+  };
+
+  /**
+   * Handler templates for re-usability.
+   */
+  utils.makeHandler = () => ({
+    getterValue: value => ({
+      apply(target, ctx, args) {
+        utils.cache.Reflect.apply(...arguments);
+        return value;
+      }
+    })
+  });
+
+  // Initialize and expose
+  utils.init();
+  window._stealthUtils = utils;
+})();

data/lib/crucible/stealth.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+module Crucible
+  # Stealth mode for browser automation
+  #
+  # Applies various evasion techniques to make headless Chrome
+  # appear as a regular browser to bot detection systems.
+  #
+  # @example
+  #   stealth = Stealth.new(profile: :maximum)
+  #   stealth.apply(browser)
+  #
+  class Stealth
+    EVASIONS_PATH = File.expand_path('stealth/evasions', __dir__)
+    UTILS_PATH = File.expand_path('stealth/utils.js', __dir__)
+
+    # Stealth profiles define which evasions are enabled
+    PROFILES = {
+      # Minimal - only essential evasions
+      minimal: %i[
+        navigator_webdriver
+        window_outerdimensions
+      ],
+
+      # Moderate - common evasions for most use cases
+      moderate: %i[
+        navigator_webdriver
+        chrome_app
+        chrome_csi
+        chrome_load_times
+        chrome_runtime
+        navigator_vendor
+        navigator_languages
+        window_outerdimensions
+      ],
+
+      # Maximum - all evasions for strictest detection
+      maximum: %i[
+        navigator_webdriver
+        chrome_app
+        chrome_csi
+        chrome_load_times
+        chrome_runtime
+        navigator_vendor
+        navigator_languages
+        navigator_plugins
+        navigator_permissions
+        navigator_hardware_concurrency
+        webgl_vendor
+        media_codecs
+        iframe_content_window
+        window_outerdimensions
+      ]
+    }.freeze
+
+    # Evasion files with their configurable options
+    EVASION_OPTIONS = {
+      navigator_vendor: { vendor: 'Google Inc.' },
+      navigator_languages: { languages: %w[en-US en] },
+      navigator_hardware_concurrency: { hardwareConcurrency: 4 },
+      webgl_vendor: { vendor: 'Intel Inc.', renderer: 'Intel Iris OpenGL Engine' }
+    }.freeze
+
+    attr_reader :profile, :enabled, :options
+
+    # @param profile [Symbol] stealth profile (:minimal, :moderate, :maximum)
+    # @param enabled [Boolean] whether stealth is enabled
+    # @param options [Hash] additional options (locale, custom evasion opts)
+    def initialize(profile: :moderate, enabled: true, **options)
+      @profile = validate_profile(profile)
+      @enabled = enabled
+      @options = {
+        locale: 'en-US,en',
+        inject_utils: true
+      }.merge(options)
+    end
+
+    # Apply stealth evasions to a browser
+    # @param browser [Ferrum::Browser] the browser instance
+    def apply(browser)
+      return unless enabled
+
+      inject_utils(browser) if options[:inject_utils]
+      inject_evasions(browser)
+      apply_user_agent_override(browser)
+    end
+
+    # Apply stealth evasions to a page (for new pages in existing session)
+    # @param page [Ferrum::Page] the page instance
+    def apply_to_page(page)
+      return unless enabled
+
+      inject_utils_to_page(page) if options[:inject_utils]
+      inject_evasions_to_page(page)
+    end
+
+    # Get list of enabled evasions for current profile
+    # @return [Array<Symbol>]
+    def enabled_evasions
+      PROFILES.fetch(profile, PROFILES[:moderate])
+    end
+
+    # Creates extensions array suitable for Ferrum browser options
+    # @return [Array<String>]
+    def extensions
+      return [] unless enabled
+
+      scripts = []
+      scripts << utils_script if options[:inject_utils]
+      scripts.concat(evasion_scripts)
+      scripts
+    end
+
+    private
+
+    def validate_profile(profile)
+      profile = profile.to_sym
+      unless PROFILES.key?(profile)
+        raise Error, "Invalid stealth profile: #{profile}. Must be one of: #{PROFILES.keys.join(', ')}"
+      end
+
+      profile
+    end
+
+    def inject_utils(browser)
+      # Use evaluate_on_new_document to inject before page loads
+      browser.evaluate_on_new_document(utils_script)
+    rescue Ferrum::Error
+      # Browser may not be ready, that's OK
+    end
+
+    def inject_utils_to_page(page)
+      # For existing pages, use command directly
+      page.command('Page.addScriptToEvaluateOnNewDocument', source: utils_script)
+    rescue Ferrum::Error
+      # Page may not be ready, that's OK
+    end
+
+    def inject_evasions(browser)
+      evasion_scripts.each do |script|
+        browser.evaluate_on_new_document(script)
+      rescue Ferrum::Error
+        # Continue with other evasions
+      end
+    end
+
+    def inject_evasions_to_page(page)
+      evasion_scripts.each do |script|
+        page.command('Page.addScriptToEvaluateOnNewDocument', source: script)
+      rescue Ferrum::Error
+        # Continue with other evasions
+      end
+    end
+
+    def apply_user_agent_override(browser)
+      page = browser.page
+      return unless page
+
+      # Get current UA and strip "Headless"
+      ua = page.evaluate('navigator.userAgent')
+      ua = ua.gsub('HeadlessChrome/', 'Chrome/')
+
+      # Mask Linux as Windows (common detection vector)
+      ua = ua.gsub(/\(([^)]+)\)/, '(Windows NT 10.0; Win64; x64)') if ua.include?('Linux') && !ua.include?('Android')
+
+      # Apply via Ferrum's page command (uses Network.setUserAgentOverride internally)
+      page.command(
+        'Network.setUserAgentOverride',
+        userAgent: ua,
+        acceptLanguage: options[:locale]
+      )
+    rescue Ferrum::Error => e
+      # Log but don't fail - UA override is optional
+      warn "[Stealth] Failed to apply user agent override: #{e.message}"
+    end
+
+    def utils_script
+      @utils_script ||= File.read(UTILS_PATH)
+    end
+
+    def evasion_scripts
+      enabled_evasions.map do |evasion|
+        script = load_evasion_script(evasion)
+        substitute_options(evasion, script)
+      end.compact
+    end
+
+    def load_evasion_script(evasion)
+      path = File.join(EVASIONS_PATH, "#{evasion}.js")
+      return nil unless File.exist?(path)
+
+      File.read(path)
+    end
+
+    def substitute_options(evasion, script)
+      return script unless script
+
+      evasion_opts = EVASION_OPTIONS[evasion]
+      return script unless evasion_opts
+
+      # Merge with user-provided options
+      merged_opts = evasion_opts.merge(options.fetch(evasion, {}))
+
+      # Substitute options in the script
+      # Scripts end with `})({ key: null }); // Will be replaced by Ruby`
+      # rubocop:disable Style/RegexpLiteral
+      script.gsub(/\}\)\(\{[^}]+\}\);?\s*(\/\/.*)?$/) do
+        # rubocop:enable Style/RegexpLiteral
+        "})(#{merged_opts.to_json});"
+      end
+    end
+  end
+end