chromate-rb 0.0.1.pre → 0.0.3.pre

data/lib/chromate/actions/dom.rb

@@ -3,36 +3,44 @@
 module Chromate
   module Actions
     module Dom
-      def find_element(selector)
-        Chromate::Element.new(selector, @client)
-      end
-
-      def click_element(selector)
-        find_element(selector).click
-      end
-
-      def hover_element(selector)
-        find_element(selector).hover
-      end
-
-      def type_text(selector, text)
-        find_element(selector).type(text)
-      end
-
-      def get_text(selector)
-        find_element(selector).text
-      end
-
-      def get_property(selector, property)
-        find_element(selector).attributes[property]
-      end
-
-      def select_option(selector, option)
-        Chromate::Elements::Select.new(selector, @client).select_option(option)
+      # @return [String]
+      def source
+        evaluate_script('document.documentElement.outerHTML')
       end
 
+      # @param selector [String] CSS selector
+      # @return [Chromate::Element]
+      def find_element(selector)
+        base_element = Chromate::Element.new(selector, @client)
+
+        options = {
+          object_id: base_element.object_id,
+          node_id: base_element.node_id,
+          root_id: base_element.root_id
+        }
+
+        if base_element.select?
+          Chromate::Elements::Select.new(selector, @client, **options)
+        elsif base_element.option?
+          Chromate::Elements::Option.new(selector, @client, **options)
+        elsif base_element.radio?
+          Chromate::Elements::Radio.new(selector, @client, **options)
+        elsif base_element.checkbox?
+          Chromate::Elements::Checkbox.new(selector, @client, **options)
+        else
+          base_element
+        end
+      end
+
+      # @param selector [String] CSS selector
+      # @return [String]
       def evaluate_script(script)
-        @client.send_message('Runtime.evaluate', expression: script)
+        result = @client.send_message('Runtime.evaluate', expression: script, returnByValue: true)
+
+        result['result']['value']
+      rescue StandardError => e
+        Chromate::CLogger.log("Error evaluating script: #{e.message}", :error)
+        nil
       end
     end
   end

data/lib/chromate/actions/navigate.rb

@@ -17,22 +17,25 @@ module Chromate
         dom_content_loaded = false
         frame_stopped_loading = false
 
-        # Utiliser un Mutex pour la synchronisation
+        # Use Mutex for synchronization
         mutex = Mutex.new
         condition = ConditionVariable.new
 
-        # S'abonner aux messages WebSocket
+        # Subscribe to websocket messages
         listener = proc do |message|
           mutex.synchronize do
             case message['method']
             when 'Page.domContentEventFired'
               dom_content_loaded = true
+              Chromate::CLogger.log('DOMContentEventFired')
               condition.signal if dom_content_loaded && page_loaded && frame_stopped_loading
             when 'Page.loadEventFired'
               page_loaded = true
+              Chromate::CLogger.log('LoadEventFired')
               condition.signal if dom_content_loaded && page_loaded && frame_stopped_loading
             when 'Page.frameStoppedLoading'
               frame_stopped_loading = true
+              Chromate::CLogger.log('FrameStoppedLoading')
               condition.signal if dom_content_loaded && page_loaded && frame_stopped_loading
             end
           end
@@ -40,15 +43,14 @@ module Chromate
 
         @client.on_message(&listener)
 
-        # Attendre les trois événements (DOMContent, Load et FrameStoppedLoading) avec un timeout
+        # Wait for all three events (DOMContent, Load and FrameStoppedLoading) with a timeout
         Timeout.timeout(15) do
           mutex.synchronize do
             condition.wait(mutex) until dom_content_loaded && page_loaded && frame_stopped_loading
           end
         end
 
-        # Nettoyer l'écouteur WebSocket
-        @client.on_message { |msg| } # Supprime tous les anciens écouteurs en ajoutant un listener vide
+        @client.on_message { |msg| } # Remove listener
 
         self
       end

data/lib/chromate/actions/screenshot.rb

@@ -3,15 +3,47 @@
 module Chromate
   module Actions
     module Screenshot
-      def screenshot_to_file(file_path, options = {})
-        image_data = screenshot(options)
+      # @param file_path [String] The path to save the screenshot to
+      # @param options [Hash] Options for the screenshot
+      # @option options [String] :format The format of the screenshot (default: 'png')
+      # @option options [Boolean] :full_page Whether to take a screenshot of the full page
+      # @option options [Boolean] :fromSurface Whether to take a screenshot from the surface
+      # @return [Hash] A hash containing the path and base64-encoded image data of the screenshot
+      def screenshot(file_path = "#{Time.now.to_i}.png", options = {})
+        file_path ||= "#{Time.now.to_i}.png"
+        return xvfb_screenshot(file_path) if @xfvb
+
+        if options[:full_page]
+          original_viewport = fetch_viewport_size
+          update_screen_size_to_full_page!
+        end
+
+        image_data = make_screenshot(options)
+        reset_screen_size! if options[:full_page]
+
         File.binwrite(file_path, image_data)
-        true
+
+        {
+          path: file_path,
+          base64: Base64.encode64(image_data)
+        }
+      ensure
+        restore_viewport_size(original_viewport) if options[:full_page]
       end
 
-      def screenshot_full_page(file_path, options = {})
-        metrics = @client.send_message('Page.getLayoutMetrics')
+      private
 
+      # @param file_path [String] The path to save the screenshot to
+      # @return [Boolean] Whether the screenshot was successful
+      def xvfb_screenshot(file_path)
+        display = ENV['DISPLAY'] || ':99'
+        system("xwd -root -display #{display} | convert xwd:- #{file_path}")
+      end
+
+      # Updates the screen size to match the full page dimensions
+      # @return [void]
+      def update_screen_size_to_full_page!
+        metrics = @client.send_message('Page.getLayoutMetrics')
         content_size = metrics['contentSize']
         width = content_size['width'].ceil
         height = content_size['height'].ceil
@@ -22,29 +54,54 @@ module Chromate
                                height: height,
                                deviceScaleFactor: 1
                              })
+      end
 
-        screenshot_to_file(file_path, options)
-
+      # Resets the device metrics override
+      # @return [void]
+      def reset_screen_size!
         @client.send_message('Emulation.clearDeviceMetricsOverride')
-        true
       end
 
-      def xvfb_screenshot(file_path)
-        display = ENV['DISPLAY'] || ':99'
-        system("xwd -root -display #{display} | convert xwd:- #{file_path}")
+      # Fetches the current viewport size
+      # @return [Hash] The current viewport dimensions
+      def fetch_viewport_size
+        metrics = @client.send_message('Page.getLayoutMetrics')
+        {
+          width: metrics['layoutViewport']['clientWidth'],
+          height: metrics['layoutViewport']['clientHeight']
+        }
       end
 
-      private
+      # Restores the viewport size to its original dimensions
+      # @param viewport [Hash] The original viewport dimensions
+      # @return [void]
+      def restore_viewport_size(viewport)
+        return unless viewport
+
+        @client.send_message('Emulation.setDeviceMetricsOverride', {
+                               mobile: false,
+                               width: viewport[:width],
+                               height: viewport[:height],
+                               deviceScaleFactor: 1
+                             })
+      end
 
-      def screenshot(options = {})
+      # @param options [Hash] Options for the screenshot
+      # @option options [String] :format The format of the screenshot
+      # @option options [Boolean] :fromSurface Whether to take a screenshot from the surface
+      # @return [String] The image data
+      def make_screenshot(options = {})
         default_options = {
           format: 'png',
-          fromSurface: true
+          fromSurface: true,
+          captureBeyondViewport: true
         }
 
         params = default_options.merge(options)
 
         @client.send_message('Page.enable')
+        @client.send_message('DOM.enable')
+        @client.send_message('DOM.getDocument', depth: -1, pierce: true)
 
         result = @client.send_message('Page.captureScreenshot', params)
 

data/lib/chromate/actions/stealth.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'user_agent_parser'
+
+module Chromate
+  module Actions
+    module Stealth
+      # @return [void]
+      def patch
+        @client.send_message('Network.enable')
+        inject_stealth_script
+
+        # TODO: Improve dynamic user agent overriding
+        # It currently breaks fingerprint validation (pixcelscan.com)
+        # override_user_agent(@user_agent)
+      end
+
+      # @return [void]
+      def inject_stealth_script
+        stealth_script = File.read(File.join(__dir__, '../files/stealth.js'))
+        @client.send_message('Page.addScriptToEvaluateOnNewDocument', { source: stealth_script })
+      end
+
+      # @param user_agent [String]
+      # @return [void]
+      def override_user_agent(user_agent) # rubocop:disable Metrics/MethodLength
+        u_agent     = UserAgentParser.parse(user_agent)
+        platform    = Chromate::UserAgent.os
+        version     = u_agent.version
+        brands      = [
+          { brand: u_agent.family || 'Not_A_Brand', version: version.major },
+          { brand: u_agent.device.brand || 'Not_A_Brand', version: u_agent.os.version.to_s }
+        ]
+
+        custom_headers = {
+          'User-Agent' => user_agent,
+          'Accept-Language' => 'en-US,en;q=0.9',
+          'Sec-CH-UA' => brands.map { |brand| "\"#{brand[:brand]}\";v=\"#{brand[:version]}\"" }.join(', '),
+          'Sec-CH-UA-Platform' => "\"#{u_agent.device.family}\"",
+          'Sec-CH-UA-Mobile' => '?0'
+        }
+        @client.send_message('Network.setExtraHTTPHeaders', headers: custom_headers)
+
+        user_agent_override = {
+          userAgent: user_agent,
+          platform: platform,
+          acceptLanguage: 'en-US,en;q=0.9',
+          userAgentMetadata: {
+            brands: brands,
+            fullVersion: version.to_s,
+            platform: platform,
+            platformVersion: u_agent.os.version.to_s,
+            architecture: Chromate::UserAgent.arch,
+            model: '',
+            mobile: false
+          }
+        }
+        @client.send_message('Network.setUserAgentOverride', user_agent_override)
+      end
+    end
+  end
+end

data/lib/chromate/binary.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Chromate
+  class Binary
+    def self.run(path, args, need_success: true)
+      command = [path] + args
+      stdout, stderr, status = Open3.capture3(*command)
+      raise stderr if need_success && !status.success?
+
+      stdout
+    end
+
+    attr_reader :pid
+
+    # @param [String] path
+    # @param [Array<String>] args
+    def initialize(path, args)
+      @path = path
+      @args = args || []
+      @pid  = nil
+    end
+
+    # @return [self]
+    def start
+      command = [@path] + @args
+      _stdin, _stdout, _stderr, wait_thr = Open3.popen3(*command)
+      CLogger.log("Started process with pid #{wait_thr.pid}", level: :debug)
+      Process.detach(wait_thr.pid)
+      CLogger.log("Process detached with pid #{wait_thr.pid}", level: :debug)
+      @pid = wait_thr.pid
+
+      self
+    end
+
+    # @return [Boolean]
+    def started?
+      !@pid.nil?
+    end
+
+    def running?
+      return false unless started?
+
+      Process.getpgid(@pid).is_a?(Integer)
+    rescue Errno::ESRCH
+      false
+    end
+
+    # @return [self]
+    def stop
+      stop_process
+    end
+
+    # @return [Boolean]
+    def stopped?
+      @pid.nil?
+    end
+
+    private
+
+    def stop_process(timeout: 5)
+      return unless pid
+
+      # Send SIGINT to the process to stop it gracefully
+      Process.kill('INT', pid)
+      begin
+        Timeout.timeout(timeout) do
+          Process.wait(pid)
+        end
+      rescue Timeout::Error
+        # If the process does not stop gracefully, send SIGKILL
+        CLogger.log("Process #{pid} did not stop gracefully. Sending SIGKILL...", level: :debug)
+        Process.kill('KILL', pid)
+        Process.wait(pid)
+      end
+    rescue Errno::ESRCH
+      # The process has already stopped
+    ensure
+      @pid = nil
+    end
+  end
+end

data/lib/chromate/browser.rb

@@ -6,14 +6,20 @@ require 'securerandom'
 require 'net/http'
 require 'websocket-client-simple'
 require_relative 'helpers'
+require_relative 'binary'
 require_relative 'client'
-require_relative 'element'
 require_relative 'hardwares'
+require_relative 'element'
 require_relative 'elements/select'
+require_relative 'elements/option'
+require_relative 'elements/tags'
+require_relative 'elements/radio'
+require_relative 'elements/checkbox'
 require_relative 'user_agent'
 require_relative 'actions/navigate'
 require_relative 'actions/screenshot'
 require_relative 'actions/dom'
+require_relative 'actions/stealth'
 
 module Chromate
   class Browser
@@ -23,7 +29,15 @@ module Chromate
     include Actions::Navigate
     include Actions::Screenshot
     include Actions::Dom
-
+    include Actions::Stealth
+
+    # @param options [Hash] Options for the browser
+    # @option options [String] :chrome_path The path to the Chrome executable
+    # @option options [String] :user_data_dir The path to the user data directory
+    # @option options [Boolean] :headless Whether to run Chrome in headless mode
+    # @option options [Boolean] :xfvb Whether to run Chrome in Xvfb
+    # @option options [Boolean] :native_control Whether to use native controls
+    # @option options [Boolean] :record Whether to record the screen
     def initialize(options = {})
       @options        = config.options.merge(options)
       @chrome_path    = @options.fetch(:chrome_path)
@@ -32,29 +46,21 @@ module Chromate
       @xfvb           = @options.fetch(:xfvb)
       @native_control = @options.fetch(:native_control)
       @record         = @options.fetch(:record, false)
-      @process        = nil
-      @xfvb_process   = nil
+      @binary         = nil
       @record_process = nil
       @client         = nil
-      @args           = [
-        @chrome_path,
-        "--user-data-dir=#{@user_data_dir}",
-        "--lang=#{@options[:lang] || "fr-FR"}"
-      ]
+      @args           = []
 
       trap('INT') { stop_and_exit }
       trap('TERM') { stop_and_exit }
-
-      at_exit { stop }
     end
 
+    # @return [self]
     def start
       build_args
       @client = Client.new(self)
       @args << "--remote-debugging-port=#{@client.port}"
 
-      start_video_recording if @record
-
       if @xfvb
         if ENV['DISPLAY'].nil?
           ENV['DISPLAY'] = ':0'   if mac? # XQuartz generally uses :0 on Mac
@@ -63,49 +69,139 @@ module Chromate
         @args << "--display=#{ENV.fetch("DISPLAY", nil)}"
       end
 
-      @process = spawn(*@args, err: 'chrome_errors.log', out: 'chrome_output.log')
-      sleep 2
+      Hardwares::MouseController.reset_mouse_position
+      Chromate::CLogger.log("Starting browser with args: #{@args}", level: :debug)
+      @binary = Binary.new(@chrome_path, @args)
 
+      @binary.start
       @client.start
+
+      start_video_recording if @record
+
+      patch if config.patch?
+
+      update_config!
+
       self
     end
 
+    # @return [Boolean]
+    def started?
+      @binary&.started? || false
+    end
+
+    # @return [self]
     def stop
-      Process.kill('TERM', @process)        if @process
-      Process.kill('TERM', @record_process) if @record_process
-      Process.kill('TERM', @xfvb_process)   if @xfvb_process
+      stop_process(@record_process) if @record_process
+      @binary.stop                  if started?
       @client&.stop
+
+      @binary = nil
+      @record_process = nil
+
+      self
     end
 
+    # @return [Boolean]
     def native_control?
       @native_control
     end
 
     private
 
+    # @return [Integer]
     def start_video_recording
-      outfile = File.join(Dir.pwd, "output_video_#{Time.now.to_i}.mp4")
-      @record_process = spawn("ffmpeg -f x11grab -r 25 -s 1920x1080 -i #{ENV.fetch("DISPLAY", ":99")} -pix_fmt yuv420p -y #{outfile}")
+      outname = @record.is_a?(String) ? @record : "output_video_#{Time.now.to_i}.mp4"
+      outfile = File.join(Dir.pwd, outname).to_s
+      args = [
+        '-f',
+        'x11grab',
+        '-draw_mouse',
+        '1',
+        '-r',
+        '30',
+        '-s',
+        '1920x1080',
+        '-i',
+        ENV.fetch('DISPLAY'),
+        '-c:v',
+        'libx264',
+        '-preset',
+        'ultrafast',
+        '-pix_fmt',
+        'yuv420p',
+        '-y',
+        outfile
+      ]
+      binary = Binary.new('ffmpeg', args)
+      binary.start
+      @record_process = binary.pid
     end
 
+    # @return [Array<String>]
     def build_args
       exclude_switches = config.exclude_switches || []
       exclude_switches += @options[:exclude_switches] if @options[:exclude_switches]
+      @user_agent = @options[:user_agent] || UserAgent.call
 
-      @args += config.generate_arguments(**@options)
-      @args += @options[:options][:args] if @options.dig(:options, :args)
-      @args << "--user-agent=#{@options[:user_agent] || UserAgent.call}"
+      @args = if @options.dig(:options, :args)
+                @options[:options][:args]
+              else
+                config.generate_arguments(**@options)
+              end
+
+      @args << "--user-agent=#{@user_agent}"
       @args << "--exclude-switches=#{exclude_switches.join(",")}" if exclude_switches.any?
+      @args << "--user-data-dir=#{@user_data_dir}"
 
       @args
     end
 
+    def set_hardwares
+      config.mouse_controller = Hardwares.mouse(client: @client, element: nil)
+      config.keyboard_controller = Hardwares.keyboard(client: @client, element: nil)
+    end
+
+    # @return [void]
+    def update_config!
+      config.args = @args
+      config.user_data_dir = @user_data_dir
+      config.headless = @headless
+      config.xfvb = @xfvb
+      config.native_control = @native_control
+
+      set_hardwares
+    end
+
+    # @param pid [Integer] PID of the process to stop
+    # @param timeout [Integer] Timeout in seconds to wait for the process to stop
+    # @return [void]
+    def stop_process(pid, timeout: 5)
+      return unless pid
+
+      # Send SIGINT to the process to stop it gracefully
+      Process.kill('INT', pid)
+      begin
+        Timeout.timeout(timeout) do
+          Process.wait(pid)
+        end
+      rescue Timeout::Error
+        # If the process does not stop gracefully, send SIGKILL
+        CLogger.log("Process #{pid} did not stop gracefully. Sending SIGKILL...", level: :debug)
+        Process.kill('KILL', pid)
+      end
+    rescue Errno::ESRCH
+      # The process has already stopped
+    end
+
+    # @return [void]
     def stop_and_exit
-      puts 'Stopping browser...'
+      CLogger.log('Stopping browser...', level: :debug)
       stop
       exit
     end
 
+    # @return [Chromate::Configuration]
     def config
       Chromate.configuration
     end

data/lib/chromate/c_logger.rb

@@ -4,17 +4,25 @@ require 'logger'
 
 module Chromate
   class CLogger < Logger
+    # @param [IO] logdev
+    # @param [Integer] shift_age
+    # @param [Integer] shift_size
     def initialize(logdev, shift_age: 0, shift_size: 1_048_576)
       super(logdev, shift_age, shift_size)
       self.formatter = proc do |severity, datetime, _progname, msg|
         "[Chromate] #{datetime.strftime("%Y-%m-%d %H:%M:%S")} #{severity}: #{msg}\n"
       end
+      self.level = ENV['CHROMATE_DEBUG'] ? :debug : :info
     end
 
+    # @return [Chromate::CLogger]
     def self.logger
       @logger ||= new($stdout)
     end
 
+    # @param [String] message
+    # @param [Symbol] level
+    # @return [void]
     def self.log(message, level: :info)
       logger.send(level, message)
     end