omniai-tools 0.5.0 → 0.6.0

data/lib/omniai/tools/browser/page_screenshot_tool.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module OmniAI
+  module Tools
+    module Browser
+      # A browser automation tool for taking screenshots of the current page.
+      class PageScreenshotTool < BaseTool
+        description "A browser automation tool for taking screenshots of the current page."
+
+        def execute
+          @logger.info("#{self.class.name}##{__method__}")
+
+          @driver.screenshot do |file|
+            "data:image/png;base64,#{Base64.strict_encode64(file.read)}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/selector_generator/base_selectors.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module SelectorGenerator
+        # Basic selector generation methods
+        module BaseSelectors
+          def placeholder_selector(element, tag)
+            valid_attribute?(element["placeholder"]) ? ["#{tag}[placeholder=\"#{element['placeholder']}\"]"] : []
+          end
+
+          def aria_label_selector(element, tag)
+            valid_attribute?(element["aria-label"]) ? ["#{tag}[aria-label=\"#{element['aria-label']}\"]"] : []
+          end
+
+          def name_selector(element, tag)
+            valid_attribute?(element["name"]) ? ["#{tag}[name=\"#{element['name']}\"]"] : []
+          end
+
+          def maxlength_selector(element, tag)
+            valid_attribute?(element["maxlength"]) ? ["#{tag}[maxlength=\"#{element['maxlength']}\"]"] : []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/selector_generator/contextual_selectors.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module SelectorGenerator
+        # Context-aware selector generation for complex elements
+        module ContextualSelectors
+          def generate_contextual_selectors(element)
+            selectors = []
+            selectors.concat(parent_class_selectors(element))
+            selectors.concat(label_based_selectors(element))
+            selectors.concat(position_based_selectors(element))
+            selectors
+          end
+
+          # Generate selectors based on parent container classes
+          def parent_class_selectors(element)
+            significant_parent = find_significant_parent(element)
+            return [] unless significant_parent
+
+            parent_class = most_specific_class(significant_parent)
+            return [] unless parent_class
+
+            build_parent_selector(element, parent_class)
+          end
+
+          # Build selector with parent class context
+          def build_parent_selector(element, parent_class)
+            base = ".#{parent_class} #{element.name}"
+            return ["#{base}[placeholder=\"#{element['placeholder']}\"]"] if element["placeholder"]
+            return ["#{base}[type=\"#{element['type']}\"]"] if element["type"]
+
+            [base]
+          end
+
+          # Find parent with meaningful class (not generic like 'row' or 'col')
+          def find_significant_parent(element)
+            parent = element.parent
+            while parent && parent.name != "body"
+              return parent if element_has_significant_class?(parent)
+
+              parent = parent.parent
+            end
+            nil
+          end
+
+          # Check if element has significant class
+          def element_has_significant_class?(element)
+            classes = element["class"]&.split || []
+            classes.any? { |c| significant_class?(c) }
+          end
+
+          # Check if class name is likely to be meaningful/specific
+          def significant_class?(class_name)
+            return false if class_name.length < 4
+            return false if generic_class?(class_name)
+
+            class_name.match?(/[a-z]+[-_]?[a-z]+/i)
+          end
+
+          # Common generic class names to ignore
+          def generic_class?(class_name)
+            %w[row col container wrapper inner outer main].include?(class_name.downcase)
+          end
+
+          # Get most specific (longest) class name
+          def most_specific_class(element)
+            classes = element["class"]&.split || []
+            classes.select { |c| significant_class?(c) }.max_by(&:length)
+          end
+
+          # Generate selectors based on label associations
+          def label_based_selectors(element)
+            return [] unless stable_id?(element)
+
+            label = find_label_for_element(element)
+            label ? ["#{element.name}##{element['id']}"] : []
+          end
+
+          # Check if element has stable (non-React) ID
+          def stable_id?(element)
+            id = element["id"]
+            id && !id.empty? && !id.match?(/^:r[0-9a-z]+:$/i)
+          end
+
+          # Find label element associated with this element
+          def find_label_for_element(element)
+            element.document.at_css("label[for=\"#{element['id']}\"]")
+          end
+
+          # Generate position-based selectors for similar elements
+          def position_based_selectors(element)
+            siblings = find_similar_siblings(element)
+            return [] unless siblings.size > 1
+
+            index = siblings.index(element) + 1
+            parent_context = parent_context_prefix(element)
+            build_position_selector(element, index, parent_context)
+          end
+
+          # Build nth-of-type selector
+          def build_position_selector(element, index, parent_context = "")
+            nth = ":nth-of-type(#{index})"
+            base = "#{parent_context}#{element.name}#{nth}"
+            return ["#{parent_context}#{element.name}[type=\"#{element['type']}\"]#{nth}"] if element["type"]
+            if element["placeholder"]
+              return ["#{parent_context}#{element.name}[placeholder=\"#{element['placeholder']}\"]#{nth}"]
+            end
+
+            [base]
+          end
+
+          # Find sibling elements of same type with similar attributes
+          def find_similar_siblings(element)
+            return [] unless element.parent
+
+            element.parent.css(element.name).select { |sibling| same_key_attributes?(element, sibling) }
+          end
+
+          # Check if two elements have same key attributes
+          def same_key_attributes?(elem1, elem2)
+            return false unless elem1.name == elem2.name
+
+            elem1.name == "input" ? elem1["type"] == elem2["type"] : true
+          end
+
+          # Get parent context for more specific position selectors
+          def parent_context_prefix(element)
+            parent = find_significant_parent(element)
+            return "" unless parent
+
+            parent_class = most_specific_class(parent)
+            parent_class ? ".#{parent_class} " : ""
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/selector_generator.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      # Generates stable CSS selectors for HTML elements
+      module SelectorGenerator
+        extend BaseSelectors
+        extend ContextualSelectors
+
+      module_function
+
+        def generate_stable_selectors(element)
+          return [] unless valid_element?(element)
+
+          selectors = []
+          selectors.concat(generate_by_type(element))
+          selectors.concat(generate_contextual_selectors(element))
+          selectors.compact.uniq
+        end
+
+        def generate_by_type(element)
+          case element.name
+          when "input" then generate_input_selectors(element)
+          when "textarea" then generate_textarea_selectors(element)
+          when "select" then generate_select_selectors(element)
+          else []
+          end
+        end
+
+        def valid_element?(element)
+          element.respond_to?(:name) && element.respond_to?(:parent)
+        end
+
+        def generate_input_selectors(element)
+          selectors = []
+          selectors.concat(placeholder_selector(element, "input"))
+          selectors.concat(aria_label_selector(element, "input"))
+          selectors.concat(type_selectors(element))
+          selectors.concat(attribute_selectors(element, "input"))
+          selectors
+        end
+
+        def generate_textarea_selectors(element)
+          placeholder_selector(element, "textarea") + name_selector(element, "textarea")
+        end
+
+        def generate_select_selectors(element)
+          name_selector(element, "select") + aria_label_selector(element, "select")
+        end
+
+        def type_selectors(element)
+          return [] unless valid_attribute?(element["type"])
+
+          base = "input[type=\"#{element['type']}\"]"
+          [base, amount_class_selector(base, element)].compact
+        end
+
+        def amount_class_selector(base, element)
+          element["class"]&.include?("wv-input--amount") ? "#{base}.wv-input--amount" : nil
+        end
+
+        def attribute_selectors(element, tag)
+          maxlength_selector(element, tag) + name_selector(element, tag)
+        end
+
+        def valid_attribute?(attribute)
+          attribute && attribute.strip.length.positive?
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/selector_inspect_tool.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module OmniAI
+  module Tools
+    module Browser
+      # A browser automation tool for inspecting elements using CSS selectors.
+      class SelectorInspectTool < BaseTool
+        include InspectUtils
+
+        description "A browser automation tool for finding and inspecting elements by CSS selector."
+
+        parameter :selector, :string, description: "CSS selector to target specific elements"
+        parameter :context_size, :integer, description: "Number of parent elements to include for context"
+
+        def execute(selector:, context_size: 2)
+          @logger.info("#{self.class.name}##{__method__}")
+
+          doc = cleaned_document(html: @driver.html)
+          target_elements = doc.css(selector)
+
+          return "No elements found matching selector: #{selector}" if target_elements.empty?
+
+          format_elements(target_elements, selector, context_size)
+        end
+
+      private
+
+        def format_elements(elements, selector, context_size)
+          result = "Found #{elements.size} elements matching '#{selector}':\n\n"
+
+          elements.each_with_index do |element, index|
+            result += "--- Element #{index + 1} ---\n"
+            result += Formatters::ElementFormatter.get_parent_context(element, context_size) if context_size.positive?
+            result += "Element: #{element.to_html}\n\n"
+          end
+
+          result
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/text_field_area_set_tool.rb

@@ -20,38 +20,9 @@ module OmniAI
         # @param selector [String] The ID / name of the text field / text area to interact with.
         # @param text [String] The text to set.
         def execute(selector:, text:)
-          @logger.info("#{self.class.name}##{__method__} selector=#{selector.inspect} text=#{text.inspect}")
+          @logger.info("#{self.class.name}##{__method__} selector=#{selector.inspect}")
 
-          element = find(id: selector) || find(name: selector)
-
-          return { error: "unknown selector=#{selector}" } if element.nil?
-
-          element.set(text)
-        end
-
-      protected
-
-        # @param selector [Hash]
-        #
-        # @return [Watir::TextArea, Watir::TextField, nil]
-        def find(selector)
-          find_text_area(selector) || find_text_field(selector)
-        end
-
-        # @param selector [Hash]
-        #
-        # @return [Watir::TextArea, nil]
-        def find_text_area(selector)
-          element = @browser.textarea(selector)
-          element if element.exists?
-        end
-
-        # @param selector [Hash]
-        #
-        # @return [Watir::TextField, nil]
-        def find_text_field(selector)
-          element = @browser.text_field(selector)
-          element if element.exists?
+          @driver.fill_in(selector:, text:)
         end
       end
     end

data/lib/omniai/tools/browser/visit_tool.rb

@@ -20,7 +20,7 @@ module OmniAI
         def execute(url:)
           @logger.info("#{self.class.name}##{__method__} url=#{url.inspect}")
 
-          @browser.goto(url)
+          @driver.goto(url:)
         end
       end
     end

data/lib/omniai/tools/browser/watir_driver.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      # @example
+      #   driver = OmniAI::Tools::Browser::WatirDriver.new
+      #   driver.visit("https://example.com")
+      #   driver.click(id: "submit-button")
+      class WatirDriver < BaseDriver
+        def initialize(logger: Logger.new(IO::NULL), browser: Watir::Browser.new(:chrome))
+          super(logger:)
+          @browser = browser
+        end
+
+        def close
+          @browser.close
+        end
+
+        # @return [String]
+        def url
+          @browser.url
+        end
+
+        # @return [String]
+        def title
+          @browser.title
+        end
+
+        # @return [String]
+        def html
+          @browser.html
+        end
+
+        # @param url [String]
+        def goto(url:)
+          @browser.goto(url)
+
+          { status: :ok }
+        end
+
+        # @yield [file]
+        # @yieldparam file [File]
+        def screenshot
+          tempfile = Tempfile.new(["screenshot", ".png"])
+          @browser.screenshot.save(tempfile.path)
+
+          yield File.open(tempfile.path, "rb")
+        ensure
+          tempfile&.close
+          tempfile&.unlink
+        end
+
+        # @param selector [String]
+        # @param text [String]
+        #
+        # @return [Hash]
+        def fill_in(selector:, text:)
+          element = find_field(selector)
+
+          return { status: :error, message: "unknown selector=#{selector.inspect}" } if element.nil?
+
+          element.set(text)
+
+          { status: :ok }
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def button_click(selector:)
+          element = find_button(selector)
+
+          return { status: error, message: "unknown selector=#{selector.inspect}" } if element.nil?
+
+          element.click
+
+          { status: :ok }
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def link_click(selector:)
+          element = find_link(selector)
+
+          return { status: :error, message: "unknown selector=#{selector.inspect}" } if element.nil?
+
+          element.click
+
+          { status: :ok }
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def element_click(selector:)
+          element = find_element(selector)
+
+          return { status: :error, message: "unknown selector=#{selector.inspect}" } if element.nil?
+
+          element.click
+
+          { status: :ok }
+        rescue TimeoutError => e
+          { status: :error, message: e.message }
+        end
+
+      protected
+
+        def wait_for_element
+          Watir::Wait.until(timeout: TIMEOUT) do
+            element = yield
+            element if element&.visible?
+          end
+        rescue Watir::Wait::TimeoutError
+          nil
+        end
+
+        # @param selector [String]
+        #
+        # @return [Watir::TextField, Watir::TextArea, nil]
+        def find_field(selector)
+          wait_for_element do
+            find_text_area_or_field_by(id: selector) ||
+              find_text_area_or_field_by(name: selector) ||
+              find_text_area_or_field_by(placeholder: selector) ||
+              find_text_area_or_field_by(class: selector) ||
+              find_text_area_or_field_by(css: selector)
+          end
+        end
+
+        # @param selector [String]
+        #
+        # @return [Watir::TextArea, Watir::TextField, nil]
+        def find_text_area_or_field_by(selector)
+          find_text_field_by(selector) || find_text_area_by(selector)
+        end
+
+        # @param selector [String]
+        #
+        # @return [Watir::Button, nil]
+        def find_button(selector)
+          wait_for_element do
+            find_button_by(text: selector) || find_button_by(id: selector)
+          end
+        end
+
+        # @param selector [String]
+        #
+        # @return [Watir::Button, nil]
+        def find_link(selector)
+          wait_for_element do
+            find_link_by(text: selector) || find_link_by(href: selector) || find_link_by(id: selector)
+          end
+        end
+
+        # @param selector [Hash] A hash with one of the following
+        #
+        # @return [Watir::Element, nil]
+        def find_element(selector)
+          wait_for_element do
+            find_element_by(css: selector) ||
+              find_element_by(text: selector) ||
+              find_element_by(id: selector) ||
+              find_element_by(xpath: selector)
+          end
+        end
+
+        # @param selector [Hash]
+        #
+        # @return [Watir::TextArea, nil]
+        def find_text_area_by(selector)
+          element = @browser.textarea(selector)
+          return unless element.respond_to?(:exists?)
+
+          element if element.exists?
+        end
+
+        # @param selector [Hash]
+        #
+        # @return [Watir::TextField, nil]
+        def find_text_field_by(selector)
+          element = @browser.text_field(selector)
+          return unless element.respond_to?(:exists?)
+
+          element if element.exists?
+        end
+
+        # @param selector [String] CSS selector to find the element
+        #
+        # @return [Watir::Element, nil]
+        def find_element_by(selector)
+          element = @browser.element(selector)
+          return nil unless element.respond_to?(:exists?)
+
+          element if element.exists?
+        end
+
+        # @param selector [Hash]
+        #
+        # @return [Watir::Anchor, nil]
+        def find_link_by(selector)
+          element = @browser.link(selector)
+          return unless element.respond_to?(:exists?)
+
+          element if element.exists?
+        end
+
+        # @param selector [Hash]
+        #
+        # @return [Watir::Button, nil]
+        def find_button_by(selector)
+          element = @browser.button(selector)
+          return unless element.respond_to?(:exists?)
+
+          element if element.exists?
+        end
+      end
+    end
+  end
+end