omniai-tools 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d27871f61c1d70be893cce085bf33c0b3d9280686ff1a0ba64f35835b7bf10d4
-  data.tar.gz: 6dfddb639a3cbb99069cbb890bb5d8529bf152e5c246bcc90aaf344505758ea9
+  metadata.gz: cc814f27a50bb0ecceaff25eb4153aed6ade80ca24f7abcb74198e0a31afe627
+  data.tar.gz: c68820e0b080dc4812cd15871c8da74963d27b20960dd9e9e254b214bb313dfa
 SHA512:
-  metadata.gz: ff02138674bc5176e869badb4268530d025e5543ca993eaea8aa74ee2ad114592bd5ac3e8ecd8ad8a882ec0af6dd57b9d74448b04669d3fe8295db3c9176e7b8
-  data.tar.gz: c8a79aa2542b81475148b878ab7785d8bea1b32e4649718bd2a5d4684ba897e010da3d392ce9f3da38374f51e10ea2fc074a3a07a2392172a6c68e93f9298aa5
+  metadata.gz: 6ad4b2eb6e1cee4bbfd6b34f1957bb663de22785f4b2071de5323d38d5614c9438f813bbab9b98b9a232335a43243e82191103e59aca054400323f6137f5886e
+  data.tar.gz: faf87bbacaf66022c34beb835674deda75b359e937bc4a734918b8f2cc28985438eba994b40ac0553c8fb1ab7bfa433532cd4cbe0d6dffa3fa5298fa904565dd

data/README.md

@@ -16,20 +16,11 @@ Database tools are focused on running SQL statements:
 require "omniai/openai"
 require "omniai/tools"
 
-require "watir"
-
-browser = Watir::Browser.new(:chrome)
-
 client = OmniAI::OpenAI::Client.new
-logger = Logger.new($stdout)
 
-tools = [
-  OmniAI::Tools::Browser::VisitTool,
-  OmniAI::Tools::Browser::InspectTool,
-  OmniAI::Tools::Browser::ButtonClickTool,
-  OmniAI::Tools::Browser::LinkClickTool,
-  OmniAI::Tools::Browser::TextFieldAreaSetTool,
-].map { |klass| klass.new(browser:, logger:) }
+logger = Logger.new($stdout)
+driver = OmniAI::Tools::Browser::WatirDriver.new
+tools = [OmniAI::Tools::BrowserTool.new(driver: logger:)]
 
 puts "Type 'exit' or 'quit' to leave."
 

data/lib/omniai/tools/browser/base_driver.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      # A base driver intended to be overridden by specific browser drivers (e.g. waitir).
+      class BaseDriver
+        TIMEOUT = Integer(ENV.fetch("OMNIAI_BROWSER_TIMEOUT", 10))
+
+        # @param logger [Logger]
+        def initialize(logger: Logger.new(IO::NULL))
+          @logger = logger
+        end
+
+        def close
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @return [String]
+        def url
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @return [String]
+        def title
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @return [String]
+        def html
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @yield [file]
+        # @yieldparam file [File]
+        def screenshot
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param url [String]
+        #
+        # @return [Hash]
+        def goto(url:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param selector [String]
+        # @param text [String]
+        #
+        # @return [Hash]
+        def fill_in(selector:, text:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def button_click(selector:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def link_click(selector:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def element_click(selector:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/base_tool.rb

@@ -10,12 +10,39 @@ module OmniAI
       #     # ...
       #   end
       class BaseTool < OmniAI::Tool
-        # @param logger [IO] An optional logger for debugging executed commands.
-        # @param browser [Watir::Browser]
-        def initialize(browser:, logger: Logger.new(IO::NULL))
+        # @param logger [Logger]
+        # @param driver [BaseDriver]
+        def initialize(driver:, logger: Logger.new(IO::NULL))
           super()
+          @driver = driver
           @logger = logger
-          @browser = browser
+        end
+
+      protected
+
+        def wait_for_element
+          return yield if defined?(RSpec) # Skip waiting in tests
+
+          Watir::Wait.until(timeout: 10) do
+            element = yield
+            element if element && element_visible?(element)
+          end
+        rescue Watir::Wait::TimeoutError
+          log_element_timeout
+          nil
+        end
+
+        def element_visible?(element)
+          return true unless element.respond_to?(:visible?)
+
+          element.visible?
+        end
+
+        def log_element_timeout
+          return unless @browser.respond_to?(:elements)
+
+          visible_elements = @browser.elements.select(&:visible?).map(&:text).compact.first(10)
+          @logger.error("Element not found after 10s. Sample visible elements: #{visible_elements}")
         end
       end
     end

data/lib/omniai/tools/browser/button_click_tool.rb

@@ -16,23 +16,10 @@ module OmniAI
 
         required %i[selector]
 
-        # @param to [String] The URL to navigate to.
         def execute(selector:)
           @logger.info("#{self.class.name}##{__method__} selector=#{selector.inspect}")
 
-          element = find(text: selector) || find(id: selector)
-
-          return { error: "unknown selector=#{selector}" } if element.nil?
-
-          element.click
-        end
-
-      protected
-
-        # @return [Watir::Anchor, nil]
-        def find(selector)
-          element = @browser.button(selector)
-          element if element.exists?
+          @driver.button_click(selector:)
         end
       end
     end

data/lib/omniai/tools/browser/element_click_tool.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      # @example
+      #   browser = Watir::Browser.new(:chrome)
+      #   tool = OmniAI::Tools::Browser::ElementClickTool.new(browser:)
+      #   tool.execute(selector: "#some-id")
+      #   tool.execute(selector: ".some-class")
+      #   tool.execute(selector: "some text")
+      #   tool.execute(selector: "//div[@role='button']")
+      class ElementClickTool < BaseTool
+        description "A browser automation tool for clicking any clickable element."
+
+        parameter :selector, :string,
+          description: "CSS selector, ID, text content, or other identifier for the element to click."
+
+        required %i[selector]
+
+        # @param selector [String] CSS selector, ID, text content, or other identifier for the element to click.
+        def execute(selector:)
+          @logger.info("#{self.class.name}##{__method__} selector=#{selector.inspect}")
+
+          @driver.element_click(selector:)
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/elements/element_grouper.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Elements
+        # Groups HTML elements by their relevance for data entry
+        module ElementGrouper
+        module_function
+
+          def group_for_data_entry(elements, text)
+            groups = {
+              inputs: [], # text, number, email inputs - primary targets
+              form_controls: {}, # radio buttons, checkboxes grouped by name
+              labels: [], # labels, headers, spans
+              actions: [], # buttons and actionable elements
+            }
+
+            elements.each do |element|
+              categorize_element(element, groups, text)
+            end
+
+            groups
+          end
+
+          def categorize_element(element, groups, text)
+            case element.name.downcase
+            when "input"
+              categorize_input_element(element, groups)
+            when "textarea", "select"
+              groups[:inputs] << element
+            when "button"
+              groups[:actions] << element if contains_text_match?(element, text)
+            when "label", "span", "div", "th"
+              groups[:labels] << element if contains_text_match?(element, text)
+            end
+          end
+
+          def categorize_input_element(element, groups)
+            if data_entry_input?(element)
+              groups[:inputs] << element
+            else
+              group_form_control(element, groups[:form_controls])
+            end
+          end
+
+          def data_entry_input?(element)
+            # Handle missing type attribute - HTML default is "text"
+            type = (element["type"] || "text").downcase
+
+            # Include common data entry input types
+            %w[text number email tel url date datetime-local time month week
+               password search].include?(type)
+          end
+
+          def group_form_control(element, form_controls)
+            type = (element["type"] || "text").downcase
+            return unless %w[radio checkbox].include?(type)
+
+            name = element["name"] || "unnamed"
+            form_controls[name] ||= []
+            form_controls[name] << element
+          end
+
+          def contains_text_match?(element, text)
+            element.text.downcase.include?(text.downcase) ||
+              element["value"]&.downcase&.include?(text.downcase)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/elements/nearby_element_detector.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Elements
+        # Handles detection of interactive elements near text matches
+        module NearbyElementDetector
+        module_function
+
+          def add_nearby_interactive_elements(elements)
+            nearby_elements = []
+            elements.each do |element|
+              nearby_elements += find_interactive_in_context(element)
+            end
+            combined = elements.to_a + nearby_elements
+            combined.uniq
+          end
+
+          def find_interactive_in_context(element)
+            interactive_elements = []
+            interactive_elements += check_table_context(element)
+            interactive_elements += check_parent_containers(element)
+            interactive_elements.uniq
+          end
+
+          def check_table_context(element)
+            elements = []
+            table_cell = element.ancestors("td").first || element.ancestors("th").first
+            elements += find_interactive_in_container(table_cell) if table_cell
+            table_row = element.ancestors("tr").first
+            elements += find_interactive_in_container(table_row) if table_row
+            if table_cell && table_row
+              table = element.ancestors("table").first
+              elements += find_interactive_in_same_column(table, table_cell, table_row) if table
+            end
+            elements
+          end
+
+          def find_interactive_in_same_column(table, header_cell, header_row)
+            return [] unless table
+
+            column_index = header_row.css("th, td").index(header_cell)
+            return [] unless column_index
+
+            interactive_elements = []
+            table.css("tbody tr").each do |row|
+              interactive_elements += find_column_cell_elements(row, column_index)
+            end
+            interactive_elements
+          end
+
+          def find_column_cell_elements(row, column_index)
+            target_cell = row.css("td, th")[column_index]
+            return [] unless target_cell
+
+            find_interactive_in_container(target_cell) + find_interactive_in_nested_tables(target_cell, column_index)
+          end
+
+          def find_interactive_in_nested_tables(cell, original_column_index)
+            interactive_elements = []
+
+            cell.css("table").each do |nested_table|
+              nested_table.css("tr").each do |nested_row|
+                nested_cells = nested_row.css("td, th")
+                if nested_cells[original_column_index]
+                  interactive_elements += find_interactive_in_container(nested_cells[original_column_index])
+                end
+              end
+            end
+
+            interactive_elements
+          end
+
+          def check_parent_containers(element)
+            interactive_elements = []
+            parent_container = element.parent
+            3.times do
+              break unless parent_container
+
+              interactive_elements += find_interactive_in_container(parent_container)
+              parent_container = parent_container.parent
+            end
+            interactive_elements
+          end
+
+          def find_interactive_in_container(container)
+            return [] unless container
+
+            container.css("input, textarea, select, button, [role='button'], [tabindex='0']")
+              .reject { |el| non_interactive_element?(el) }
+          end
+
+          def non_interactive_element?(element)
+            return true if element["type"] == "hidden"
+            return true if element["tabindex"] == "-1"
+            return true if element["aria-hidden"] == "true"
+
+            style = element["style"]
+            return true if style&.include?("display: none") || style&.include?("visibility: hidden")
+
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/formatters/action_formatter.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Formatters
+        # Handles formatting of action elements (buttons, links)
+        module ActionFormatter
+        module_function
+
+          def format_actions(actions)
+            result = "⚡ Available Actions:\n"
+            actions.first(5).each do |action|
+              result += format_action_element(action)
+            end
+            result += "  ... and #{actions.size - 5} more\n" if actions.size > 5
+            result += "\n"
+          end
+
+          def format_action_element(action)
+            text = action.text.strip
+            selector = get_action_selector(action)
+            "  • #{text} (#{selector})\n"
+          end
+
+          def get_action_selector(action)
+            return action["id"] if action["id"] && !action["id"].empty?
+            return "text:#{action.text.strip}" if action.text.strip.length > 2
+            return action["class"].split.first if action["class"]
+
+            "css-needed"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/formatters/data_entry_formatter.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Formatters
+        # Formats grouped elements for optimal data entry display
+        module DataEntryFormatter
+        module_function
+
+          def format_groups(grouped_elements, total_count, text)
+            result = build_header(total_count, text)
+            result += format_each_group(grouped_elements)
+            result
+          end
+
+          def format_inputs(inputs)
+            result = "📝 Data Entry Fields:\n"
+            table_groups = group_by_table_context(inputs)
+            result += format_table_groups(table_groups)
+            "#{result}\n"
+          end
+
+          def format_form_controls(form_controls)
+            result = "🎛️ Form Controls:\n"
+            form_controls.each do |name, controls|
+              result += format_control_group(name, controls)
+            end
+            result += "\n"
+          end
+
+          def format_labels(labels)
+            result = "🏷️ Labels & Headers:\n"
+            labels.first(3).each do |label|
+              result += "  • #{label.name}: #{label.text.strip}\n"
+            end
+            result += "  ... and #{labels.size - 3} more\n" if labels.size > 3
+            result += "\n"
+          end
+
+          def build_header(total_count, text)
+            "Found #{total_count} elements containing '#{text}':\n\n"
+          end
+
+          def format_each_group(grouped_elements)
+            [
+              format_inputs_section(grouped_elements[:inputs]),
+              format_actions_section(grouped_elements[:actions]),
+              format_controls_section(grouped_elements[:form_controls]),
+              format_labels_section(grouped_elements[:labels]),
+            ].join
+          end
+
+          def format_inputs_section(inputs)
+            inputs.any? ? format_inputs(inputs) : ""
+          end
+
+          def format_actions_section(actions)
+            actions.any? ? ActionFormatter.format_actions(actions) : ""
+          end
+
+          def format_controls_section(form_controls)
+            form_controls.any? ? format_form_controls(form_controls) : ""
+          end
+
+          def format_labels_section(labels)
+            labels.any? ? format_labels(labels) : ""
+          end
+
+          def format_control_group(name, controls)
+            result = "\n  #{name.humanize} options:\n"
+            controls.each do |control|
+              checked = control["checked"] ? " \u2713" : ""
+              result += "    • #{control['value']}#{checked}\n"
+            end
+            result
+          end
+
+          def format_table_groups(table_groups)
+            return format_single_group(table_groups.values.first) if table_groups.size == 1
+
+            result = ""
+            table_groups.each do |(context, group_inputs)|
+              result += "\n  #{context}:\n"
+              group_inputs.each { |input| result += InputFormatter.format_input_field(input, "    ") }
+            end
+            result
+          end
+
+          def format_single_group(inputs)
+            result = ""
+            inputs.each { |input| result += InputFormatter.format_input_field(input, "  ") }
+            result
+          end
+
+          def group_by_table_context(inputs)
+            groups = {}
+            inputs.each do |input|
+              context = find_context(input)
+              groups[context] ||= []
+              groups[context] << input
+            end
+            groups
+          end
+
+          def find_context(input)
+            table = input.ancestors("table").first
+            return "Form Fields" unless table
+
+            find_table_context(input)
+          end
+
+          def find_table_context(input)
+            row = input.ancestors("tr").first
+            return "Table" unless row
+
+            find_meaningful_cell_text(row)
+          end
+
+          def find_meaningful_cell_text(row)
+            meaningful_cell = row.css("td, th").find do |cell|
+              cell.text.strip.length > 2 && !cell.text.match?(/^\d+\.?\d*$/)
+            end
+
+            meaningful_cell ? meaningful_cell.text.strip[0..30] : "Table"
+          end
+
+          def humanize(text)
+            text.gsub(/[-_]/, " ").split.map(&:capitalize).join(" ")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/formatters/element_formatter.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Formatters
+        # Handles formatting of HTML elements for display with data entry focus
+        module ElementFormatter
+        module_function
+
+          def format_matching_elements(elements, text, _context_size = nil)
+            grouped = Elements::ElementGrouper.group_for_data_entry(elements, text)
+            DataEntryFormatter.format_groups(grouped, elements.size, text)
+          end
+
+          # Keep existing methods for backward compatibility
+          def format_single_element(element, index, context_size)
+            result = "--- Element #{index + 1} ---\n"
+            result += "Tag: #{element.name}\n"
+            result += format_element_attributes(element)
+            result += get_parent_context(element, context_size) if context_size.positive?
+            result += "HTML: #{element.to_html}\n\n"
+            result
+          end
+
+          def format_element_attributes(element)
+            result = ""
+            result += "ID: #{element['id']}\n" if element["id"]
+            result += "Classes: #{element['class']}\n" if element["class"]
+            %w[href src alt type value placeholder].each do |attr|
+              result += "#{attr}: #{element[attr]}\n" if element[attr] && !element[attr].empty?
+            end
+            result
+          end
+
+          def get_parent_context(element, context_size)
+            result = ""
+            parent = element.parent
+            context_count = 0
+            while parent && context_count < context_size
+              attrs = parent.attributes.map { |name, attr| " #{name}=\"#{attr.value}\"" }.join
+              result += "Parent #{context_count + 1}: <#{parent.name}#{attrs}>\n"
+              parent = parent.parent
+              context_count += 1
+            end
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/formatters/input_formatter.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Formatters
+        # Handles formatting of input elements
+        module InputFormatter
+        module_function
+
+          def format_input_field(input, indent = "")
+            result = format_basic_line(input, indent)
+            result += format_selectors_line(input, indent)
+            result
+          end
+
+          def format_basic_line(input, indent)
+            result = "#{indent}• #{input_type_display(input)}"
+            result += input_id_display(input)
+            result += input_value_display(input)
+            result += input_placeholder_display(input)
+            "#{result}\n"
+          end
+
+          def format_selectors_line(input, indent)
+            selectors = SelectorGenerator.generate_stable_selectors(input)
+            return "" if selectors.empty?
+
+            format_selector_list(selectors, indent)
+          end
+
+          def format_selector_list(selectors, indent)
+            result = "#{indent}  Stable selectors:\n"
+            selectors.each { |sel| result += "#{indent}    - #{sel}\n" }
+            "#{result}\n"
+          end
+
+          def input_type_display(input)
+            (input["type"] || input.name).capitalize
+          end
+
+          def input_id_display(input)
+            input["id"] ? " (#{input['id']})" : ""
+          end
+
+          def input_value_display(input)
+            value = input["value"]
+            value && !value.empty? ? " = '#{value}'" : ""
+          end
+
+          def input_placeholder_display(input)
+            placeholder = input["placeholder"]
+            placeholder && !placeholder.empty? ? " [#{placeholder}]" : ""
+          end
+        end
+      end
+    end
+  end
+end