omniai-tools 0.4.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f361e26584723aa8d30ba41edaefafb44a7e08c5c6345656dbf25e37c965448d
-  data.tar.gz: c718feeb39dfdd1c2c5ff9a2976ece26eec48aae12f52405724fc21209101d32
+  metadata.gz: cc814f27a50bb0ecceaff25eb4153aed6ade80ca24f7abcb74198e0a31afe627
+  data.tar.gz: c68820e0b080dc4812cd15871c8da74963d27b20960dd9e9e254b214bb313dfa
 SHA512:
-  metadata.gz: fa14b3333edc0b235e046d9f408f75f201ea37be76435d897873fb7585999cd0aeac3d36c86c16e72a772ab6a28f48dd40560074767720736583de79a80d8847
-  data.tar.gz: 6ec511fc27bae83abdf0d827572ad2784419039b0d37d128d5670b19d58645a8e6efd82bfb7132397473181b56897f966314396a76172aaec33883826ff14f12
+  metadata.gz: 6ad4b2eb6e1cee4bbfd6b34f1957bb663de22785f4b2071de5323d38d5614c9438f813bbab9b98b9a232335a43243e82191103e59aca054400323f6137f5886e
+  data.tar.gz: faf87bbacaf66022c34beb835674deda75b359e937bc4a734918b8f2cc28985438eba994b40ac0553c8fb1ab7bfa433532cd4cbe0d6dffa3fa5298fa904565dd

data/Gemfile

@@ -6,6 +6,11 @@ gemspec
 
 gem "factory_bot"
 gem "irb"
+gem "nokogiri"
+gem "omniai-anthropic"
+gem "omniai-google"
+gem "omniai-mistral"
+gem "omniai-openai"
 gem "rake"
 gem "redcarpet"
 gem "rspec"
@@ -17,4 +22,5 @@ gem "rubocop-rake"
 gem "rubocop-rspec"
 gem "simplecov"
 gem "sqlite3"
+gem "watir"
 gem "yard"

data/README.md

@@ -8,6 +8,51 @@
 
 `OmniAI::Tools` is a library of pre-built tools to simplify integrating common tasks with [OmniAI](https://github.com/ksylvest/omniai).
 
+## Browser
+
+Database tools are focused on running SQL statements:
+
+```ruby
+require "omniai/openai"
+require "omniai/tools"
+
+client = OmniAI::OpenAI::Client.new
+
+logger = Logger.new($stdout)
+driver = OmniAI::Tools::Browser::WatirDriver.new
+tools = [OmniAI::Tools::BrowserTool.new(driver: logger:)]
+
+puts "Type 'exit' or 'quit' to leave."
+
+prompt = OmniAI::Chat::Prompt.build do |builder|
+  builder.system <<~TEXT
+    You are tasked with assisting a user in browsing the web.
+  TEXT
+end
+
+loop do
+  print "# "
+  text = gets.strip
+  break if %w[exit quit].include?(text)
+
+  prompt.user(text)
+  response = client.chat(prompt, stream: $stdout, tools:)
+  prompt.assistant(response.text)
+end
+```
+
+```
+Type 'exit' or 'quit' to leave.
+# Visit news.ycombinator.com and list the top 5 posts.
+
+[browser] OmniAI::Tools::Browser::VisitTool#execute url="https://news.ycombinator.com"
+[browser] OmniAI::Tools::Browser::InspectTool#execute
+
+Here are the top 5 posts on Hacker News right now:
+
+...
+```
+
 ## Database
 
 Database tools are focused on running SQL statements:

data/lib/omniai/tools/browser/base_driver.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      # A base driver intended to be overridden by specific browser drivers (e.g. waitir).
+      class BaseDriver
+        TIMEOUT = Integer(ENV.fetch("OMNIAI_BROWSER_TIMEOUT", 10))
+
+        # @param logger [Logger]
+        def initialize(logger: Logger.new(IO::NULL))
+          @logger = logger
+        end
+
+        def close
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @return [String]
+        def url
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @return [String]
+        def title
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @return [String]
+        def html
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @yield [file]
+        # @yieldparam file [File]
+        def screenshot
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param url [String]
+        #
+        # @return [Hash]
+        def goto(url:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param selector [String]
+        # @param text [String]
+        #
+        # @return [Hash]
+        def fill_in(selector:, text:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def button_click(selector:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def link_click(selector:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+
+        # @param selector [String]
+        #
+        # @return [Hash]
+        def element_click(selector:)
+          raise NotImplementedError, "#{self.class.name}#{__method__} undefined"
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/base_tool.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "watir"
+
+module OmniAI
+  module Tools
+    module Browser
+      # @example
+      #   class SeleniumTool < BaseTool
+      #     # ...
+      #   end
+      class BaseTool < OmniAI::Tool
+        # @param logger [Logger]
+        # @param driver [BaseDriver]
+        def initialize(driver:, logger: Logger.new(IO::NULL))
+          super()
+          @driver = driver
+          @logger = logger
+        end
+
+      protected
+
+        def wait_for_element
+          return yield if defined?(RSpec) # Skip waiting in tests
+
+          Watir::Wait.until(timeout: 10) do
+            element = yield
+            element if element && element_visible?(element)
+          end
+        rescue Watir::Wait::TimeoutError
+          log_element_timeout
+          nil
+        end
+
+        def element_visible?(element)
+          return true unless element.respond_to?(:visible?)
+
+          element.visible?
+        end
+
+        def log_element_timeout
+          return unless @browser.respond_to?(:elements)
+
+          visible_elements = @browser.elements.select(&:visible?).map(&:text).compact.first(10)
+          @logger.error("Element not found after 10s. Sample visible elements: #{visible_elements}")
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/button_click_tool.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+
+module OmniAI
+  module Tools
+    module Browser
+      # @example
+      #   browser = Watir::Browser.new(:chrome)
+      #   tool = OmniAI::Tools::Browser::VisitTool.new(browser:)
+      #   tool.click_link(selector: "link_id")
+      class ButtonClickTool < BaseTool
+        description "A browser automation tool for clicking a specific button."
+
+        parameter :selector, :string, description: "The ID or text of the button to interact with."
+
+        required %i[selector]
+
+        def execute(selector:)
+          @logger.info("#{self.class.name}##{__method__} selector=#{selector.inspect}")
+
+          @driver.button_click(selector:)
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/element_click_tool.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      # @example
+      #   browser = Watir::Browser.new(:chrome)
+      #   tool = OmniAI::Tools::Browser::ElementClickTool.new(browser:)
+      #   tool.execute(selector: "#some-id")
+      #   tool.execute(selector: ".some-class")
+      #   tool.execute(selector: "some text")
+      #   tool.execute(selector: "//div[@role='button']")
+      class ElementClickTool < BaseTool
+        description "A browser automation tool for clicking any clickable element."
+
+        parameter :selector, :string,
+          description: "CSS selector, ID, text content, or other identifier for the element to click."
+
+        required %i[selector]
+
+        # @param selector [String] CSS selector, ID, text content, or other identifier for the element to click.
+        def execute(selector:)
+          @logger.info("#{self.class.name}##{__method__} selector=#{selector.inspect}")
+
+          @driver.element_click(selector:)
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/elements/element_grouper.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Elements
+        # Groups HTML elements by their relevance for data entry
+        module ElementGrouper
+        module_function
+
+          def group_for_data_entry(elements, text)
+            groups = {
+              inputs: [], # text, number, email inputs - primary targets
+              form_controls: {}, # radio buttons, checkboxes grouped by name
+              labels: [], # labels, headers, spans
+              actions: [], # buttons and actionable elements
+            }
+
+            elements.each do |element|
+              categorize_element(element, groups, text)
+            end
+
+            groups
+          end
+
+          def categorize_element(element, groups, text)
+            case element.name.downcase
+            when "input"
+              categorize_input_element(element, groups)
+            when "textarea", "select"
+              groups[:inputs] << element
+            when "button"
+              groups[:actions] << element if contains_text_match?(element, text)
+            when "label", "span", "div", "th"
+              groups[:labels] << element if contains_text_match?(element, text)
+            end
+          end
+
+          def categorize_input_element(element, groups)
+            if data_entry_input?(element)
+              groups[:inputs] << element
+            else
+              group_form_control(element, groups[:form_controls])
+            end
+          end
+
+          def data_entry_input?(element)
+            # Handle missing type attribute - HTML default is "text"
+            type = (element["type"] || "text").downcase
+
+            # Include common data entry input types
+            %w[text number email tel url date datetime-local time month week
+               password search].include?(type)
+          end
+
+          def group_form_control(element, form_controls)
+            type = (element["type"] || "text").downcase
+            return unless %w[radio checkbox].include?(type)
+
+            name = element["name"] || "unnamed"
+            form_controls[name] ||= []
+            form_controls[name] << element
+          end
+
+          def contains_text_match?(element, text)
+            element.text.downcase.include?(text.downcase) ||
+              element["value"]&.downcase&.include?(text.downcase)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/elements/nearby_element_detector.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Elements
+        # Handles detection of interactive elements near text matches
+        module NearbyElementDetector
+        module_function
+
+          def add_nearby_interactive_elements(elements)
+            nearby_elements = []
+            elements.each do |element|
+              nearby_elements += find_interactive_in_context(element)
+            end
+            combined = elements.to_a + nearby_elements
+            combined.uniq
+          end
+
+          def find_interactive_in_context(element)
+            interactive_elements = []
+            interactive_elements += check_table_context(element)
+            interactive_elements += check_parent_containers(element)
+            interactive_elements.uniq
+          end
+
+          def check_table_context(element)
+            elements = []
+            table_cell = element.ancestors("td").first || element.ancestors("th").first
+            elements += find_interactive_in_container(table_cell) if table_cell
+            table_row = element.ancestors("tr").first
+            elements += find_interactive_in_container(table_row) if table_row
+            if table_cell && table_row
+              table = element.ancestors("table").first
+              elements += find_interactive_in_same_column(table, table_cell, table_row) if table
+            end
+            elements
+          end
+
+          def find_interactive_in_same_column(table, header_cell, header_row)
+            return [] unless table
+
+            column_index = header_row.css("th, td").index(header_cell)
+            return [] unless column_index
+
+            interactive_elements = []
+            table.css("tbody tr").each do |row|
+              interactive_elements += find_column_cell_elements(row, column_index)
+            end
+            interactive_elements
+          end
+
+          def find_column_cell_elements(row, column_index)
+            target_cell = row.css("td, th")[column_index]
+            return [] unless target_cell
+
+            find_interactive_in_container(target_cell) + find_interactive_in_nested_tables(target_cell, column_index)
+          end
+
+          def find_interactive_in_nested_tables(cell, original_column_index)
+            interactive_elements = []
+
+            cell.css("table").each do |nested_table|
+              nested_table.css("tr").each do |nested_row|
+                nested_cells = nested_row.css("td, th")
+                if nested_cells[original_column_index]
+                  interactive_elements += find_interactive_in_container(nested_cells[original_column_index])
+                end
+              end
+            end
+
+            interactive_elements
+          end
+
+          def check_parent_containers(element)
+            interactive_elements = []
+            parent_container = element.parent
+            3.times do
+              break unless parent_container
+
+              interactive_elements += find_interactive_in_container(parent_container)
+              parent_container = parent_container.parent
+            end
+            interactive_elements
+          end
+
+          def find_interactive_in_container(container)
+            return [] unless container
+
+            container.css("input, textarea, select, button, [role='button'], [tabindex='0']")
+              .reject { |el| non_interactive_element?(el) }
+          end
+
+          def non_interactive_element?(element)
+            return true if element["type"] == "hidden"
+            return true if element["tabindex"] == "-1"
+            return true if element["aria-hidden"] == "true"
+
+            style = element["style"]
+            return true if style&.include?("display: none") || style&.include?("visibility: hidden")
+
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/formatters/action_formatter.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Formatters
+        # Handles formatting of action elements (buttons, links)
+        module ActionFormatter
+        module_function
+
+          def format_actions(actions)
+            result = "⚡ Available Actions:\n"
+            actions.first(5).each do |action|
+              result += format_action_element(action)
+            end
+            result += "  ... and #{actions.size - 5} more\n" if actions.size > 5
+            result += "\n"
+          end
+
+          def format_action_element(action)
+            text = action.text.strip
+            selector = get_action_selector(action)
+            "  • #{text} (#{selector})\n"
+          end
+
+          def get_action_selector(action)
+            return action["id"] if action["id"] && !action["id"].empty?
+            return "text:#{action.text.strip}" if action.text.strip.length > 2
+            return action["class"].split.first if action["class"]
+
+            "css-needed"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/formatters/data_entry_formatter.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Formatters
+        # Formats grouped elements for optimal data entry display
+        module DataEntryFormatter
+        module_function
+
+          def format_groups(grouped_elements, total_count, text)
+            result = build_header(total_count, text)
+            result += format_each_group(grouped_elements)
+            result
+          end
+
+          def format_inputs(inputs)
+            result = "📝 Data Entry Fields:\n"
+            table_groups = group_by_table_context(inputs)
+            result += format_table_groups(table_groups)
+            "#{result}\n"
+          end
+
+          def format_form_controls(form_controls)
+            result = "🎛️ Form Controls:\n"
+            form_controls.each do |name, controls|
+              result += format_control_group(name, controls)
+            end
+            result += "\n"
+          end
+
+          def format_labels(labels)
+            result = "🏷️ Labels & Headers:\n"
+            labels.first(3).each do |label|
+              result += "  • #{label.name}: #{label.text.strip}\n"
+            end
+            result += "  ... and #{labels.size - 3} more\n" if labels.size > 3
+            result += "\n"
+          end
+
+          def build_header(total_count, text)
+            "Found #{total_count} elements containing '#{text}':\n\n"
+          end
+
+          def format_each_group(grouped_elements)
+            [
+              format_inputs_section(grouped_elements[:inputs]),
+              format_actions_section(grouped_elements[:actions]),
+              format_controls_section(grouped_elements[:form_controls]),
+              format_labels_section(grouped_elements[:labels]),
+            ].join
+          end
+
+          def format_inputs_section(inputs)
+            inputs.any? ? format_inputs(inputs) : ""
+          end
+
+          def format_actions_section(actions)
+            actions.any? ? ActionFormatter.format_actions(actions) : ""
+          end
+
+          def format_controls_section(form_controls)
+            form_controls.any? ? format_form_controls(form_controls) : ""
+          end
+
+          def format_labels_section(labels)
+            labels.any? ? format_labels(labels) : ""
+          end
+
+          def format_control_group(name, controls)
+            result = "\n  #{name.humanize} options:\n"
+            controls.each do |control|
+              checked = control["checked"] ? " \u2713" : ""
+              result += "    • #{control['value']}#{checked}\n"
+            end
+            result
+          end
+
+          def format_table_groups(table_groups)
+            return format_single_group(table_groups.values.first) if table_groups.size == 1
+
+            result = ""
+            table_groups.each do |(context, group_inputs)|
+              result += "\n  #{context}:\n"
+              group_inputs.each { |input| result += InputFormatter.format_input_field(input, "    ") }
+            end
+            result
+          end
+
+          def format_single_group(inputs)
+            result = ""
+            inputs.each { |input| result += InputFormatter.format_input_field(input, "  ") }
+            result
+          end
+
+          def group_by_table_context(inputs)
+            groups = {}
+            inputs.each do |input|
+              context = find_context(input)
+              groups[context] ||= []
+              groups[context] << input
+            end
+            groups
+          end
+
+          def find_context(input)
+            table = input.ancestors("table").first
+            return "Form Fields" unless table
+
+            find_table_context(input)
+          end
+
+          def find_table_context(input)
+            row = input.ancestors("tr").first
+            return "Table" unless row
+
+            find_meaningful_cell_text(row)
+          end
+
+          def find_meaningful_cell_text(row)
+            meaningful_cell = row.css("td, th").find do |cell|
+              cell.text.strip.length > 2 && !cell.text.match?(/^\d+\.?\d*$/)
+            end
+
+            meaningful_cell ? meaningful_cell.text.strip[0..30] : "Table"
+          end
+
+          def humanize(text)
+            text.gsub(/[-_]/, " ").split.map(&:capitalize).join(" ")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/formatters/element_formatter.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module Formatters
+        # Handles formatting of HTML elements for display with data entry focus
+        module ElementFormatter
+        module_function
+
+          def format_matching_elements(elements, text, _context_size = nil)
+            grouped = Elements::ElementGrouper.group_for_data_entry(elements, text)
+            DataEntryFormatter.format_groups(grouped, elements.size, text)
+          end
+
+          # Keep existing methods for backward compatibility
+          def format_single_element(element, index, context_size)
+            result = "--- Element #{index + 1} ---\n"
+            result += "Tag: #{element.name}\n"
+            result += format_element_attributes(element)
+            result += get_parent_context(element, context_size) if context_size.positive?
+            result += "HTML: #{element.to_html}\n\n"
+            result
+          end
+
+          def format_element_attributes(element)
+            result = ""
+            result += "ID: #{element['id']}\n" if element["id"]
+            result += "Classes: #{element['class']}\n" if element["class"]
+            %w[href src alt type value placeholder].each do |attr|
+              result += "#{attr}: #{element[attr]}\n" if element[attr] && !element[attr].empty?
+            end
+            result
+          end
+
+          def get_parent_context(element, context_size)
+            result = ""
+            parent = element.parent
+            context_count = 0
+            while parent && context_count < context_size
+              attrs = parent.attributes.map { |name, attr| " #{name}=\"#{attr.value}\"" }.join
+              result += "Parent #{context_count + 1}: <#{parent.name}#{attrs}>\n"
+              parent = parent.parent
+              context_count += 1
+            end
+            result
+          end
+        end
+      end
+    end
+  end
+end