omniai-tools 0.5.0 → 0.5.1

data/lib/omniai/tools/browser/inspect_tool.rb

@@ -5,25 +5,58 @@ require "nokogiri"
 module OmniAI
   module Tools
     module Browser
-      # @example
-      #   browser = Watir::Browser.new(:chrome)
-      #   tool = OmniAI::Tools::Browser::InspectTool.new(browser:)
-      #   tool.execute
+      # A browser automation tool for finding UI elements by their text content.
       class InspectTool < BaseTool
-        description "A browser automation tool for viewing the HTML for the browser."
+        include InspectUtils
 
-        # @return [String]
-        def execute
+        description "A browser automation tool for finding UI elements by their text content."
+
+        parameter :text_content, :string, description: "Search for elements containing this text"
+        parameter :selector, :string, description: "Optional CSS selector to further filter results"
+        parameter :context_size, :integer, description: "Number of parent elements to include for context"
+
+        def execute(text_content:, selector: nil, context_size: 2)
           @logger.info("#{self.class.name}##{__method__}")
 
-          html = @browser.html
-          doc = Nokogiri::HTML(html)
+          html = @driver.html
+
+          @logger.info("#{self.class.name}##{__method__} html=#{html}")
+
+          doc = cleaned_document(html: @driver.html)
+          find_elements_by_text(doc, text_content, context_size, selector)
+        end
+
+      private
+
+        def find_elements_by_text(doc, text, context_size, additional_selector = nil)
+          elements = get_elements_matching_text(doc, text, additional_selector)
+
+          return "No elements found containing text: #{text}" if elements.empty?
+
+          adjusted_context_size = additional_selector ? 0 : context_size
+
+          Formatters::ElementFormatter.format_matching_elements(elements, text, adjusted_context_size)
+        end
+
+        def get_elements_matching_text(doc, text, additional_selector)
+          text_downcase = text.downcase
+
+          elements = find_elements_with_matching_text(doc, text_downcase)
+
+          elements = add_elements_from_matching_labels(doc, text_downcase, elements)
+
+          unless additional_selector && !additional_selector.empty?
+            elements = Elements::NearbyElementDetector.add_nearby_interactive_elements(elements)
+          end
+
+          apply_additional_selector(doc, elements, additional_selector)
+        end
 
-          doc.css("link").each(&:remove)
-          doc.css("style").each(&:remove)
-          doc.css("script").each(&:remove)
+        def apply_additional_selector(doc, elements, additional_selector)
+          return elements.uniq unless additional_selector && !additional_selector.empty?
 
-          doc.to_html
+          css_matches = doc.css(additional_selector)
+          elements.select { |el| css_matches.include?(el) }.uniq
         end
       end
     end

data/lib/omniai/tools/browser/inspect_utils.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      # Utility methods for browser inspection tools that handle HTML document cleaning
+      # and various element searching functionalities.
+      module InspectUtils
+        def cleaned_document(html:)
+          clean_document(Nokogiri::HTML(html))
+        end
+
+        def clean_document(doc)
+          doc.css("link, style, script").each(&:remove)
+          doc
+        end
+
+        def ci_contains(attribute, value)
+          "contains(translate(#{attribute}, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', " \
+            "'abcdefghijklmnopqrstuvwxyz'), '#{value}')"
+        end
+
+        def find_elements_with_matching_text(doc, text_downcase)
+          xpath_conditions = [
+            ci_contains("text()", text_downcase),
+            ci_contains("@value", text_downcase),
+            ci_contains("@placeholder", text_downcase),
+            ci_contains("@type", text_downcase),
+          ].join(" or ")
+
+          doc.xpath("//*[#{xpath_conditions}]")
+        end
+
+        def add_elements_from_matching_labels(doc, text_downcase, elements)
+          label_condition = ci_contains(".//text()", text_downcase)
+          matching_labels = doc.xpath("//label[#{label_condition}]")
+
+          matching_labels.each do |label|
+            for_attr = label["for"]
+            next unless for_attr && !for_attr.empty?
+
+            associated_input = doc.css("[id='#{for_attr}']")
+            elements += associated_input if associated_input.any?
+          end
+
+          elements
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/link_click_tool.rb

@@ -14,23 +14,11 @@ module OmniAI
 
         parameter :selector, :string, description: "The ID or text of the link to interact with."
 
-        # @param to [String] The ID or text of the link to interact with.
+        # @param selector [String] The ID or text of the link to interact with.
         def execute(selector:)
           @logger.info("#{self.class.name}##{__method__} selector=#{selector.inspect}")
 
-          element = find(text: selector) || find(value: selector) || find(id: selector)
-
-          return { error: "unknown selector=#{selector}" } if element.nil?
-
-          element.click
-        end
-
-      protected
-
-        # @return [Watir::Anchor, nil]
-        def find(selector)
-          element = @browser.a(selector)
-          element if element.exists?
+          @driver.link_click(selector:)
         end
       end
     end

data/lib/omniai/tools/browser/page_inspect/button_summarizer.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module PageInspect
+        # Module to handle button elements summarization for AI agents
+        module ButtonSummarizer
+        module_function
+
+          def summarize_primary_actions(doc)
+            buttons = find_primary_buttons(doc)
+            return "" if buttons.empty?
+
+            format_primary_actions(buttons)
+          end
+
+          def find_primary_buttons(doc)
+            all_buttons = doc.css('button, input[type="button"], input[type="submit"], [role="button"], [tabindex="0"]')
+            all_buttons.select { |btn| primary_action?(btn) && !skip_button?(btn) }
+          end
+
+          def skip_button?(button)
+            button["disabled"] ||
+              button["style"]&.include?("display: none") ||
+              button["aria-hidden"] == "true"
+          end
+
+          def primary_action?(button)
+            return true if button["type"] == "submit"
+            return true if primary_button_text?(button)
+            return true if primary_button_class?(button)
+            return true if workflow_action_button?(button)
+
+            false
+          end
+
+          def primary_button_text?(button)
+            text = get_button_text(button).downcase
+
+            # Direct keyword matches
+            primary_keywords = %w[save submit continue next finish send create update
+                                  delete cancel close done confirm proceed add edit]
+            return true if primary_keywords.any? { |keyword| text.include?(keyword) }
+
+            # Workflow action patterns
+            return true if text.include?("add") && text.match?(/item|customer|discount|product|contact|line/)
+            return true if text.include?("choose") || text.include?("select")
+
+            false
+          end
+
+          def primary_button_class?(button)
+            classes = button["class"] || ""
+
+            # Generic primary button patterns (universal)
+            primary_classes = %w[primary submit btn-primary button--primary save continue]
+            generic_match = primary_classes.any? { |css_class| classes.include?(css_class) }
+
+            # Generic link-button patterns (works across frameworks)
+            link_button_patterns = %w[button--link btn-link link-button button-link]
+            link_match = link_button_patterns.any? { |pattern| classes.include?(pattern) }
+
+            generic_match || link_match
+          end
+
+          def workflow_action_button?(button)
+            text = get_button_text(button).downcase
+
+            # Check for common workflow patterns
+            return true if text.match?(/add.*(item|customer|discount|product|contact)/i)
+            return true if text.match?(/edit.*(column|field|profile)/i)
+            return true if text.match?(/choose.*(different|customer)/i)
+            return true if text.match?(/(create|new).*(item|customer|product)/i)
+
+            false
+          end
+
+          def get_button_text(button)
+            text = button.text.strip
+            text = button["value"] if text.empty? && button["value"]
+            text = button["aria-label"] if text.empty? && button["aria-label"]
+
+            text.empty? ? "Button" : text
+          end
+
+          def format_primary_actions(buttons)
+            result = "⚡ Primary Actions:\n"
+
+            # Group by importance
+            critical = buttons.select(&method(:critical_action?))
+            regular = buttons - critical
+
+            result += format_button_group(critical, "🔥 Critical")
+            result += format_button_group(regular, "📝 Actions")
+
+            "#{result}\n"
+          end
+
+          def critical_action?(button)
+            text = get_button_text(button).downcase
+            %w[save submit send create].any? do |keyword|
+              text.include?(keyword)
+            end
+          end
+
+          def format_button_group(buttons, title)
+            return "" if buttons.empty?
+
+            result = "#{title}:\n"
+            buttons.first(5).each do |btn|
+              result += format_action_button(btn)
+            end
+            result += "  ... and #{buttons.size - 5} more\n" if buttons.size > 5
+            result += "\n"
+          end
+
+          def format_action_button(button)
+            text = get_button_text(button)
+            selector = get_button_selector(button)
+
+            "  • #{text} (#{selector})\n"
+          end
+
+          def get_button_selector(button)
+            return button["id"] if button["id"] && !button["id"].empty?
+            return "text:#{get_button_text(button)}" if distinctive_text?(button)
+
+            "css-selector-needed"
+          end
+
+          def distinctive_text?(button)
+            text = get_button_text(button)
+            text.length > 2 && text != "Button"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/page_inspect/form_summarizer.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module PageInspect
+        # Module to handle form elements summarization for AI agents
+        module FormSummarizer
+        module_function
+
+          def summarize_data_entry_opportunities(doc)
+            fields = find_data_entry_fields(doc)
+            return "" if fields.empty?
+
+            format_for_agents(fields)
+          end
+
+          def summarize_form_structure(doc)
+            fields = doc.css("input, textarea, select")
+            return "📝 No form fields available.\n\n" if fields.empty?
+
+            "📝 Form Fields Available: #{fields.size} fields found\n\n"
+          end
+
+          def find_data_entry_fields(doc)
+            doc.css("input, textarea, select").reject { |f| skip_field?(f) }
+          end
+
+          def skip_field?(field)
+            field["type"] == "hidden" ||
+              field["disabled"] ||
+              %w[button submit reset].include?(field["type"])
+          end
+
+          def format_for_agents(fields)
+            result = "📝 Data Entry Fields:\n"
+
+            # Group important fields first
+            date_fields = fields.select { |f| date_field?(f) }
+            text_fields = fields.select { |f| text_field?(f) }
+            other_fields = fields - date_fields - text_fields
+
+            result += format_field_group("📅 Date Fields", date_fields)
+            result += format_field_group("📝 Text Fields", text_fields)
+            result += format_field_group("🔧 Other Fields", other_fields)
+
+            "#{result}\n"
+          end
+
+          def format_field_group(title, fields)
+            return "" if fields.empty?
+
+            result = "#{title}:\n"
+            fields.first(8).each { |field| result += format_agent_field(field) }
+            result += "  ... and #{fields.size - 8} more\n" if fields.size > 8
+            result += "\n"
+          end
+
+          def format_agent_field(field)
+            label = get_field_label(field)
+            id = field["id"] || "no-id"
+            value = field["value"] ? " = '#{field['value']}'" : ""
+            placeholder = field["placeholder"] ? " [#{field['placeholder']}]" : ""
+
+            "  • #{label} (#{id})#{value}#{placeholder}\n"
+          end
+
+          def get_field_label(field)
+            return get_associated_label(field) if field["id"]
+            return field["placeholder"] if field["placeholder"]
+
+            field_type = field.name == "input" ? (field["type"] || "text") : field.name
+            field_type.capitalize
+          end
+
+          def get_associated_label(field)
+            label = field.document.at_css("label[for='#{field['id']}']")
+            return nil unless label&.text
+
+            text = label.text.strip
+            text.empty? ? nil : text
+          end
+
+          def date_field?(field)
+            field["type"] == "date" ||
+              field["placeholder"]&.match?(/date|yyyy|mm|dd/i) ||
+              get_associated_label(field)&.match?(/date|due/i)
+          end
+
+          def text_field?(field)
+            %w[text email tel url textarea].include?(field["type"]) ||
+              field.name == "textarea"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/page_inspect/html_summarizer.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module PageInspect
+        # Module to handle HTML formatting and summary generation for AI agents
+        module HtmlSummarizer
+        module_function
+
+          def summarize_interactive_elements(doc)
+            title = doc.at_css("title")&.text || "Untitled Page"
+
+            summary = "#{title}\n\n"
+
+            # Primary focus: What can agents fill out?
+            data_entry = FormSummarizer.summarize_data_entry_opportunities(doc)
+            summary += data_entry unless data_entry.empty?
+
+            # Secondary: What actions can agents take?
+            primary_actions = ButtonSummarizer.summarize_primary_actions(doc)
+            summary += primary_actions unless primary_actions.empty?
+
+            # Tertiary: Key navigation (only if relevant)
+            navigation = LinkSummarizer.summarize_key_navigation(doc)
+            summary += navigation unless navigation.empty?
+
+            # Fallback: If no data entry found, show form structure
+            summary += FormSummarizer.summarize_form_structure(doc) if data_entry.empty?
+
+            summary.strip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/page_inspect/link_summarizer.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module PageInspect
+        # Module to handle link elements summarization for AI agents
+        module LinkSummarizer
+        module_function
+
+          def summarize_key_navigation(doc)
+            nav_links = find_navigation_links(doc)
+            return "" if nav_links.empty?
+
+            format_navigation(nav_links)
+          end
+
+          def find_navigation_links(doc)
+            links = doc.css("a[href]").reject { |link| skip_link?(link) }
+            links.select { |link| navigation_link?(link) }
+          end
+
+          def skip_link?(link)
+            href = link["href"]
+            return true if href.nil? || href.empty? || href == "#"
+            return true if href.start_with?("javascript:")
+            return true if link["style"]&.include?("display: none")
+
+            false
+          end
+
+          def navigation_link?(link)
+            return true if main_navigation?(link)
+            return true if workflow_link?(link)
+
+            false
+          end
+
+          def main_navigation?(link)
+            ancestors = link.ancestors.map { |el| el["class"] }.compact.join(" ")
+            nav_indicators = %w[nav navigation menu main-nav primary-nav app-menu breadcrumb]
+
+            nav_indicators.any? { |indicator| ancestors.include?(indicator) }
+          end
+
+          def workflow_link?(link)
+            text = get_link_text(link).downcase
+            workflow_keywords = %w[dashboard home create new add edit settings
+                                   invoice estimate customer payment back continue]
+
+            workflow_keywords.any? { |keyword| text.include?(keyword) }
+          end
+
+          def get_link_text(link)
+            text = link.text.strip
+            text = link["title"] if text.empty? && link["title"]
+            text = link["aria-label"] if text.empty? && link["aria-label"]
+
+            text.empty? ? "Link" : text
+          end
+
+          def format_navigation(links)
+            result = "🧭 Key Navigation:\n"
+
+            main_nav = links.select { |l| main_navigation?(l) }
+            actions = links.select { |l| workflow_link?(l) && !main_navigation?(l) }
+
+            result += format_link_group(main_nav, "📍 Main Menu")
+            result += format_link_group(actions, "🔗 Quick Actions")
+
+            "#{result}\n"
+          end
+
+          def format_link_group(links, title)
+            return "" if links.empty?
+
+            result = "#{title}:\n"
+            links.first(6).each { |link| result += format_nav_link(link) }
+            result += "  ... and #{links.size - 6} more\n" if links.size > 6
+            result += "\n"
+          end
+
+          def format_nav_link(link)
+            text = get_link_text(link)
+            destination = extract_destination(link["href"])
+
+            "  • #{text}#{destination}\n"
+          end
+
+          def extract_destination(href)
+            return "" if href.nil? || href.empty?
+            return "" unless href.include?("/")
+
+            path = href.split("/").last
+            return "" if path.nil? || path.empty?
+
+            " → #{path}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/page_inspect_tool.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module OmniAI
+  module Tools
+    module Browser
+      # A browser automation tool for viewing the full HTML of the page.
+      class PageInspectTool < BaseTool
+        include InspectUtils
+
+        description "A browser automation tool for viewing the full HTML of the current page."
+
+        parameter :summarize, :boolean, description: "If true, returns a summary instead of full HTML"
+
+        def execute(summarize: false)
+          @logger.info("#{self.class.name}##{__method__}")
+
+          doc = cleaned_document(html: @driver.html)
+
+          if summarize
+            PageInspect::HtmlSummarizer.summarize_interactive_elements(doc)
+          else
+            doc.to_html
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/page_screenshot_tool.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "base64"
+
+module OmniAI
+  module Tools
+    module Browser
+      # A browser automation tool for taking screenshots of the current page.
+      class PageScreenshotTool < BaseTool
+        description "A browser automation tool for taking screenshots of the current page."
+
+        def execute
+          @logger.info("#{self.class.name}##{__method__}")
+
+          @driver.screenshot do |file|
+            "data:image/png;base64,#{Base64.strict_encode64(file.read)}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/browser/selector_generator/base_selectors.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Browser
+      module SelectorGenerator
+        # Basic selector generation methods
+        module BaseSelectors
+          def placeholder_selector(element, tag)
+            valid_attribute?(element["placeholder"]) ? ["#{tag}[placeholder=\"#{element['placeholder']}\"]"] : []
+          end
+
+          def aria_label_selector(element, tag)
+            valid_attribute?(element["aria-label"]) ? ["#{tag}[aria-label=\"#{element['aria-label']}\"]"] : []
+          end
+
+          def name_selector(element, tag)
+            valid_attribute?(element["name"]) ? ["#{tag}[name=\"#{element['name']}\"]"] : []
+          end
+
+          def maxlength_selector(element, tag)
+            valid_attribute?(element["maxlength"]) ? ["#{tag}[maxlength=\"#{element['maxlength']}\"]"] : []
+          end
+        end
+      end
+    end
+  end
+end