omniai-tools 0.5.0 → 0.6.0

data/lib/omniai/tools/computer_tool.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    # A tool for interacting with a computer. Be careful with using as it can perform actions on your computer!
+    #
+    # @example
+    #   computer = OmniAI::Tools::Computer::MacTool.new
+    #   computer.display # { "width": 2560, "height": 1440, "scale": 1 }
+    #   computer.screenshot
+    class ComputerTool < OmniAI::Tool
+      description "A tool for interacting with a computer."
+
+      module Action
+        KEY = "key" # press a key
+        HOLD_KEY = "hold_key" # hold a key
+        MOUSE_POSITION = "mouse_position" # get the current (x, y) pixel coordinate of the cursor on the screen
+        MOUSE_MOVE = "mouse_move" # move the cursor to a specific (x, y) pixel coordinate on the screen
+        MOUSE_CLICK = "mouse_click" # click at a specific x / y coordinate
+        MOUSE_DOWN = "mouse_down" # press the mouse button down
+        MOUSE_DRAG = "mouse_drag" # drag the mouse to a specific x / y coordinate
+        MOUSE_UP = "mouse_up" # release the mouse button
+        MOUSE_DOUBLE_CLICK = "mouse_double_click" # double click at a specific x / y coordinate
+        MOUSE_TRIPLE_CLICK = "mouse_triple_click" # triple click at a specific x / y coordinate
+        TYPE = "type" # type a string
+        SCROLL = "scroll"
+        WAIT = "wait"
+      end
+
+      module MouseButton
+        LEFT = "left"
+        MIDDLE = "middle"
+        RIGHT = "right"
+      end
+
+      module ScrollDirection
+        UP = "up"
+        DOWN = "down"
+        LEFT = "left"
+        RIGHT = "right"
+      end
+
+      ACTIONS = [
+        Action::KEY,
+        Action::HOLD_KEY,
+        Action::MOUSE_POSITION,
+        Action::MOUSE_MOVE,
+        Action::MOUSE_CLICK,
+        Action::MOUSE_DOWN,
+        Action::MOUSE_DRAG,
+        Action::MOUSE_UP,
+        Action::TYPE,
+        Action::SCROLL,
+        Action::WAIT,
+      ].freeze
+
+      MOUSE_BUTTON_OPTIONS = [
+        MouseButton::LEFT,
+        MouseButton::MIDDLE,
+        MouseButton::RIGHT,
+      ].freeze
+
+      SCROLL_DIRECTION_OPTIONS = [
+        ScrollDirection::UP,
+        ScrollDirection::DOWN,
+        ScrollDirection::LEFT,
+        ScrollDirection::RIGHT,
+      ].freeze
+
+      parameter :action, :string, enum: ACTIONS, description: <<~TEXT
+        Options:
+        * `#{Action::KEY}`: Press a single key / combination of keys on the keyboard:
+          - supports xdotool's `key` syntax (e.g. "alt+Tab", "Return", "ctrl+s", etc)
+        * `#{Action::HOLD_KEY}`: Hold down a key or multiple keys for a specified duration (in seconds):
+          - supports xdotool's `key` syntax (e.g. "alt+Tab", "Return", "ctrl+s", etc)
+        * `#{Action::MOUSE_POSITION}`: Get the current (x,y) pixel coordinate of the cursor on the screen.
+        * `#{Action::MOUSE_MOVE}`: Move the cursor to a specified (x,y) pixel coordinate on the screen.
+        * `#{Action::MOUSE_CLICK}`: Click the mouse button at the specified (x,y) pixel coordinate on the screen.
+        * `#{Action::MOUSE_DOUBLE_CLICK}`: Double click at the specified (x,y) pixel coordinate on the screen.
+        * `#{Action::MOUSE_TRIPLE_CLICK}`: Triple click at the specified (x,y) pixel coordinate on the screen.
+        * `#{Action::MOUSE_DOWN}`: Press the mouse button at the specified (x,y) pixel coordinate on the screen.
+        * `#{Action::MOUSE_DRAG}`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.
+        * `#{Action::MOUSE_UP}`: Release the mouse button at the specified (x,y) pixel coordinate on the screen.
+        * `#{Action::TYPE}`: Type a string of text on the keyboard.
+        * `#{Action::SCROLL}`: Scroll the screen in a specified direction by a specified amount of clicks of the scroll wheel.
+        * `#{Action::WAIT}`: Wait for a specified duration (in seconds).
+      TEXT
+
+      parameter :coordinate, :object, properties: {
+        x: OmniAI::Schema.integer(description: "The x position in pixels"),
+        y: OmniAI::Schema.integer(description: "The y position in pixels"),
+      }, required: %i[x y], description: <<~TEXT
+        An (x,y) coordinate. Required for the following actions:
+        * `#{Action::MOUSE_MOVE}`
+        * `#{Action::MOUSE_CLICK}`
+        * `#{Action::MOUSE_DOWN}`
+        * `#{Action::MOUSE_DRAG}`
+        * `#{Action::MOUSE_UP}`
+        * `#{Action::MOUSE_DOUBLE_CLICK}`
+        * `#{Action::MOUSE_TRIPLE_CLICK}`
+      TEXT
+
+      parameter :text, :string, description: <<~TEXT
+        The text to type. Required for the following actions:
+        * `#{Action::KEY}`
+        * `#{Action::HOLD_KEY}`
+        * `#{Action::TYPE}`
+      TEXT
+
+      parameter :duration, :integer, description: <<~TEXT
+        A duration in seconds. Required for the following actions:
+        * `#{Action::HOLD_KEY}`
+        * `#{Action::WAIT}`
+      TEXT
+
+      parameter :mouse_button, :string, enum: MOUSE_BUTTON_OPTIONS, description: <<~TEXT
+        The mouse button to use. Required for the following actions:
+        * `#{Action::MOUSE_CLICK}`
+        * `#{Action::MOUSE_DOWN}`
+        * `#{Action::MOUSE_DRAG}`
+        * `#{Action::MOUSE_UP}`
+        * `#{Action::MOUSE_DOUBLE_CLICK}`
+        * `#{Action::MOUSE_TRIPLE_CLICK}`
+      TEXT
+
+      parameter :scroll_direction, :string, enum: SCROLL_DIRECTION_OPTIONS, description: <<~TEXT
+        The direction to scroll. Required for the following actions:
+        * `#{Action::SCROLL}`
+      TEXT
+
+      parameter :scroll_amount, :integer, description: <<~TEXT
+        The amount of clicks to scroll. Required for the following actions:
+        * `#{Action::SCROLL}`
+      TEXT
+
+      required %i[action]
+
+      # @param driver [Computer::Driver]
+      def initialize(driver:, logger: Logger.new(IO::NULL))
+        @driver = driver
+        @logger = logger
+        super()
+      end
+
+      # @param action [String]
+      # @param coordinate [Hash<{ width: Integer, height: Integer }>] the (x,y) coordinate
+      # @param text [String]
+      # @param duration [Integer] the duration in seconds
+      # @param mouse_button [String] e.g. "left", "middle", "right"
+      # @param scroll_direction [String] e.g. "up", "down", "left", "right"
+      # @param scroll_amount [Integer] the amount of clicks to scroll
+      def execute(
+        action:,
+        coordinate: nil,
+        text: nil,
+        duration: nil,
+        mouse_button: nil,
+        scroll_direction: nil,
+        scroll_amount: nil
+      )
+        @logger.info({
+          action:,
+          coordinate:,
+          text:,
+          duration:,
+          mouse_button:,
+          scroll_direction:,
+          scroll_amount:,
+        }.compact.map { |key, value| "#{key}=#{value.inspect}" }.join(" "))
+
+        case action
+        when Action::KEY then @driver.key(text:)
+        when Action::HOLD_KEY then @driver.hold_key(text:, duration:)
+        when Action::MOUSE_POSITION then @driver.mouse_position
+        when Action::MOUSE_MOVE then @driver.mouse_move(coordinate:)
+        when Action::MOUSE_CLICK then @driver.mouse_click(coordinate:, button: mouse_button)
+        when Action::MOUSE_DOUBLE_CLICK then @driver.mouse_double_click(coordinate:, button: mouse_button)
+        when Action::MOUSE_TRIPLE_CLICK then @driver.mouse_triple_click(coordinate:, button: mouse_button)
+        when Action::MOUSE_DOWN then @driver.mouse_down(coordinate:, button: mouse_button)
+        when Action::MOUSE_UP then @driver.mouse_up(coordinate:, button: mouse_button)
+        when Action::MOUSE_DRAG then @driver.mouse_drag(coordinate:, button: mouse_button)
+        when Action::TYPE then @driver.type(text:)
+        when Action::SCROLL then @driver.scroll(amount: scroll_amount, direction: scroll_direction)
+        when Action::WAIT then @driver.wait(duration:)
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/database/base_driver.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Database
+      # Base class for database drivers (e.g. sqlite, postgres, mysql, etc).
+      class BaseDriver
+        # @param statement [String] e.g. "SELECT * FROM people"
+        #
+        # @return [Hash] e.g. { status: :ok, result: [["id", "name"], [1, "John"], [2, "Paul"], ...] }
+        def perform(statement:)
+          raise NotImplementedError, "#{self.class}##{__method__} undefined"
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/database/postgres_driver.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Database
+      # @example
+      #   connection = PG.connect(dbname: "testdb")
+      #   driver = OmniAI::Tools::Database::PostgresDriver.new
+      #   driver.perform(statement: "SELECT * FROM people")
+      class PostgresDriver < BaseDriver
+        # @param connection [Sqlite3::Database]
+        def initialize(connection:)
+          super()
+          @connection = connection
+        end
+
+        # @param statement [String]
+        #
+        # @return [Hash]
+        def perform(statement:)
+          @connection.exec(statement) do |result|
+            { status: :ok, result: [result.fields] + result.values }
+          end
+        rescue ::PG::Error => e
+          { status: :error, message: e.message }
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/database/sqlite_driver.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    module Database
+      # @example
+      #   driver = OmniAI::Tools::Database::SqliteDriver.new
+      #   driver.perform(statement: "SELECT * FROM people")
+      class SqliteDriver < BaseDriver
+        # @param db [Sqlite3::Database]
+        def initialize(db:)
+          super()
+          @db = db
+        end
+
+        # @param statement [String]
+        #
+        # @return [Hash]
+        def perform(statement:)
+          result = @db.execute2(statement)
+
+          { status: :ok, result: }
+        rescue ::SQLite3::Exception => e
+          { status: :error, message: e.message }
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/database_tool.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Tools
+    # @example
+    #   db = Sqlite3::Database.new("./db.sqlite")
+    #   driver = OmniAI::Tools::Database::Sqlite.new(db:)
+    #   tool = OmniAI::Tools::DatabaseTool.new(driver:)
+    #   tool.execute(statements: ["SELECT * FROM people"])
+    class DatabaseTool < OmniAI::Tool
+      description <<~TEXT
+        Executes SQL commands (INSERT / UPDATE / SELECT / etc) on a database.
+
+        Example:
+
+        STATEMENTS:
+
+            [
+              'CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL)',
+              'INSERT INTO people (name) VALUES ('John')',
+              'INSERT INTO people (name) VALUES ('Paul')',
+              'SELECT * FROM people',
+              'DROP TABLE people'
+            ]
+
+        RESULT:
+
+            [
+              {
+                "status": "OK",
+                "statement": "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL)",
+                "result": "..."
+              },
+              {
+                "status": "OK",
+                "statement": "INSERT INTO people (name) VALUES ('John')"
+                "result": "..."
+              },
+              {
+                "status": "OK",
+                "statement": "INSERT INTO people (name) VALUES ('Paul')",
+                "result": "..."
+              },
+              {
+                "status": "OK",
+                "statement": "SELECT * FROM people",
+                "result": "..."
+              },
+              {
+                "status": "OK",
+                "statement": "DROP TABLE people",
+                "result": "..."
+              }
+            ]
+      TEXT
+
+      parameter(
+        :statements,
+        :array,
+        description: "A list of SQL statements to run sequentially.",
+        items: OmniAI::Schema.string(description: 'A SQL statement to run (e.g. "SELECT * FROM ...").')
+      )
+
+      required %i[statements]
+
+      # @param driver [OmniAI::Tools::Database::BaseDriver]
+      # @param logger [IO] An optional logger for debugging executed commands.
+      def initialize(driver:, logger: Logger.new(IO::NULL))
+        super()
+        @driver = driver
+        @logger = logger
+      end
+
+      # @example
+      #   tool = OmniAI::Tools::Database::BaseTool.new
+      #   tool.execute(statements: ["SELECT * FROM people"])
+      #
+      # @param statements [Array<String>]
+      #
+      # @return [Array<Hash>]
+      def execute(statements:)
+        [].tap do |executions|
+          statements.map do |statement|
+            execution = perform(statement:).merge(statement:)
+            executions << execution
+            break unless execution[:status].eql?(:ok)
+          end
+        end
+      end
+
+      def perform(statement:)
+        @logger&.info("#perform statement=#{statement.inspect}")
+
+        @driver.perform(statement:).tap do |result|
+          @logger&.info(JSON.generate(result))
+        end
+      end
+    end
+  end
+end

data/lib/omniai/tools/version.rb

@@ -2,6 +2,6 @@
 
 module OmniAI
   module Tools
-    VERSION = "0.5.0"
+    VERSION = "0.6.0"
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: omniai-tools
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Kevin Sylvestre
 bindir: exe
 cert_chain: []
-date: 2025-05-15 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: omniai
@@ -49,14 +49,40 @@ files:
 - bin/console
 - bin/setup
 - lib/omniai/tools.rb
+- lib/omniai/tools/browser/base_driver.rb
 - lib/omniai/tools/browser/base_tool.rb
 - lib/omniai/tools/browser/button_click_tool.rb
+- lib/omniai/tools/browser/element_click_tool.rb
+- lib/omniai/tools/browser/elements/element_grouper.rb
+- lib/omniai/tools/browser/elements/nearby_element_detector.rb
+- lib/omniai/tools/browser/formatters/action_formatter.rb
+- lib/omniai/tools/browser/formatters/data_entry_formatter.rb
+- lib/omniai/tools/browser/formatters/element_formatter.rb
+- lib/omniai/tools/browser/formatters/input_formatter.rb
 - lib/omniai/tools/browser/inspect_tool.rb
+- lib/omniai/tools/browser/inspect_utils.rb
 - lib/omniai/tools/browser/link_click_tool.rb
+- lib/omniai/tools/browser/page_inspect/button_summarizer.rb
+- lib/omniai/tools/browser/page_inspect/form_summarizer.rb
+- lib/omniai/tools/browser/page_inspect/html_summarizer.rb
+- lib/omniai/tools/browser/page_inspect/link_summarizer.rb
+- lib/omniai/tools/browser/page_inspect_tool.rb
+- lib/omniai/tools/browser/page_screenshot_tool.rb
+- lib/omniai/tools/browser/selector_generator.rb
+- lib/omniai/tools/browser/selector_generator/base_selectors.rb
+- lib/omniai/tools/browser/selector_generator/contextual_selectors.rb
+- lib/omniai/tools/browser/selector_inspect_tool.rb
 - lib/omniai/tools/browser/text_field_area_set_tool.rb
 - lib/omniai/tools/browser/visit_tool.rb
-- lib/omniai/tools/database/base_tool.rb
-- lib/omniai/tools/database/sqlite_tool.rb
+- lib/omniai/tools/browser/watir_driver.rb
+- lib/omniai/tools/browser_tool.rb
+- lib/omniai/tools/computer/base_driver.rb
+- lib/omniai/tools/computer/mac_driver.rb
+- lib/omniai/tools/computer_tool.rb
+- lib/omniai/tools/database/base_driver.rb
+- lib/omniai/tools/database/postgres_driver.rb
+- lib/omniai/tools/database/sqlite_driver.rb
+- lib/omniai/tools/database_tool.rb
 - lib/omniai/tools/disk/base_tool.rb
 - lib/omniai/tools/disk/directory_create_tool.rb
 - lib/omniai/tools/disk/directory_delete_tool.rb
@@ -92,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.3
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A set of tools built for usage with OmniAI.
 test_files: []

data/lib/omniai/tools/database/base_tool.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-require "sqlite3"
-
-module OmniAI
-  module Tools
-    module Database
-      # @example
-      #   tool = OmniAI::Tools::Database::SqliteTool.new
-      #   tool.execute(path: "./foo/bar")
-      class BaseTool < OmniAI::Tool
-        # @param logger [IO] An optional logger for debugging executed commands.
-        def initialize(logger: Logger.new(IO::NULL))
-          super()
-          @logger = logger
-        end
-
-        # @example
-        #   tool = OmniAI::Tools::Database::BaseTool.new
-        #   tool.execute(statements: ["SELECT * FROM people"])
-        #
-        # @param statements [Array<String>]
-        #
-        # @return [Array<Hash>]
-        def execute(statements:)
-          [].tap do |executions|
-            statements.map do |statement|
-              execution = perform(statement:)
-              executions << execution
-              break unless execution[:status].eql?(:ok)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/omniai/tools/database/sqlite_tool.rb

@@ -1,110 +0,0 @@
-# frozen_string_literal: true
-
-require "sqlite3"
-
-module OmniAI
-  module Tools
-    module Database
-      # @example
-      #   tool = OmniAI::Tools::Database::SqliteTool.new
-      #   tool.execute(path: "./foo/bar")
-      class SqliteTool < BaseTool
-        description <<~TEXT
-          Executes SQL commands (INSERT / UPDATE / SELECT / etc) on a database.
-
-          Example:
-
-          STATEMENTS:
-
-              [
-                'CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL)',
-                'INSERT INTO people (name) VALUES ('John')',
-                'INSERT INTO people (name) VALUES ('Paul')',
-                'SELECT * FROM people',
-                'DROP TABLE people'
-              ]
-
-          RESULT:
-
-              [
-                {
-                  "status": "OK",
-                  "statement": "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT NOT NULL)",
-                  "result": "..."
-                },
-                {
-                  "status": "OK",
-                  "statement": "INSERT INTO people (name) VALUES ('John')"
-                  "result": "..."
-                },
-                {
-                  "status": "OK",
-                  "statement": "INSERT INTO people (name) VALUES ('Paul')",
-                  "result": "..."
-                },
-                {
-                  "status": "OK",
-                  "statement": "SELECT * FROM people",
-                  "result": "..."
-                },
-                {
-                  "status": "OK",
-                  "statement": "DROP TABLE people",
-                  "result": "..."
-                }
-              ]
-        TEXT
-
-        parameter(
-          :statements,
-          :array,
-          description: "A list of SQL statements to run sequentially.",
-          items: OmniAI::Schema.string(description: 'A SQL statement to run (e.g. "SELECT * FROM ...").')
-        )
-
-        required %i[statements]
-
-        # @param logger [IO] An optional logger for debugging executed commands.
-        # @param db [SQLite3::Database] A sqlite database.
-        def initialize(db:, logger: Logger.new(IO::NULL))
-          super(logger:)
-          @db = db
-        end
-
-        # @example
-        #   tool = OmniAI::Tools::Database::BaseTool.new
-        #   tool.execute(statements: ["SELECT * FROM people"])
-        #
-        # @param statements [Array<String>]
-        #
-        # @return [Array<Hash>]
-        def execute(statements:)
-          @logger.info("#{self.class.name}#{__method__} statements=#{statements.inspect}")
-
-          [].tap do |executions|
-            statements.map do |statement|
-              execution = perform(statement:)
-              executions << execution
-              break unless execution[:status].eql?(:ok)
-            end
-          end
-        end
-
-      protected
-
-        # @param statement [String]
-        #
-        # @return [Hash]
-        def perform(statement:)
-          result = @db.execute2(statement)
-
-          { status: :ok, statement:, result: }
-        rescue ::SQLite3::Exception => e
-          @logger.warn("ERROR: #{e.message}")
-
-          { status: :error, statement:, result: e.message }
-        end
-      end
-    end
-  end
-end