omniai-anthropic 1.9.2 → 1.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55c1ba530498bc6cfe194b42c12402ad79d3d9f9c13fa3b7922fecbfa1a8d1c8
-  data.tar.gz: edc2e433bbf5107177ddd6d90df9f3d856e209316a10ecfa4deeaec85c72378c
+  metadata.gz: 3b3723aa11fbf3b8def1f87dee64a9bc9a171e7d2e8e3292354c5476c24d4233
+  data.tar.gz: 667d9b3ce3e9fb6cd4061372358b555def837325f3107404bf0a83fbd10a9af4
 SHA512:
-  metadata.gz: d9e23e858ef9276bfb2ab2d63836170d6e9c21972a4d8fca7be09620d9585b325615168e06d771cf6609320ea6c9471a6145d674f1c997fe074bb02d9cdfcc7c
-  data.tar.gz: 4da1388834893087d8405e3aeede1130c096b84acba445b6e08f10209d6df51157f63e32bb571e6a611b9db591e9ea5d676136d1fb0f287ed6362802a9e49fc6
+  metadata.gz: e448d21a9d4d615b2679015bd8ada729b23a407a1707d956baff61cb1fcc629e26cfaf7752b18f6dc52abed4b1a22cea884d95b2c423391be8e5ca5329b4ea5d
+  data.tar.gz: c68070843dc6c36ffb8639873eee20d61382c44b035f41ca147ac458fecb58709a52f06d3bad476fb91d835315eeed47c8f7dfb0af10566ed5e65566f033690a

data/README.md

@@ -104,3 +104,19 @@ JSON.parse(completion.text) # { "name": "Ringo" }
 ```
 
 [Anthropic API Reference `control-output-format`](https://docs.anthropic.com/en/docs/control-output-format)
+
+### Computers
+
+```bash
+sudo apt-get install convert # screenshots
+sudo apt-get install scrot # screenshots
+sudo apt-get install xdotool # mouse / keyboard
+```
+
+```ruby
+computer = OmniAI::Anthropic::Computer.new
+
+completion = client.chat(tools: [computer]) do |prompt|
+  prompt.user('Please signup for reddit')
+end
+```

data/lib/omniai/anthropic/computer.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module OmniAI
+  module Anthropic
+    # A reference implementation of an OmniAI computer tool using xdotool for mouse / keyboard:
+    # https://docs.anthropic.com/en/docs/build-with-claude/computer-use#computer-tool
+    #
+    # Usage:
+    #
+    #   computer = OmniAI::Anthropic::Computer.new()
+    class Computer
+      TYPE = 'computer_20241022'
+
+      SCREENSHOT_DELAY = 2.0 # seconds
+      TYPING_DELAY = 20 # milliseconds
+
+      module Action
+        KEY = 'key'
+        TYPE = 'type'
+        CURSOR_POSITION = 'cursor_position'
+        MOUSE_MOVE = 'mouse_move'
+        LEFT_CLICK = 'left_click'
+        RIGHT_CLICK = 'right_click'
+        MIDDLE_CLICK = 'middle_click'
+        LEFT_CLICK_DRAG = 'left_click_drag'
+        RIGHT_CLICK_DRAG = 'right_click_drag'
+        MIDDLE_CLICK_DRAG = 'middle_click_drag'
+        DOUBLE_CLICK = 'double_click'
+        SCREENSHOT = 'screenshot'
+      end
+
+      module Button
+        LEFT = 1
+        MIDDLE = 2
+        RIGHT = 3
+      end
+
+      # @param name [String] optional
+      # @param display_width_px [Integer]
+      # @param display_height_px [Integer]
+      # @param display_number [Integer] optional
+      def initialize(display_width_px:, display_height_px:, display_number: 1, name: 'computer')
+        @name = name
+        @display_width_px = display_width_px
+        @display_height_px = display_height_px
+        @display_number = display_number
+      end
+
+      # @example
+      #   tool.serialize # =>
+      #   # {
+      #   #  "type": "computer_20241022",
+      #   #  "name": "computer",
+      #   #  "display_width_px": 1024,
+      #   #  "display_height_px": 768,
+      #   #  "display_number": 1,
+      #   # }
+      #
+      # @return [Hash]
+      def serialize(*)
+        {
+          type: TYPE,
+          name: @name,
+          display_width_px: @display_width_px,
+          display_height_px: @display_height_px,
+          display_number: @display_number,
+        }
+      end
+
+      # @example
+      #   computer.call({ "action" => 'type', "text" => 'Hello' })
+      #
+      # @param args [Hash]
+      # @return [String]
+      def call(args = {})
+        perform(
+          action: args['action'],
+          text: args['text'],
+          coordinate: args['coordinate']
+        )
+      end
+
+      # @param action [String]
+      # @param coordinate [Array] [x, y] optional
+      # @param text [String] optional
+      #
+      # @return [Array<Hash>]
+      def perform(action:, text: nil, coordinate: nil) # rubocop:disable Metrics/CyclomaticComplexity
+        case action
+        when Action::KEY then key(text: text)
+        when Action::TYPE then type(text: text)
+        when Action::CURSOR_POSITION then mouse_location
+        when Action::LEFT_CLICK then click(button: Button::LEFT)
+        when Action::MIDDLE_CLICK then click(button: Button::MIDDLE)
+        when Action::RIGHT_CLICK then click(button: Button::RIGHT)
+        when Action::LEFT_CLICK_DRAG then mouse_down_move_up(coordinate:, button: Button::LEFT)
+        when Action::MIDDLE_CLICK_DRAG then mouse_down_move_up(coordinate:, button: Button::MIDDLE)
+        when Action::RIGHT_CLICK_DRAG then mouse_down_move_up(coordinate:, button: Button::RIGHT)
+        when Action::MOUSE_MOVE then mouse_move(coordinate:)
+        when Action::DOUBLE_CLICK then double_click(button: Button::LEFT)
+        when Action::SCREENSHOT then screenshot
+        end
+      end
+
+      # @param cmd [String]
+      #
+      # @return [String]
+      def shell(cmd, ...)
+        stdout, stderr, status = Open3.capture3(cmd, ...)
+
+        "stdout=#{stdout.inspect} stderr=#{stderr.inspect} status=#{status}"
+      end
+
+      # @param cmd [String]
+      #
+      # @return [String]
+      def xdotool(...)
+        shell('xdotool', ...)
+      end
+
+      # @param button [Integer]
+      #
+      # @return [String]
+      def click(button:)
+        xdotool('click', button)
+      end
+
+      # @param button [Integer]
+      #
+      # @return [String]
+      def double_click(button:)
+        xdotool('click', button, '--repeat', 2)
+      end
+
+      # @param coordinate [Array] [x, y]
+      #
+      # @return [String]
+      def mouse_move(coordinate:)
+        x, y = coordinate
+        xdotool('mousemove', '--sync', x, y)
+      end
+
+      # @param coordinate [Array] [x, y]
+      # @param button [Integer]
+      #
+      # @return [String]
+      def mouse_down_move_up(coordinate:, button:)
+        x, y = coordinate
+        xdotool('mousedown', button, 'mousemove', '--sync', x, y, 'mouseup', button)
+      end
+
+      # @return [String]
+      def mouse_location
+        xdotool('getmouselocation')
+      end
+
+      # @param text [String]
+      # @param delay [Integer] milliseconds
+      #
+      # @return [String]
+      def type(text:, delay: TYPING_DELAY)
+        xdotool('type', '--delay', delay, '--', text)
+      end
+
+      # @param text [String]
+      #
+      # @return [String]
+      def key(text:)
+        xdotool('key', '--', text)
+      end
+
+      # @return [Hash]
+      def screenshot
+        tempfile = Tempfile.new(['screenshot', '.png'])
+        Kernel.system('gnome-screenshot', '-w', '-f', tempfile.path)
+        tempfile.rewind
+        data = Base64.encode64(tempfile.read)
+
+        { type: 'base64', media_type: 'image/png', data: data }
+      ensure
+        tempfile.close
+        tempfile.unlink
+      end
+    end
+  end
+end

data/lib/omniai/anthropic/version.rb

@@ -2,6 +2,6 @@
 
 module OmniAI
   module Anthropic
-    VERSION = '1.9.2'
+    VERSION = '1.9.3'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omniai-anthropic
 version: !ruby/object:Gem::Version
-  version: 1.9.2
+  version: 1.9.3
 platform: ruby
 authors:
 - Kevin Sylvestre
@@ -75,6 +75,7 @@ files:
 - lib/omniai/anthropic/chat/tool_call_serializer.rb
 - lib/omniai/anthropic/chat/tool_serializer.rb
 - lib/omniai/anthropic/client.rb
+- lib/omniai/anthropic/computer.rb
 - lib/omniai/anthropic/config.rb
 - lib/omniai/anthropic/version.rb
 homepage: https://github.com/ksylvest/omniai-anthropic