omniai-anthropic 1.9.2 → 1.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55c1ba530498bc6cfe194b42c12402ad79d3d9f9c13fa3b7922fecbfa1a8d1c8
-  data.tar.gz: edc2e433bbf5107177ddd6d90df9f3d856e209316a10ecfa4deeaec85c72378c
+  metadata.gz: fe02992338864e110012ee2d463d0922deb06d0e536b246546eae3b3cfd9d5aa
+  data.tar.gz: 3ad82e6af98e100ac3186120a5622c87184f759dc669bea4f26a99489b116411
 SHA512:
-  metadata.gz: d9e23e858ef9276bfb2ab2d63836170d6e9c21972a4d8fca7be09620d9585b325615168e06d771cf6609320ea6c9471a6145d674f1c997fe074bb02d9cdfcc7c
-  data.tar.gz: 4da1388834893087d8405e3aeede1130c096b84acba445b6e08f10209d6df51157f63e32bb571e6a611b9db591e9ea5d676136d1fb0f287ed6362802a9e49fc6
+  metadata.gz: 180cea8404f6740edbd1c219e21089a40604e5cfdef051e0dfe1f6ba0b624f55bab73d69c8629e22a17e16b6e0008f32b6498694cedb525defd0bc70c6810ccf
+  data.tar.gz: c955b172f69deb6afd9d03d215e1ac1aa2d60407d8d8b214bbe381085c55af5659f64a82dab20aa95be4a3254c49abdabf87e2a90ca004d3a9598c32dec37ecc

data/README.md

@@ -104,3 +104,19 @@ JSON.parse(completion.text) # { "name": "Ringo" }
 ```
 
 [Anthropic API Reference `control-output-format`](https://docs.anthropic.com/en/docs/control-output-format)
+
+### Computers
+
+```bash
+sudo apt-get install convert # screenshots
+sudo apt-get install scrot # screenshots
+sudo apt-get install xdotool # mouse / keyboard
+```
+
+```ruby
+computer = OmniAI::Anthropic::Computer.new
+
+completion = client.chat(tools: [computer]) do |prompt|
+  prompt.user('Please signup for reddit')
+end
+```

data/lib/omniai/anthropic/client.rb

@@ -25,12 +25,14 @@ module OmniAI
       # @param api_key [String] optional - defaults to `OmniAI::Anthropic.config.api_key`
       # @param host [String] optional - defaults to `OmniAI::Anthropic.config.host`
       # @param version [String] optional - defaults to `OmniAI::Anthropic.config.version`
+      # @param beta [String] optional - defaults to `OmniAI::Anthropic.config.beta`
       # @param logger [Logger] optional - defaults to `OmniAI::Anthropic.config.logger`
       # @param timeout [Integer] optional - defaults to `OmniAI::Anthropic.config.timeout`
       def initialize(
         api_key: OmniAI::Anthropic.config.api_key,
         host: OmniAI::Anthropic.config.host,
         version: OmniAI::Anthropic.config.version,
+        beta: OmniAI::Anthropic.config.beta,
         logger: OmniAI::Anthropic.config.logger,
         timeout: OmniAI::Anthropic.config.timeout
       )
@@ -40,6 +42,7 @@ module OmniAI
 
         @host = host
         @version = version
+        @beta = beta
       end
 
       # @return [HTTP::Client]
@@ -47,6 +50,7 @@ module OmniAI
         @connection ||= super.headers({
           'x-api-key': @api_key,
           'anthropic-version': @version,
+          'anthropic-beta': @beta,
         }.compact)
       end
 

data/lib/omniai/anthropic/computer.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module OmniAI
+  module Anthropic
+    # A reference implementation of an OmniAI computer tool using xdotool for mouse / keyboard:
+    # https://docs.anthropic.com/en/docs/build-with-claude/computer-use#computer-tool
+    #
+    # Usage:
+    #
+    #   computer = OmniAI::Anthropic::Computer.new()
+    class Computer
+      TYPE = 'computer_20241022'
+
+      SCREENSHOT_DELAY = 2.0 # seconds
+      TYPING_DELAY = 20 # milliseconds
+
+      module Action
+        KEY = 'key'
+        TYPE = 'type'
+        CURSOR_POSITION = 'cursor_position'
+        MOUSE_MOVE = 'mouse_move'
+        LEFT_CLICK = 'left_click'
+        RIGHT_CLICK = 'right_click'
+        MIDDLE_CLICK = 'middle_click'
+        LEFT_CLICK_DRAG = 'left_click_drag'
+        RIGHT_CLICK_DRAG = 'right_click_drag'
+        MIDDLE_CLICK_DRAG = 'middle_click_drag'
+        DOUBLE_CLICK = 'double_click'
+        SCREENSHOT = 'screenshot'
+      end
+
+      module Button
+        LEFT = 1
+        MIDDLE = 2
+        RIGHT = 3
+      end
+
+      # @param name [String] optional
+      # @param display_width_px [Integer]
+      # @param display_height_px [Integer]
+      # @param display_number [Integer] optional
+      def initialize(display_width_px:, display_height_px:, display_number: 1, name: 'computer')
+        @name = name
+        @display_width_px = display_width_px
+        @display_height_px = display_height_px
+        @display_number = display_number
+      end
+
+      # @example
+      #   tool.serialize # =>
+      #   # {
+      #   #  "type": "computer_20241022",
+      #   #  "name": "computer",
+      #   #  "display_width_px": 1024,
+      #   #  "display_height_px": 768,
+      #   #  "display_number": 1,
+      #   # }
+      #
+      # @return [Hash]
+      def serialize(*)
+        {
+          type: TYPE,
+          name: @name,
+          display_width_px: @display_width_px,
+          display_height_px: @display_height_px,
+          display_number: @display_number,
+        }
+      end
+
+      # @example
+      #   computer.call({ "action" => 'type', "text" => 'Hello' })
+      #
+      # @param args [Hash]
+      # @return [String]
+      def call(args = {})
+        perform(
+          action: args['action'],
+          text: args['text'],
+          coordinate: args['coordinate']
+        )
+      end
+
+      # @param action [String]
+      # @param coordinate [Array] [x, y] optional
+      # @param text [String] optional
+      #
+      # @return [Array<Hash>]
+      def perform(action:, text: nil, coordinate: nil) # rubocop:disable Metrics/CyclomaticComplexity
+        case action
+        when Action::KEY then key(text: text)
+        when Action::TYPE then type(text: text)
+        when Action::CURSOR_POSITION then mouse_location
+        when Action::LEFT_CLICK then click(button: Button::LEFT)
+        when Action::MIDDLE_CLICK then click(button: Button::MIDDLE)
+        when Action::RIGHT_CLICK then click(button: Button::RIGHT)
+        when Action::LEFT_CLICK_DRAG then mouse_down_move_up(coordinate:, button: Button::LEFT)
+        when Action::MIDDLE_CLICK_DRAG then mouse_down_move_up(coordinate:, button: Button::MIDDLE)
+        when Action::RIGHT_CLICK_DRAG then mouse_down_move_up(coordinate:, button: Button::RIGHT)
+        when Action::MOUSE_MOVE then mouse_move(coordinate:)
+        when Action::DOUBLE_CLICK then double_click(button: Button::LEFT)
+        when Action::SCREENSHOT then screenshot
+        end
+      end
+
+      # @param cmd [String]
+      #
+      # @return [String]
+      def shell(cmd, ...)
+        stdout, stderr, status = Open3.capture3(cmd, ...)
+
+        "stdout=#{stdout.inspect} stderr=#{stderr.inspect} status=#{status}"
+      end
+
+      # @param cmd [String]
+      #
+      # @return [String]
+      def xdotool(...)
+        shell('xdotool', ...)
+      end
+
+      # @param button [Integer]
+      #
+      # @return [String]
+      def click(button:)
+        xdotool('click', button)
+      end
+
+      # @param button [Integer]
+      #
+      # @return [String]
+      def double_click(button:)
+        xdotool('click', button, '--repeat', 2)
+      end
+
+      # @param coordinate [Array] [x, y]
+      #
+      # @return [String]
+      def mouse_move(coordinate:)
+        x, y = coordinate
+        xdotool('mousemove', '--sync', x, y)
+      end
+
+      # @param coordinate [Array] [x, y]
+      # @param button [Integer]
+      #
+      # @return [String]
+      def mouse_down_move_up(coordinate:, button:)
+        x, y = coordinate
+        xdotool('mousedown', button, 'mousemove', '--sync', x, y, 'mouseup', button)
+      end
+
+      # @return [String]
+      def mouse_location
+        xdotool('getmouselocation')
+      end
+
+      # @param text [String]
+      # @param delay [Integer] milliseconds
+      #
+      # @return [String]
+      def type(text:, delay: TYPING_DELAY)
+        xdotool('type', '--delay', delay, '--', text)
+      end
+
+      # @param text [String]
+      #
+      # @return [String]
+      def key(text:)
+        xdotool('key', '--', text)
+      end
+
+      # @return [Hash]
+      def screenshot
+        tempfile = Tempfile.new(['screenshot', '.png'])
+        Kernel.system('gnome-screenshot', '-w', '-f', tempfile.path)
+        tempfile.rewind
+        data = Base64.encode64(tempfile.read)
+
+        { type: 'base64', media_type: 'image/png', data: data }
+      ensure
+        tempfile.close
+        tempfile.unlink
+      end
+    end
+  end
+end

data/lib/omniai/anthropic/config.rb

@@ -11,20 +11,27 @@ module OmniAI
       #   @return [String, nil] passed as `anthropic-version` if specified
       attr_accessor :version
 
+      # @!attribute [rw] beta
+      #   @return [String, nil] passed as `anthropic-beta` if specified
+      attr_accessor :beta
+
       # @param api_key [String, nil] optional - defaults to `ENV['ANTHROPIC_API_KEY']`
       # @param host [String, nil] optional - defaults to `ENV['ANTHROPIC_HOST'] w/ fallback to `DEFAULT_HOST`
       # @param version [String, nil] optional - defaults to `ENV['ANTHROPIC_VERSION'] w/ fallback to `DEFAULT_VERSION`
+      # @param beta [String, nil] optional - defaults to `ENV['ANTHROPIC_BETA']`
       # @param logger [Logger, nil] optional - defaults to
       # @param timeout [Integer, Hash, nil] optional
       def initialize(
         api_key: ENV.fetch('ANTHROPIC_API_KEY', nil),
         host: ENV.fetch('ANTHROPIC_HOST', DEFAULT_HOST),
         version: ENV.fetch('ANTHROPIC_VERSION', DEFAULT_VERSION),
+        beta: ENV.fetch('ANTHROPIC_BETA', nil),
         logger: nil,
         timeout: nil
       )
         super(api_key:, host:, logger:, timeout:)
         @version = version
+        @beta = beta
         @chat_options[:max_tokens] = 4096
       end
     end

data/lib/omniai/anthropic/version.rb

@@ -2,6 +2,6 @@
 
 module OmniAI
   module Anthropic
-    VERSION = '1.9.2'
+    VERSION = '1.9.4'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omniai-anthropic
 version: !ruby/object:Gem::Version
-  version: 1.9.2
+  version: 1.9.4
 platform: ruby
 authors:
 - Kevin Sylvestre
@@ -75,6 +75,7 @@ files:
 - lib/omniai/anthropic/chat/tool_call_serializer.rb
 - lib/omniai/anthropic/chat/tool_serializer.rb
 - lib/omniai/anthropic/client.rb
+- lib/omniai/anthropic/computer.rb
 - lib/omniai/anthropic/config.rb
 - lib/omniai/anthropic/version.rb
 homepage: https://github.com/ksylvest/omniai-anthropic