charai 0.1.0

data/lib/charai/input_tool.rb

@@ -0,0 +1,393 @@
+module Charai
+  # Hub class for performing actions on the browser
+  # Mainly used by AI.
+  class InputTool
+    # callback
+    #  - on_assertion_ok(description)
+    #  - on_assertion_fail(description)
+    #  - on_action_start(action, params)
+    def initialize(browsing_context, callback: nil)
+      @browsing_context = browsing_context
+      @callback = callback
+    end
+
+    def on_send_message(&block)
+      @message_sender = block
+    end
+
+    def assertion_ok(description)
+      trigger_callback(:on_assertion_ok, description)
+    end
+
+    def assertion_fail(description)
+      trigger_callback(:on_assertion_fail, description)
+
+      if defined?(RSpec::Expectations)
+        RSpec::Expectations.fail_with(description)
+      elsif defined?(MiniTest::Assertion)
+        raise MiniTest::Assertion, description
+      else
+        raise description
+      end
+    end
+
+    def capture_screenshot
+      trigger_callback(:on_action_start, 'capture_screenshot', {})
+
+      current_url = @browsing_context.url
+      @browsing_context.capture_screenshot(format: 'png').tap do |binary|
+        if @message_sender
+          message = Agent::Message.new(
+            text: "Capture of #{current_url}",
+            images: [
+              { png: Base64.strict_encode64(binary) },
+            ],
+          )
+          @message_sender.call(message)
+        end
+      end
+    end
+
+    def click(x:, y:, delay: 50)
+      trigger_callback(:on_action_start, 'click', { x: x, y: y, delay: delay })
+
+      @browsing_context.perform_mouse_actions do |q|
+        q.pointer_move(x: x.to_i, y: y.to_i)
+        q.pointer_down(button: 0)
+        q.pause(duration: delay)
+        q.pointer_up(button: 0)
+      end
+    end
+
+    def execute_script(script)
+      trigger_callback(:on_action_start, 'execute_script', { script: script })
+
+      begin
+        result = @browsing_context.default_realm.script_evaluate(script)
+      rescue BrowsingContext::Realm::ScriptEvaluationError => e
+        result = e.message
+      end
+
+      notify_to_sender(result) unless "#{result}" == ''
+
+      result
+    end
+
+    def on_pressing_key(key, &block)
+      trigger_callback(:on_action_start, 'key_down', { key: key })
+
+      value = convert_key(key)
+      @browsing_context.perform_keyboard_actions do |q|
+        q.key_down(value: value)
+      end
+
+      begin
+        block.call
+      ensure
+        trigger_callback(:on_action_start, 'key_up', { key: key })
+
+        @browsing_context.perform_keyboard_actions do |q|
+          q.key_up(value: value)
+        end
+      end
+    end
+
+    def press_key(key, delay: 50)
+      trigger_callback(:on_action_start, 'press_key', { key: key, delay: delay })
+
+      value = convert_key(key)
+      @browsing_context.perform_keyboard_actions do |q|
+        q.key_down(value: value)
+        q.pause(duration: delay)
+        q.key_up(value: value)
+      end
+    end
+
+    def sleep_seconds(seconds)
+      trigger_callback(:on_action_start, 'sleep_seconds', { seconds: seconds })
+
+      sleep seconds
+    end
+
+    def type_text(text, delay: 50)
+      trigger_callback(:on_action_start, 'type_text', { text: text, delay: delay })
+
+      text.each_char do |c|
+        @browsing_context.perform_keyboard_actions do |q|
+          q.key_down(value: c)
+          q.pause(duration: delay / 2)
+          q.key_up(value: c)
+          q.pause(duration: delay - delay / 2)
+        end
+      end
+    end
+
+    # velocity:
+    #  500 - weak
+    # 1000 - normal
+    # 2000 - strong
+    def scroll_down(x: 0, y: 0, velocity: 1000)
+      raise ArgumentError, 'velocity must be positive' if velocity <= 0
+      trigger_callback(:on_action_start, 'scroll_down', { x: x, y: y, velocity: velocity })
+
+      @browsing_context.perform_mouse_wheel_actions do |q|
+        deceleration = SplineDeceleration.new(velocity)
+        loop do
+          delta_y = deceleration.calc
+          break if delta_y.zero?
+          q.scroll(x: x, y: y, delta_y: delta_y, duration: 16)
+        end
+      end
+    end
+
+    # velocity:
+    #  500 - weak
+    # 1000 - normal
+    # 2000 - strong
+    def scroll_up(x: 0, y: 0, velocity: 1000)
+      raise ArgumentError, 'velocity must be positive' if velocity <= 0
+      trigger_callback(:on_action_start, 'scroll_up', { x: x, y: y, velocity: velocity })
+
+      @browsing_context.perform_mouse_wheel_actions do |q|
+        deceleration = SplineDeceleration.new(velocity)
+        loop do
+          delta_y = -deceleration.calc
+          break if delta_y.zero?
+          q.scroll(x: x, y: y, delta_y: delta_y, duration: 16)
+        end
+      end
+    end
+
+    private
+
+    def notify_to_sender(message)
+      if @message_sender
+        message = Agent::Message.new(text: "result is `#{message}`", images: [])
+        @message_sender.call(message)
+      end
+    end
+
+    def trigger_callback(method_name, ...)
+      if @callback.respond_to?(method_name)
+        @callback.public_send(method_name, ...)
+      elsif @callback.is_a?(Hash) && @callback[method_name].is_a?(Proc)
+        @callback[method_name].call(...)
+      end
+    end
+
+    # ref: https://github.com/puppeteer/puppeteer/blob/puppeteer-v23.5.3/packages/puppeteer-core/src/bidi/Input.ts#L52
+    # Converted using ChatGPT 4o
+    def convert_key(key)
+      return key if key.length == 1
+
+      case key
+      when 'Cancel'
+        "\uE001"
+      when 'Help'
+        "\uE002"
+      when 'Backspace'
+        "\uE003"
+      when 'Tab'
+        "\uE004"
+      when 'Clear'
+        "\uE005"
+      when 'Enter'
+        "\uE007"
+      when 'Shift', 'ShiftLeft'
+        "\uE008"
+      when 'Control', 'ControlLeft', 'Ctrl'
+        "\uE009"
+      when 'ControlOrMeta', 'CtrlOrMeta'
+        Charai::Util.macos? ? "\uE03D" : "\uE009"
+      when 'Alt', 'AltLeft'
+        "\uE00A"
+      when 'Pause'
+        "\uE00B"
+      when 'Escape'
+        "\uE00C"
+      when 'PageUp'
+        "\uE00E"
+      when 'PageDown'
+        "\uE00F"
+      when 'End'
+        "\uE010"
+      when 'Home'
+        "\uE011"
+      when 'ArrowLeft'
+        "\uE012"
+      when 'ArrowUp'
+        "\uE013"
+      when 'ArrowRight'
+        "\uE014"
+      when 'ArrowDown'
+        "\uE015"
+      when 'Insert'
+        "\uE016"
+      when 'Delete'
+        "\uE017"
+      when 'NumpadEqual'
+        "\uE019"
+      when 'Numpad0'
+        "\uE01A"
+      when 'Numpad1'
+        "\uE01B"
+      when 'Numpad2'
+        "\uE01C"
+      when 'Numpad3'
+        "\uE01D"
+      when 'Numpad4'
+        "\uE01E"
+      when 'Numpad5'
+        "\uE01F"
+      when 'Numpad6'
+        "\uE020"
+      when 'Numpad7'
+        "\uE021"
+      when 'Numpad8'
+        "\uE022"
+      when 'Numpad9'
+        "\uE023"
+      when 'NumpadMultiply'
+        "\uE024"
+      when 'NumpadAdd'
+        "\uE025"
+      when 'NumpadSubtract'
+        "\uE027"
+      when 'NumpadDecimal'
+        "\uE028"
+      when 'NumpadDivide'
+        "\uE029"
+      when 'F1'
+        "\uE031"
+      when 'F2'
+        "\uE032"
+      when 'F3'
+        "\uE033"
+      when 'F4'
+        "\uE034"
+      when 'F5'
+        "\uE035"
+      when 'F6'
+        "\uE036"
+      when 'F7'
+        "\uE037"
+      when 'F8'
+        "\uE038"
+      when 'F9'
+        "\uE039"
+      when 'F10'
+        "\uE03A"
+      when 'F11'
+        "\uE03B"
+      when 'F12'
+        "\uE03C"
+      when 'Meta', 'MetaLeft'
+        "\uE03D"
+      when 'ShiftRight'
+        "\uE050"
+      when 'ControlRight'
+        "\uE051"
+      when 'AltRight'
+        "\uE052"
+      when 'MetaRight'
+        "\uE053"
+      when 'Digit0'
+        '0'
+      when 'Digit1'
+        '1'
+      when 'Digit2'
+        '2'
+      when 'Digit3'
+        '3'
+      when 'Digit4'
+        '4'
+      when 'Digit5'
+        '5'
+      when 'Digit6'
+        '6'
+      when 'Digit7'
+        '7'
+      when 'Digit8'
+        '8'
+      when 'Digit9'
+        '9'
+      when 'KeyA'
+        'a'
+      when 'KeyB'
+        'b'
+      when 'KeyC'
+        'c'
+      when 'KeyD'
+        'd'
+      when 'KeyE'
+        'e'
+      when 'KeyF'
+        'f'
+      when 'KeyG'
+        'g'
+      when 'KeyH'
+        'h'
+      when 'KeyI'
+        'i'
+      when 'KeyJ'
+        'j'
+      when 'KeyK'
+        'k'
+      when 'KeyL'
+        'l'
+      when 'KeyM'
+        'm'
+      when 'KeyN'
+        'n'
+      when 'KeyO'
+        'o'
+      when 'KeyP'
+        'p'
+      when 'KeyQ'
+        'q'
+      when 'KeyR'
+        'r'
+      when 'KeyS'
+        's'
+      when 'KeyT'
+        't'
+      when 'KeyU'
+        'u'
+      when 'KeyV'
+        'v'
+      when 'KeyW'
+        'w'
+      when 'KeyX'
+        'x'
+      when 'KeyY'
+        'y'
+      when 'KeyZ'
+        'z'
+      when 'Semicolon'
+        ';'
+      when 'Equal'
+        '='
+      when 'Comma'
+        ','
+      when 'Minus'
+        '-'
+      when 'Period'
+        '.'
+      when 'Slash'
+        '/'
+      when 'Backquote'
+        '`'
+      when 'BracketLeft'
+        '['
+      when 'Backslash'
+        '\\'
+      when 'BracketRight'
+        ']'
+      when 'Quote'
+        '"'
+      else
+        raise ArgumentError, "Unknown key: \"#{key}\""
+      end
+    end
+  end
+end

data/lib/charai/openai_chat.rb

@@ -0,0 +1,163 @@
+require 'base64'
+require 'json'
+require 'net/http'
+require 'uri'
+
+module Charai
+  class OpenaiChat
+    # callback
+    #  - on_chat_start(introduction)
+    #  - on_chat_question(content: Array|String)
+    #  - on_chat_answer(answer_text)
+    #  - on_chat_conversation(content, answer_text)
+    def initialize(configuration, introduction: nil, callback: nil)
+      @configuration = configuration
+      @introduction = introduction
+      @callback = callback
+      @mutex = Mutex.new
+      clear
+    end
+
+    def clear
+      trigger_callback(:on_chat_start, @introduction)
+
+      @messages = []
+      if @introduction
+        @messages << { role: 'system', content: @introduction }
+      end
+    end
+
+    # .push('How are you?')
+    # .push('How many people is here?', images: [ { jpg: 'xXxXxxxxxxxxx' }, { png: 'xXxXxxxxxxxxx' } ])
+    def push(question, images: [])
+      content = build_question(question, images)
+      message = {
+        role: 'user',
+        content: content,
+      }
+
+      @mutex.synchronize do
+        trigger_callback(:on_chat_question, content)
+        fetch_openai(message).tap do |answer|
+          trigger_callback(:on_chat_answer, answer)
+          trigger_callback(:on_chat_conversation, content, answer)
+
+          @messages << message
+          @messages << { role: 'assistant', content: answer }
+        end
+      end
+    end
+
+    def pop
+      @mutex.synchronize do
+        @messages.pop
+        @messages.pop[:content]
+      end
+    end
+
+    private
+
+    def build_question(question, images)
+      if images.empty?
+        question
+      else
+        [
+          {
+            type: 'text',
+            text: question,
+          },
+          *(images.map { |image| build_image_payload(image) }),
+        ]
+      end
+    end
+
+    def build_image_payload(image)
+      if image.is_a?(String) && image.start_with?('http')
+        return build_image_payload(fetch_image_url(image))
+      end
+
+      raise ArgumentError, "image must be a Hash, but got #{image.class}" unless image.is_a?(Hash)
+      raise ArgumentError, "image must have only one key, but got #{image.keys}" unless image.keys.size == 1
+      type = image.keys.first
+      raise ArgumentError, "image key must be one of [:jpg, :jpeg, :png], but got #{type}" unless %i[jpg jpeg png].include?(type)
+
+      b64 = image[type]
+
+      {
+        type: 'image_url',
+        image_url: {
+          url: "data:image/#{type};base64,#{b64}",
+        },
+      }
+    end
+
+    def fetch_image_url(url)
+      uri = URI.parse(url)
+
+      response = Net::HTTP.get_response(uri)
+
+      if response.is_a?(Net::HTTPSuccess)
+        image_data = response.body
+        mime_type = response['content-type']
+
+        base64_image = Base64.strict_encode64(image_data)
+
+        case mime_type
+        when 'image/png'
+          { png: base64_image }
+        when 'image/jpeg'
+          { jpeg: base64_image }
+        else
+          raise "Unsupported image type: #{mime_type}"
+        end
+      else
+        raise "Failed to fetch image: #{response.code} #{response.message}"
+      end
+    end
+
+    def fetch_openai(message)
+      uri = URI(@configuration.endpoint_url)
+
+      response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https', read_timeout: 120) do |http|
+        http.post(
+          uri,
+          @configuration.decorate_body({
+            messages: with_message_history(message),
+          }).to_json,
+          @configuration.add_auth_header({
+            'Content-Type' => 'application/json',
+            'Accept' => 'application/json',
+          }),
+        )
+      end
+      if response.is_a?(Net::HTTPSuccess)
+        body = JSON.parse(response.body)
+        body.dig('choices', 0, 'message', 'content')
+      else
+        raise "Failed to fetch OpenAI: #{response.code} #{response.message}"
+      end
+    end
+
+    def with_message_history(new_message, omit_images_except_last: 3)
+      Enumerator.new do |out|
+        len = @messages.size
+        @messages.each_with_index do |message, i|
+          if i < len - omit_images_except_last && message[:content].is_a?(Array)
+            out << message.merge(content: message[:content].find { |c| c[:type] == 'text'}[:text])
+          else
+            out << message
+          end
+        end
+        out << new_message
+      end.to_a
+    end
+
+    def trigger_callback(method_name, ...)
+      if @callback.respond_to?(method_name)
+        @callback.public_send(method_name, ...)
+      elsif @callback.is_a?(Hash) && @callback[method_name].is_a?(Proc)
+        @callback[method_name].call(...)
+      end
+    end
+  end
+end

data/lib/charai/openai_configuration.rb

@@ -0,0 +1,58 @@
+module Charai
+  class OpenaiConfiguration
+    def initialize(model:, api_key:)
+      @endpoint_url = 'https://api.openai.com/v1/chat/completions'
+      @model = model
+      @api_key = api_key
+    end
+
+    attr_reader :endpoint_url
+
+    def add_auth_header(headers)
+      headers['Authorization'] = "Bearer #{@api_key}"
+      headers
+    end
+
+    def decorate_body(payload)
+      payload[:model] = @model
+      payload
+    end
+  end
+
+  class AzureOpenaiConfiguration
+    def initialize(endpoint_url:, api_key:)
+      @endpoint_url = endpoint_url
+      @api_key = api_key
+    end
+
+    attr_reader :endpoint_url
+
+    def add_auth_header(headers)
+      headers['api-key'] = @api_key
+      headers
+    end
+
+    def decorate_body(payload)
+      payload
+    end
+  end
+
+  class OllamaConfiguration
+    def initialize(endpoint_url:, model:)
+      @endpoint_url = endpoint_url
+      @model = model
+    end
+
+    attr_reader :endpoint_url
+
+    def add_auth_header(headers)
+      # auth header is not required.
+      headers
+    end
+
+    def decorate_body(payload)
+      payload[:model] = @model
+      payload
+    end
+  end
+end