charai 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a2bb9857e550fece14adf8d4b0ff13d3f1bd5006be2cad4a047e03a23e6192ef
+  data.tar.gz: bed2120f92dc260ea88d6f38bcef45f533705f2b027239e4e21068d17ed1e59d
+SHA512:
+  metadata.gz: 295c7c1f728df8c6daa02814c0a00a21c02c9b4182e361688a38a4e011b90368b98f78efcce5eb0baf996862ee4081c2c1a5b6d2b7359e948dad0d71ba1fadc0
+  data.tar.gz: 85660fa979d2f8476022a625eed86c2b870f1d6983c194a0f846bafbfa3a9c27fefd558bbde1bcf522f4bc1cf183782bd09e4ba67797198691e9cbdac4c13bea

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,13 @@
+AllCops:
+  TargetRubyVersion: 3.2
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120

data/Gemfile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem 'allure-rspec'
+gem "pry-byebug"
+gem "puma"
+gem "rake"
+gem "rspec"
+gem "rubocop"
+gem "sinatra", require: false

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 YusukeIwaki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,169 @@
+# Charai
+
+Chat + Ruby + AI = Charai
+
+## Setup
+
+Add `gem 'charai'` into your project's Gemfile, and then `bundle install`
+
+Also, this gem requires **Firefox Developer Edition** to be installed on the location below:
+
+- /Applications/Firefox Developer Edition.app (macOS)
+- /usr/bin/firefox-devedition (Linux)
+
+## Configuration
+
+Configure your Capybara driver like below.
+
+```
+Capybara.register_driver :charai do |app|
+  Charai::Driver.new(app, openai_configuration: config)
+end
+
+Capybara.register_driver :charai_headless do |app|
+  Charai::Driver.new(app, openai_configuration: config, headless: true)
+end
+```
+
+Please note that this driver required OpenAI service.
+
+### OpenAI
+
+```
+config = Charai::OpenaiConfiguration.new(
+  model: 'gpt-4o',
+  api_key: 'sk-xxxxxxxxxxx'
+)
+```
+
+### Azure OpenAI (Recommended)
+
+```
+config = Charai::AzureOpenaiConfiguration.new(
+  endpoint_url: 'https://YOUR-APP.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-05-01-preview',
+  api_key: 'aabbcc00112233445566'
+)
+```
+
+## Usage
+
+Since this driver works with the OpenAI service, we can easily describe E2E test like below :)
+
+```ruby
+before do
+  Capybara.current_driver    = :charai
+  Capybara.javascript_driver = :charai
+
+  page.driver.additional_instruction = <<~MARKDOWN
+  * このページは、3ペイン構造です。ユーザが仕事を探すためのページです。
+  * 左ペインには仕事の絞り込みができるフィルター、中央ペインが仕事（求人）の一覧で、30件ずつ表示されます。
+  * 左ペインにマウスを置いてスクロールしても、中央ペインはスクロールされません。一覧をスクロールしたいときには、中央ペインの座標を確認し、その中央にマウスを置いてスクロールしてください。
+  * 右ペインは、広告エリアです。検索条件に応じた広告が表示されます。
+  MARKDOWN
+end
+
+it 'should work' do
+  page.driver << <<~MARKDOWN
+  * 仕事の一覧が表示されたら、左ペインでサーバーサイドエンジニアで「Ruby on Rails」の仕事に絞り込みをしてください。
+  * 左ペインで絞り込んだら、中央ペインにRuby on Railsに関する仕事が表示されていることを確認してください。
+  * Ruby on Railsに関係のない仕事が、検索結果件数の半分以上あるばあいには、このテスト「検索結果不適合」として失敗としてください。
+  MARKDOWN
+end
+```
+
+### Report for long-running E2E tests
+
+We often trigger E2E test during the night since it costs a lot of time. It is really boring to sit down in front of the PC during E2E testing.
+
+This driver provides an extension (callback) feature for reporting.
+
+First, let's prepare a report formatter like this.
+
+```ruby
+class HtmlReport
+  def initialize
+    @content = []
+  end
+
+  def start(introduction)
+    @content << <<~HTML
+    <html>
+    <head><link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/mini.css/3.0.1/mini-default.min.css"></head>
+    <body>
+    <pre style="margin: 100px 75px; border-left: 0px;">#{introduction}</pre>
+    HTML
+  end
+
+  def add_conversation(content, answer)
+    if content.is_a?(Array)
+      text = content.find { |c| c[:type] == 'text' }[:text]
+
+      @content << <<~HTML
+      <div style="margin: 40px 25px" class="card fluid">
+      <h4 style="border-left: .25rem solid var(--pre-color);">user</h4>
+      <pre>#{text}</pre>
+      HTML
+
+      content.each do |c|
+        next unless c[:type] == 'image_url'
+
+        @content << <<~HTML
+        <img src="#{c[:image_url][:url]}" width="480" />
+        HTML
+      end
+
+      @content << <<~HTML
+      <h4 style="text-align: end; border-right: .25rem solid var(--pre-color);">assistant</h4>
+      <pre style="border-left: 0px; border-right: .25rem solid var(--pre-color);">#{answer}</pre>
+      </div>
+      HTML
+    else
+      @content << <<~HTML
+      <div style="margin: 40px 25px" class="card fluid">
+      <h4 style="border-left: .25rem solid var(--pre-color);">user</h4>
+      <pre>#{content}</pre>
+
+      <h4 style="text-align: end; border-right: .25rem solid var(--pre-color);">assistant</h4>
+      <pre style="border-left: 0px; border-right: .25rem solid var(--pre-color);">#{answer}</pre>
+      </div>
+      HTML
+    end
+  end
+
+  def to_html
+    @content << "</body></html>"
+    @content.join("\n")
+  end
+end
+```
+
+and then configure a callback for recording test-results into the report. It would be a good choice to use Allure reports's attachment feature for this.
+
+```ruby
+config.around(:each, type: :feature) do |example|
+  report = HtmlReport.new
+  Capybara.current_session.driver.callback = {
+    on_chat_start: -> (introduction) {
+      report.start(introduction)
+    },
+    on_chat_conversation: ->(content_hash, answer) {
+      puts answer
+      report.add_conversation(content_hash, answer)
+    },
+  }
+
+  example.run
+
+  Allure.add_attachment(
+    name: "Chat Report",
+    source: report.to_html,
+    type: 'text/html',
+  )
+end
+```
+
+With this report, we can check evidences for each test and investigate failed tests (postmotem).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "charai"
+
+require "pry"
+Pry.start

data/charai.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "lib/charai/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "charai"
+  spec.version = Charai::VERSION
+  spec.authors = ["YusukeIwaki"]
+  spec.email = ["q7w8e9w8q7w8e9@yahoo.co.jp"]
+
+  spec.summary = "charai(Chat + Ruby + AI) Capybara driver"
+  spec.description = "Prototype impl for Kaigi on Rails 2024 presentation."
+  spec.homepage = "https://github.com/YusukeIwaki/charai"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.2.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/}) || f.include?(".git")
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'capybara'
+  spec.add_dependency "concurrent-ruby", ">= 1.1.6"
+  spec.add_dependency "websocket-driver", ">= 0.6.1"
+end

data/lib/charai/action_queue.rb

@@ -0,0 +1,41 @@
+module Charai
+  class ActionQueue
+    def initialize
+      @actions = []
+    end
+
+    def to_a
+      @actions
+    end
+
+    def pause(duration:)
+      @actions << { type: 'pause', duration: duration }
+    end
+
+    def key_down(value:)
+      @actions << { type: 'keyDown', value: value }
+    end
+
+    def key_up(value:)
+      @actions << { type: 'keyUp', value: value }
+    end
+
+    # button: 0 for left, 1 for middle, 2 for right
+    def pointer_down(button:)
+      @actions << { type: 'pointerDown', button: button }
+    end
+
+    def pointer_move(x:, y:, duration: nil)
+      @actions << { type: 'pointerMove', x: x, y: y, duration: duration }.compact
+    end
+
+    # button: 0 for left, 1 for middle, 2 for right
+    def pointer_up(button:)
+      @actions << { type: 'pointerUp', button: button }
+    end
+
+    def scroll(x:, y:, delta_x: 0, delta_y: 0, duration: nil)
+      @actions << { type: 'scroll', x: x, y: y, deltaX: delta_x, deltaY: delta_y, duration: duration }.compact
+    end
+  end
+end

data/lib/charai/agent.rb

@@ -0,0 +1,97 @@
+module Charai
+  class Agent
+    class Message < Data.define(:text, :images)
+      def initialize(text:, images: [])
+        super(text: text, images: images)
+      end
+    end
+
+
+    def initialize(input_tool:, openai_chat:)
+      input_tool.on_send_message do |message|
+        send_message_to_openai_chat(message)
+      end
+      @sandbox = Sandbox.new(input_tool)
+      @openai_chat = openai_chat
+    end
+
+    def <<(text)
+      message = Message.new(text: text)
+      send_message_to_openai_chat(message)
+    end
+
+    attr_reader :last_message
+
+    private
+
+    def send_message_to_openai_chat(message)
+      if @should_use_message_queue
+        @message_queue << message
+      else
+        answer = @openai_chat.push(message.text, images: message.images)
+        @last_message = answer
+        handle_message_from_openai_chat(answer)
+      end
+    end
+
+    def with_aggregating_failures(&block)
+      if defined?(RSpec::Expectations)
+        label = nil
+        metadata = {}
+        RSpec::Expectations::FailureAggregator.new(label, metadata).aggregate(&block)
+      else
+        block.call
+      end
+    end
+
+    def with_message_queuing(&block)
+      @should_use_message_queue = true
+      @message_queue = []
+
+      begin
+        block.call
+      ensure
+        @should_use_message_queue = false
+        # Handle only the last message
+        if (message = @message_queue.last)
+          send_message_to_openai_chat(message)
+        end
+      end
+    end
+
+    class HandleMessageError < StandardError ; end
+
+    def handle_message_from_openai_chat(answer)
+      with_message_queuing do
+        with_aggregating_failures do
+          begin
+            answer.scan(/```[a-zA-Z]*\n(.*?)\n```/m).map(&:first).each do |code|
+              if code.include?('`') # Avoid OS shell execution.
+                raise HandleMessageError, "It is not allowed to use backquote"
+              end
+              @sandbox.instance_eval(code)
+            end
+          rescue HandleMessageError => e
+            send_message_to_openai_chat(Message.new(text: e.message))
+          rescue Browser::Error => e
+            send_message_to_openai_chat(Message.new(text: "Error: #{e.message}"))
+          rescue => e
+            send_message_to_openai_chat(Message.new(text: "ERROR: #{e.message}"))
+          end
+        end
+      end
+    end
+
+    class Sandbox
+      def initialize(input_tool)
+        @input_tool = input_tool
+      end
+
+      private
+
+      def driver
+        @input_tool
+      end
+    end
+  end
+end

data/lib/charai/browser.rb

@@ -0,0 +1,142 @@
+module Charai
+  class Browser
+    def self.launch(...)
+      BrowserLauncher.new.launch(...)
+    end
+
+    def initialize(web_socket, debug_protocol: false)
+      @web_socket = web_socket
+      @debug_protocol = debug_protocol
+      @browsing_contexts = {}
+
+      web_socket.on_message do |message|
+        handle_received_message_from_websocket(JSON.parse(message))
+      end
+
+      bidi_call_async('session.new', {
+        capabilities: {
+          alwaysMatch: {
+            acceptInsecureCerts: false,
+            webSocketUrl: true,
+          },
+        },
+      }).value!
+      sync_browsing_contexts
+      bidi_call_async('session.subscribe', {
+        events: %w[browsingContext],
+      }) # do no await
+    end
+
+    def create_browsing_context
+      result = bidi_call_async('browsingContext.create', { type: :tab, userContext: :default }).value!
+      browsing_context_id = result['context']
+      @browsing_contexts[browsing_context_id] ||= BrowsingContext.new(self, browsing_context_id)
+    end
+
+    def close
+      bidi_call_async('browser.close').value!
+      @web_socket.close
+    end
+
+    def bidi_call_async(method_, params = {})
+      with_message_id do |message_id|
+        @message_results[message_id] = Concurrent::Promises.resolvable_future
+
+        send_message_to_websocket({
+          id: message_id,
+          method: method_,
+          params: params,
+        })
+
+        @message_results[message_id]
+      end
+    end
+
+    private
+
+    def with_message_id(&block)
+      unless @message_id
+        @message_id = 1
+        @message_results = {}
+      end
+
+      message_id = @message_id
+      @message_id += 1
+      block.call(message_id)
+    end
+
+    def send_message_to_websocket(payload)
+      debug_print_send(payload)
+      message = JSON.generate(payload)
+      @web_socket.send_text(message)
+    end
+
+    def handle_received_message_from_websocket(payload)
+      debug_print_recv(payload)
+
+      if payload['id']
+        if promise = @message_results.delete(payload['id'])
+          case payload['type']
+          when 'success'
+            promise.fulfill(payload['result'])
+          when 'error'
+            promise.reject(Error.new("#{payload['error']}: #{payload['message']}\n#{payload['stacktrace']}"))
+          end
+        end
+      elsif payload['type'] == 'event'
+        handle_received_event(payload['method'], payload['params'])
+      end
+    end
+
+    class Error < StandardError ; end
+
+    def handle_received_event(method_, params)
+      case method_
+      when 'browsingContext.contextCreated'
+        browsing_context_id = params['context']
+        return unless browsing_context_id
+        return unless browsing_context_id.split("-").count == 5
+        @browsing_contexts[browsing_context_id] ||= BrowsingContext.new(self, browsing_context_id)
+        if params['url']
+          @browsing_contexts[browsing_context_id]._update_url(params['url'])
+        end
+      when 'browsingContext.contextDestroyed'
+        browsing_context_id = params['context']
+        @browsing_contexts.delete(browsing_context_id)
+      when 'browsingContext.domContentLoaded'
+        browsing_context_id = params['context']
+        if params['url']
+          @browsing_contexts[browsing_context_id]&._update_url(params['url'])
+        end
+      end
+    end
+
+    def sync_browsing_contexts
+      result = bidi_call_async('browsingContext.getTree').value!
+      browsing_contexts = result['contexts']
+
+      extra_context_ids = @browsing_contexts.keys - browsing_contexts.map { |context| context['context'] }
+      extra_context_ids.each do |context_id|
+        @browsing_contexts.delete(context_id)
+      end
+      result['contexts'].each do |context|
+        @browsing_contexts[context['context']] ||= BrowsingContext.new(self, context)
+        if context['url']
+          @browsing_contexts[context['context']]._update_url(context['url'])
+        end
+      end
+    end
+
+    def debug_print_send(hash)
+      return unless @debug_protocol
+
+      puts "SEND > #{hash}"
+    end
+
+    def debug_print_recv(hash)
+      return unless @debug_protocol
+
+      puts "RECV < #{hash}"
+    end
+  end
+end