charai 0.1.0 → 0.2.0.beta2

data/lib/charai/injected_script.rb

@@ -0,0 +1,40 @@
+module Charai
+  class InjectedScript
+    def initialize(realm, handle)
+      @realm = realm
+      @handle = handle
+    end
+
+    def e(value)
+      @realm.script_evaluate(value, as_handle: false)
+    end
+
+    def h(value)
+      @realm.script_evaluate(value, as_handle: true)
+    end
+
+    def getprop(name, as_handle: false)
+      c(
+        function_declaration: "(injected) => injected.#{name}",
+        as_handle: as_handle,
+      )
+    end
+
+    def call(name, *args, as_handle: false)
+      c(
+        function_declaration: "(injected, ...args) => injected.#{name}(...args)",
+        args: args,
+        as_handle: as_handle,
+      )
+    end
+
+    private
+
+    def c(function_declaration:, args: [], as_handle: false)
+      @realm.script_call_function(
+        function_declaration,
+        arguments: [@handle, *args],
+        as_handle: as_handle)
+    end
+  end
+end

data/lib/charai/input_tool.rb

@@ -31,11 +31,37 @@ module Charai
       end
     end
 
+    def aria_snapshot(root_locator: 'document.body', ref: false)
+      trigger_callback(:on_action_start, 'aria_snapshot', { root_locator: root_locator, ref: ref })
+
+      current_url = @browsing_context.url
+      snapshot = @browsing_context.default_realm._with_injected_script do |script|
+        if ref
+          body_handle = script.getprop("document.body", as_handle: true)
+          result = script.call(:ariaSnapshot, body_handle, { mode: 'ai' })
+          result.split("\n")
+        else
+          not_found = script.e("!#{root_locator}")
+          if not_found
+            raise ArgumentError, "Element not found: #{root_locator}"
+          end
+          result = script.call(:ariaSnapshot, script.h(root_locator), { mode: 'autoexpect' })
+          result.split("\n")
+        end
+      end
+
+      if @message_sender
+        message = Agent::Message.new(text: "ARIA snapshot of #{current_url}\n\n#{snapshot}")
+        @message_sender.call(message)
+      end
+    end
+
     def capture_screenshot
       trigger_callback(:on_action_start, 'capture_screenshot', {})
 
       current_url = @browsing_context.url
-      @browsing_context.capture_screenshot(format: 'png').tap do |binary|
+          binding.irb
+      @browsing_context.capture_screenshot(format: { type: 'png' }).tap do |binary|
         if @message_sender
           message = Agent::Message.new(
             text: "Capture of #{current_url}",
@@ -73,6 +99,23 @@ module Charai
       result
     end
 
+    def execute_script_with_ref(ref, element_function_declaration)
+      resolved = resolve_selector_for_ref(ref)
+      trigger_callback(:on_action_start, 'execute_script_with_ref', { script: element_function_declaration, ref: ref, selector: resolved[:selector] })
+
+      begin
+        result = @browsing_context.default_realm.script_call_function(
+          element_function_declaration,
+          arguments: [resolved[:element_handle]])
+      rescue BrowsingContext::Realm::ScriptEvaluationError => e
+        result = e.message
+      end
+
+      notify_to_sender(result) unless "#{result}" == ''
+
+      result
+    end
+
     def on_pressing_key(key, &block)
       trigger_callback(:on_action_start, 'key_down', { key: key })
 
@@ -175,6 +218,23 @@ module Charai
       end
     end
 
+    def resolve_selector_for_ref(ref)
+      @browsing_context.default_realm._with_injected_script do |script|
+        parsed = script.call(:parseSelector, "aria-ref=#{ref}")
+
+        # ensure ref is attached.
+        body_handle = script.getprop("document.body", as_handle: true)
+        script.call(:ariaSnapshot, body_handle, { mode: 'ai' })
+
+        document_handle = script.getprop('document', as_handle: true)
+        element_handle = script.call(:querySelector, parsed, document_handle, false, as_handle: true)
+
+        selector = script.call(:generateSelectorSimple, element_handle, { omitInternalEngines: true })
+
+        { selector: selector, element_handle: element_handle }
+      end
+    end
+
     # ref: https://github.com/puppeteer/puppeteer/blob/puppeteer-v23.5.3/packages/puppeteer-core/src/bidi/Input.ts#L52
     # Converted using ChatGPT 4o
     def convert_key(key)

data/lib/charai/openai_configuration.rb

@@ -19,6 +19,26 @@ module Charai
     end
   end
 
+  class GeminiOpenaiConfiguration
+    def initialize(model:, api_key:)
+      @endpoint_url = 'https://generativelanguage.googleapis.com/v1beta/openai/'
+      @model = model
+      @api_key = api_key
+    end
+
+    attr_reader :endpoint_url
+
+    def add_auth_header(headers)
+      headers['Authorization'] = "Bearer #{@api_key}"
+      headers
+    end
+
+    def decorate_body(payload)
+      payload[:model] = @model
+      payload
+    end
+  end
+
   class AzureOpenaiConfiguration
     def initialize(endpoint_url:, api_key:)
       @endpoint_url = endpoint_url

data/lib/charai/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Charai
-  VERSION = "0.1.0"
+  VERSION = "0.2.0.beta2"
 end

data/lib/charai.rb

@@ -10,6 +10,7 @@ require 'charai/browser_launcher'
 require 'charai/browser_process'
 require 'charai/browsing_context'
 require 'charai/driver'
+require 'charai/injected_script'
 require 'charai/input_tool'
 require 'charai/openai_chat'
 require 'charai/openai_configuration'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: charai
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0.beta2
 platform: ruby
 authors:
 - YusukeIwaki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-25 00:00:00.000000000 Z
+date: 2025-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -75,6 +75,8 @@ files:
 - lib/charai/browser_process.rb
 - lib/charai/browsing_context.rb
 - lib/charai/driver.rb
+- lib/charai/injectedScriptSource.js
+- lib/charai/injected_script.rb
 - lib/charai/input_tool.rb
 - lib/charai/openai_chat.rb
 - lib/charai/openai_configuration.rb