ruby-utcp 1.1.0

data/lib/utcp/openapi_converter.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module UTCP
+  class OpenAPIConverter
+    HTTP_METHODS = %w[get post put delete patch head options].freeze
+
+    def initialize(spec, spec_url: nil, call_template_name: nil, auth_tools: nil, base_url: nil)
+      @spec = Utils.stringify_keys(Utils.hash!(spec, "OpenAPI document"))
+      @spec_url = spec_url
+      @call_template_name = call_template_name
+      @auth_tools = auth_tools
+      @base_url = base_url
+    end
+
+    def convert
+      paths = @spec["paths"]
+      raise ValidationError, "OpenAPI document must contain a paths object" unless paths.is_a?(Hash)
+
+      tools = []
+      paths.each do |path, path_item|
+        next unless path_item.is_a?(Hash)
+
+        HTTP_METHODS.each do |method|
+          operation = path_item[method]
+          next unless operation.is_a?(Hash)
+
+          tools << convert_operation(path, method, path_item, operation)
+        end
+      end
+
+      Manual.new(
+        utcp_version: VERSION,
+        manual_version: "1.0.0",
+        info: @spec["info"] || {},
+        tools: tools
+      )
+    end
+
+    private
+
+    def convert_operation(path, method, path_item, operation)
+      parameters = Array(path_item["parameters"]) + Array(operation["parameters"])
+      properties = {}
+      required = []
+      header_fields = []
+      body_field = "body"
+
+      parameters.each do |parameter|
+        next unless parameter.is_a?(Hash)
+
+        parameter = Utils.stringify_keys(parameter)
+        name = parameter["name"].to_s
+        next if name.empty?
+
+        if parameter["in"] == "body"
+          properties[body_field] = resolve_schema(parameter["schema"] || {})
+          required << body_field if parameter["required"]
+        else
+          properties[name] = resolve_schema(parameter["schema"] || parameter_schema(parameter))
+          required << name if parameter["required"] || parameter["in"] == "path"
+          header_fields << name if parameter["in"] == "header"
+        end
+      end
+
+      request_body = operation["requestBody"]
+      if request_body.is_a?(Hash)
+        content_type, media = Array(request_body["content"]).first
+        if media.is_a?(Hash)
+          properties[body_field] = resolve_schema(media["schema"] || {})
+          required << body_field if request_body["required"]
+        end
+      else
+        content_type = nil
+      end
+
+      tool_name = operation["operationId"].to_s
+      tool_name = generated_name(method, path) if tool_name.empty?
+      description = operation["description"] || operation["summary"] || "#{method.upcase} #{path}"
+      input_schema = { "type" => "object", "properties" => properties }
+      input_schema["required"] = required.uniq unless required.empty?
+
+      Tool.new(
+        name: tool_name,
+        description: description.to_s,
+        inputs: input_schema,
+        outputs: response_schema(operation),
+        tags: Array(operation["tags"]),
+        tool_call_template: HttpCallTemplate.new(
+          name: tool_name,
+          url: join_url(resolve_base_url, path),
+          http_method: method.upcase,
+          content_type: content_type || Array(operation["consumes"]).first || "application/json",
+          body_field: body_field,
+          header_fields: header_fields,
+          auth: operation_requires_auth?(operation) ? @auth_tools : nil
+        )
+      )
+    end
+
+    def parameter_schema(parameter)
+      parameter.each_with_object({}) do |(key, value), schema|
+        schema[key] = value if %w[type format enum default items minimum maximum pattern minLength maxLength].include?(key)
+      end
+    end
+
+    def resolve_schema(schema, seen = [])
+      data = Utils.stringify_keys(schema || {})
+      reference = data["$ref"]
+      return data unless reference&.start_with?("#/")
+      return {} if seen.include?(reference)
+
+      resolved = reference.sub(%r{\A#/}, "").split("/").reduce(@spec) do |node, component|
+        break nil unless node.is_a?(Hash)
+
+        node[component.gsub("~1", "/").gsub("~0", "~")]
+      end
+      resolved ? resolve_schema(resolved, seen + [reference]) : data
+    end
+
+    def response_schema(operation)
+      responses = operation["responses"]
+      return {} unless responses.is_a?(Hash)
+
+      _status, response = responses.find { |status, _value| status.to_s.match?(/\A2\d\d\z/) } || responses.first
+      return {} unless response.is_a?(Hash)
+
+      if response["content"].is_a?(Hash)
+        media = response["content"].values.first
+        resolve_schema(media.is_a?(Hash) ? media["schema"] : {})
+      else
+        resolve_schema(response["schema"] || {})
+      end
+    end
+
+    def operation_requires_auth?(operation)
+      security = operation.key?("security") ? operation["security"] : @spec["security"]
+      security.is_a?(Array) && !security.empty?
+    end
+
+    def resolve_base_url
+      return @base_url if @base_url && !@base_url.empty?
+
+      server = Array(@spec["servers"]).first
+      return substitute_server_variables(server) if server.is_a?(Hash) && server["url"]
+
+      if @spec["host"]
+        scheme = Array(@spec["schemes"]).first || "https"
+        return "#{scheme}://#{@spec['host']}#{@spec['basePath']}"
+      end
+
+      if @spec_url&.match?(/\Ahttps?:/)
+        uri = URI.parse(@spec_url)
+        return "#{uri.scheme}://#{uri.host}#{uri.port && ![80, 443].include?(uri.port) ? ":#{uri.port}" : ""}"
+      end
+
+      raise ValidationError, "OpenAPI document does not define a server URL; pass base_url"
+    end
+
+    def substitute_server_variables(server)
+      variables = Utils.stringify_keys(server["variables"] || {})
+      server["url"].gsub(/\{([^}]+)\}/) do
+        variable = variables[Regexp.last_match(1)]
+        variable.is_a?(Hash) ? variable["default"].to_s : Regexp.last_match(0)
+      end
+    end
+
+    def join_url(base, path)
+      "#{base.to_s.sub(%r{/+\z}, '')}/#{path.to_s.sub(%r{\A/+}, '')}"
+    end
+
+    def generated_name(method, path)
+      "#{method}_#{path}".gsub(/[^A-Za-z0-9_]+/, "_").gsub(/\A_+|_+\z/, "")
+    end
+  end
+  OpenApiConverter = OpenAPIConverter
+end
+

data/lib/utcp/protocols/base.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "json"
+
+module UTCP
+  class CommunicationProtocol
+    def register_manual(_client, _manual_call_template)
+      raise NotImplementedError
+    end
+
+    def deregister_manual(_client, _manual_call_template)
+      nil
+    end
+
+    def call_tool(_client, _tool_name, _tool_args, _tool_call_template)
+      raise NotImplementedError
+    end
+
+    def call_tool_streaming(client, tool_name, tool_args, tool_call_template)
+      return enum_for(__method__, client, tool_name, tool_args, tool_call_template) unless block_given?
+
+      yield call_tool(client, tool_name, tool_args, tool_call_template)
+    end
+
+    private
+
+    def manual_from_payload(template, payload, source: "protocol response")
+      data = if payload.is_a?(String)
+               Utils.parse_document(payload, source: source)
+             else
+               Utils.stringify_keys(payload)
+             end
+      data = { "tools" => data } if data.is_a?(Array)
+      data = Migration.manual_v0_1_to_v1_1(data) if Migration.v0_1_manual?(data)
+      data = Utils.hash!(data, source)
+      data["utcp_version"] ||= VERSION
+      data["manual_version"] ||= "1.0.0"
+      data["tools"] = Utils.array!(data.fetch("tools", []), "#{source}.tools").map do |raw_tool|
+        tool = Utils.stringify_keys(Utils.hash!(raw_tool, "#{source}.tool"))
+        tool["tool_call_template"] ||= tool.delete("tool_provider") || template.to_h
+        tool
+      end
+      Manual.from_h(data)
+    end
+
+    def decode_json_or_text(value)
+      return value unless value.is_a?(String)
+
+      stripped = value.strip
+      return value if stripped.empty? || !stripped.start_with?("{", "[", '"') && stripped !~ /\A(?:true|false|null|-?\d)/
+
+      JSON.parse(stripped)
+    rescue JSON::ParserError
+      value
+    end
+
+    def substitute_message_template(value, arguments)
+      args = Utils.stringify_keys(arguments || {})
+      case value
+      when Hash
+        value.each_with_object({}) do |(key, item), result|
+          result[key.to_s] = substitute_message_template(item, args)
+        end
+      when Array
+        value.map { |item| substitute_message_template(item, args) }
+      when String
+        value.gsub(/UTCP_ARG_([A-Za-z0-9_]+)_UTCP_ARG/) do
+          name = Regexp.last_match(1)
+          raise ToolCallError, "Missing required transport argument: #{name}" unless args.key?(name)
+
+          replacement = args[name]
+          replacement.is_a?(String) ? replacement : JSON.generate(replacement)
+        end
+      else
+        value
+      end
+    end
+
+    def success(template, manual)
+      RegisterManualResult.new(
+        manual_call_template: template,
+        manual: manual,
+        success: true,
+        errors: []
+      )
+    end
+
+    def failure(template, error)
+      RegisterManualResult.new(
+        manual_call_template: template,
+        manual: Manual.new(tools: [], manual_version: "0.0.0"),
+        success: false,
+        errors: [error.is_a?(Exception) ? error.message : error.to_s]
+      )
+    end
+  end
+end

data/lib/utcp/protocols/cli.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+require "timeout"
+
+module UTCP
+  class CLIProtocol < CommunicationProtocol
+    PLACEHOLDER = /UTCP_ARG_([A-Za-z0-9_]+)_UTCP_END/.freeze
+    DEFAULT_ENVIRONMENT = %w[
+      PATH HOME LANG LANGUAGE LC_ALL LC_CTYPE TMPDIR TEMP TMP SHELL USER LOGNAME
+    ].freeze
+
+    def register_manual(client, template)
+      assert_template!(template)
+      output = execute(client, template, {})
+      data = Utils.parse_document(output, source: "CLI output")
+      data = { "utcp_version" => VERSION, "manual_version" => "1.0.0", "tools" => data } if data.is_a?(Array)
+      data = Migration.manual_v0_1_to_v1_1(data) if Migration.v0_1_manual?(data)
+      success(template, Manual.from_h(data))
+    rescue StandardError => error
+      client.logger.warn("Unable to register CLI manual #{template.name.inspect}: #{error.message}")
+      failure(template, error)
+    end
+
+    def call_tool(client, tool_name, tool_args, template)
+      assert_template!(template)
+      output = execute(client, template, tool_args || {})
+      stripped = output.strip
+      if stripped.start_with?("{", "[")
+        JSON.parse(stripped)
+      else
+        stripped
+      end
+    rescue JSON::ParserError
+      output.strip
+    rescue Error
+      raise
+    rescue StandardError => error
+      raise ToolCallError.new("CLI tool #{tool_name.inspect} failed: #{error.message}", tool_name: tool_name)
+    end
+
+    private
+
+    def assert_template!(template)
+      return if template.is_a?(CliCallTemplate)
+
+      raise ValidationError, "CLI protocol requires a CliCallTemplate"
+    end
+
+    def execute(client, template, arguments)
+      env = environment_for(template)
+      commands = template.commands.each_with_index.map do |step, index|
+        command, argument_env = interpolate(step.command, arguments)
+        env.merge!(argument_env)
+        [command, include_output?(step, index, template.commands.length)]
+      end
+      script = build_script(commands)
+      working_dir = template.working_dir ? File.expand_path(template.working_dir, client.root_dir) : client.root_dir
+      raise ToolCallError, "CLI working directory does not exist: #{working_dir}" unless File.directory?(working_dir)
+
+      run_script(script, env, working_dir, template.timeout)
+    end
+
+    def environment_for(template)
+      names = template.inherit_env_vars.nil? ? DEFAULT_ENVIRONMENT : template.inherit_env_vars
+      environment = names.each_with_object({}) do |name, selected|
+        selected[name] = ENV[name] if ENV.key?(name)
+      end
+      template.env_vars.each { |name, value| environment[name] = value.to_s }
+      environment
+    end
+
+    def interpolate(command, arguments)
+      args = Utils.stringify_keys(arguments)
+      env = {}
+      result = +""
+      quote = nil
+      escaped = false
+      index = 0
+      variable_numbers = {}
+
+      while index < command.length
+        match = PLACEHOLDER.match(command, index)
+        if match && match.begin(0) == index
+          name = match[1]
+          raise ToolCallError, "Missing required CLI argument: #{name}" unless args.key?(name)
+
+          number = variable_numbers.fetch(name) { variable_numbers[name] = variable_numbers.length }
+          variable = "UTCP_TOOL_ARG_#{number}"
+          value = args[name]
+          env[variable] = value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : value.to_s
+          result << case quote
+                    when "'" then %Q('"${#{variable}}"')
+                    when '"' then "${#{variable}}"
+                    else %Q("${#{variable}}")
+                    end
+          index = match.end(0)
+          next
+        end
+
+        character = command[index]
+        result << character
+        if escaped
+          escaped = false
+        elsif character == "\\" && quote != "'"
+          escaped = true
+        elsif character == "'" && quote != '"'
+          quote = quote == "'" ? nil : "'"
+        elsif character == '"' && quote != "'"
+          quote = quote == '"' ? nil : '"'
+        end
+        index += 1
+      end
+      [result, env]
+    end
+
+    def include_output?(step, index, length)
+      step.append_to_final_output.nil? ? index == length - 1 : step.append_to_final_output
+    end
+
+    def build_script(commands)
+      lines = []
+      commands.each_with_index do |(command, append), index|
+        lines << "__utcp_output_#{index}=$( { #{command}; } 2>&1 )"
+        lines << "__utcp_status_#{index}=$?"
+        lines << "CMD_#{index}_OUTPUT=$__utcp_output_#{index}"
+        lines << "export CMD_#{index}_OUTPUT"
+        lines << "printf '%s\\n' \"$__utcp_output_#{index}\"" if append
+        lines << "if [ \"$__utcp_status_#{index}\" -ne 0 ]; then printf '%s\\n' \"$__utcp_output_#{index}\" >&2; exit \"$__utcp_status_#{index}\"; fi"
+      end
+      lines.join("\n")
+    end
+
+    def run_script(script, environment, working_dir, timeout_seconds)
+      stdout_text = nil
+      stderr_text = nil
+      status = nil
+      wait_thread = nil
+
+      Open3.popen3(
+        environment,
+        "/bin/sh", "-c", script,
+        unsetenv_others: true,
+        chdir: working_dir,
+        pgroup: true
+      ) do |stdin, stdout, stderr, thread|
+        wait_thread = thread
+        stdin.close
+        stdout_reader = Thread.new { stdout.read }
+        stderr_reader = Thread.new { stderr.read }
+        begin
+          Timeout.timeout(timeout_seconds) do
+            stdout_text = stdout_reader.value
+            stderr_text = stderr_reader.value
+            status = thread.value
+          end
+        rescue Timeout::Error
+          terminate_process_group(thread.pid)
+          stdout_reader.kill
+          stderr_reader.kill
+          raise TimeoutError, "CLI command timed out after #{timeout_seconds} seconds"
+        end
+      end
+
+      unless status&.success?
+        detail = stderr_text.to_s.strip
+        detail = stdout_text.to_s.strip if detail.empty?
+        raise ToolCallError.new("CLI command exited with status #{status&.exitstatus}: #{detail}", status: status&.exitstatus)
+      end
+      stdout_text.to_s
+    ensure
+      terminate_process_group(wait_thread.pid) if wait_thread&.alive?
+    end
+
+    def terminate_process_group(pid)
+      Process.kill("TERM", -pid)
+      sleep(0.05)
+      Process.kill("KILL", -pid)
+    rescue Errno::ESRCH, Errno::EPERM
+      nil
+    end
+  end
+  CliCommunicationProtocol = CLIProtocol
+end
+

data/lib/utcp/protocols/file.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module UTCP
+  class FileProtocol < CommunicationProtocol
+    def register_manual(client, template)
+      assert_template!(template)
+      path = resolve_path(client, template.file_path)
+      data = Utils.load_document(path)
+      manual = if openapi?(data)
+                 OpenAPIConverter.new(
+                   data,
+                   spec_url: Pathname.new(path).expand_path.to_s,
+                   call_template_name: template.name,
+                   auth_tools: template.auth_tools
+                 ).convert
+               else
+                 Manual.from_h(data)
+               end
+      success(template, manual)
+    rescue StandardError => error
+      client.logger.warn("Unable to register file manual #{template.name.inspect}: #{error.message}")
+      failure(template, error)
+    end
+
+    def call_tool(client, _tool_name, _tool_args, template)
+      assert_template!(template)
+      File.read(resolve_path(client, template.file_path), mode: "r:bom|utf-8")
+    rescue Errno::ENOENT, Errno::EACCES => error
+      raise ToolCallError, "Unable to read tool file: #{error.message}"
+    end
+
+    private
+
+    def assert_template!(template)
+      return if template.is_a?(FileCallTemplate)
+
+      raise ValidationError, "file protocol requires a FileCallTemplate"
+    end
+
+    def resolve_path(client, path)
+      File.expand_path(path, client.root_dir)
+    end
+
+    def openapi?(data)
+      data.is_a?(Hash) && (data.key?("openapi") || data.key?("swagger") || data.key?("paths"))
+    end
+  end
+  FileCommunicationProtocol = FileProtocol
+end
+