ruby-utcp 1.1.0

data/lib/utcp/client.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module UTCP
+  class Client
+    attr_reader :config, :root_dir, :variable_substitutor, :logger, :registration_results
+
+    def self.create(root_dir: nil, config: nil, logger: nil)
+      client = new(root_dir: root_dir, config: config, logger: logger)
+      client.register_configured_manuals
+      client
+    end
+
+    def initialize(root_dir: nil, config: nil, logger: nil)
+      @root_dir = File.expand_path(root_dir || Dir.pwd)
+      @logger = logger || Logger.new($stderr, level: Logger::WARN)
+      @config = ClientConfig.from(config, root_dir: @root_dir)
+      @variable_substitutor = VariableSubstitutor.new
+      @registration_results = []
+    end
+
+    def register_configured_manuals
+      @registration_results = register_manuals(config.manual_call_templates)
+      self
+    end
+
+    def register_manual(manual_call_template)
+      template = copy_template(CallTemplate.from_h(manual_call_template))
+      template.name = sanitize_name(template.name)
+      if config.tool_repository.get_manual(template.name)
+        raise ManualAlreadyRegisteredError,
+              "Manual #{template.name.inspect} is already registered; deregister it or use another name"
+      end
+
+      template = substitute_template(template, template.name)
+      protocol = fetch_protocol(template.call_template_type)
+      result = protocol.register_manual(self, template)
+      return result unless result.success?
+
+      allowed = template.allowed_protocols
+      filtered = result.manual.tools.each_with_object([]) do |tool, tools|
+        type = tool.tool_call_template&.call_template_type || template.call_template_type
+        if allowed.include?(type)
+          tool.name = "#{template.name}.#{tool.name}" unless tool.name.start_with?("#{template.name}.")
+          tools << tool
+        else
+          logger.warn(
+            "Tool #{tool.name.inspect} uses protocol #{type.inspect}, which is not allowed " \
+            "by manual #{template.name.inspect}; allowed protocols: #{allowed.inspect}"
+          )
+        end
+      end
+      result.manual.tools = filtered
+      result.manual_call_template = template
+      config.tool_repository.save_manual(template, result.manual)
+      result
+    end
+
+    def register_manuals(manual_call_templates)
+      Array(manual_call_templates).map do |template|
+        register_manual(template)
+      rescue VariableNotFoundError
+        raise
+      rescue StandardError => error
+        parsed = CallTemplate.from_h(template)
+        logger.warn("Unable to register manual #{parsed.name.inspect}: #{error.message}")
+        RegisterManualResult.new(
+          manual_call_template: parsed,
+          manual: Manual.new(tools: [], manual_version: "0.0.0"),
+          success: false,
+          errors: [error.message]
+        )
+      end
+    end
+
+    def deregister_manual(manual_name)
+      template = config.tool_repository.get_manual_call_template(manual_name)
+      return false unless template
+
+      fetch_protocol(template.call_template_type).deregister_manual(self, template)
+      config.tool_repository.remove_manual(manual_name)
+    end
+
+    def call_tool(tool_name, tool_args = {})
+      tool = config.tool_repository.get_tool(tool_name)
+      raise ToolNotFoundError, "Tool not found: #{tool_name}" unless tool
+
+      manual_name = tool_name.to_s.split(".", 2).first
+      template = substitute_template(tool.tool_call_template, manual_name)
+      enforce_allowed_protocol!(manual_name, tool_name, template.call_template_type)
+      result = fetch_protocol(template.call_template_type).call_tool(self, tool_name, tool_args, template)
+      apply_post_processing(result, tool, template)
+    rescue Error
+      raise
+    rescue StandardError => error
+      raise ToolCallError.new("Tool #{tool_name.inspect} failed: #{error.message}", tool_name: tool_name)
+    end
+
+    def call_tool_streaming(tool_name, tool_args = {})
+      return enum_for(__method__, tool_name, tool_args) unless block_given?
+
+      tool = config.tool_repository.get_tool(tool_name)
+      raise ToolNotFoundError, "Tool not found: #{tool_name}" unless tool
+
+      manual_name = tool_name.to_s.split(".", 2).first
+      template = substitute_template(tool.tool_call_template, manual_name)
+      enforce_allowed_protocol!(manual_name, tool_name, template.call_template_type)
+      protocol = fetch_protocol(template.call_template_type)
+      protocol.call_tool_streaming(self, tool_name, tool_args, template) do |item|
+        yield apply_post_processing(item, tool, template)
+      end
+    end
+
+    def search_tools(query, limit: 10, any_of_tags_required: nil)
+      config.tool_search_strategy.search_tools(
+        tool_repository: config.tool_repository,
+        query: query,
+        limit: limit,
+        any_of_tags_required: any_of_tags_required
+      )
+    end
+
+    def list_tools
+      config.tool_repository.get_tools
+    end
+
+    def get_required_variables_for_manual_and_tools(manual_call_template)
+      template = copy_template(CallTemplate.from_h(manual_call_template))
+      template.name = sanitize_name(template.name)
+      required = variable_substitutor.find_required_variables(template.to_h, template.name)
+      return required unless required.empty?
+
+      substituted = substitute_template(template, template.name)
+      result = fetch_protocol(substituted.call_template_type).register_manual(self, substituted)
+      return [] unless result.success?
+
+      result.manual.tools.flat_map do |tool|
+        variable_substitutor.find_required_variables(tool.tool_call_template.to_h, template.name)
+      end.uniq
+    end
+
+    def get_required_variables_for_registered_tool(tool_name)
+      tool = config.tool_repository.get_tool(tool_name)
+      raise ToolNotFoundError, "Tool not found: #{tool_name}" unless tool
+
+      manual_name = tool_name.to_s.split(".", 2).first
+      variable_substitutor.find_required_variables(tool.tool_call_template.to_h, manual_name)
+    end
+
+    def manual(name)
+      config.tool_repository.get_manual(name)
+    end
+
+    def manuals
+      config.tool_repository.get_manuals
+    end
+
+    def close
+      config.tool_repository.get_manual_call_templates.each do |template|
+        deregister_manual(template.name)
+      rescue StandardError => error
+        logger.warn("Unable to deregister manual #{template.name.inspect}: #{error.message}")
+      end
+      nil
+    end
+
+    private
+
+    def fetch_protocol(type)
+      UTCP.protocol(type) || raise(
+        ProtocolNotFoundError,
+        "No communication protocol registered for #{type.inspect}; available: #{UTCP.protocol_types.join(', ')}"
+      )
+    end
+
+    def sanitize_name(name)
+      sanitized = name.to_s.gsub(/[^[:alnum:]_]/, "_")
+      raise ValidationError, "manual name cannot be empty" if sanitized.empty?
+
+      sanitized
+    end
+
+    def substitute_template(template, namespace)
+      substituted = variable_substitutor.substitute(template.to_h, config, namespace)
+      CallTemplate.from_h(substituted)
+    end
+
+    def copy_template(template)
+      CallTemplate.from_h(template.to_h)
+    end
+
+    def enforce_allowed_protocol!(manual_name, tool_name, type)
+      manual_template = config.tool_repository.get_manual_call_template(manual_name)
+      return unless manual_template
+      return if manual_template.allowed_protocols.include?(type)
+
+      raise ProtocolNotAllowedError,
+            "Tool #{tool_name.inspect} uses protocol #{type.inspect}, which is not allowed by " \
+            "manual #{manual_name.inspect}; allowed protocols: #{manual_template.allowed_protocols.inspect}"
+    end
+
+    def apply_post_processing(result, tool, template)
+      config.post_processing.reduce(result) do |value, processor|
+        if processor.respond_to?(:post_process)
+          processor.post_process(self, tool, template, value)
+        elsif processor.respond_to?(:call)
+          processor.call(value)
+        else
+          raise ValidationError, "post processor must respond to call or post_process"
+        end
+      end
+    end
+  end
+  UtcpClient = Client
+end
+

data/lib/utcp/config.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module UTCP
+  class ClientConfig
+    include ModelSerialization
+    attr_accessor :variables, :load_variables_from, :tool_repository,
+                  :tool_search_strategy, :post_processing, :manual_call_templates
+
+    def self.from(value = nil, root_dir: Dir.pwd)
+      return new(root_dir: root_dir) if value.nil?
+      return value if value.is_a?(self)
+
+      data = if value.respond_to?(:to_path) || value.is_a?(String)
+               Utils.load_document(File.expand_path(value.to_s, root_dir))
+             else
+               value
+             end
+      new(root_dir: root_dir, **Utils.symbolize_keys(Utils.stringify_keys(Utils.hash!(data, "client config"))))
+    end
+
+    def self.from_h(value, root_dir: Dir.pwd)
+      from(value, root_dir: root_dir)
+    end
+
+    def initialize(variables: nil, load_variables_from: nil, tool_repository: nil,
+                   tool_search_strategy: nil, post_processing: nil,
+                   manual_call_templates: nil, root_dir: Dir.pwd, **_extra)
+      @variables = Utils.stringify_keys(variables || {})
+      @load_variables_from = Array(load_variables_from).map do |loader|
+        VariableLoader.from_h(loader, root_dir: root_dir)
+      end
+      @tool_repository = build_repository(tool_repository)
+      @tool_search_strategy = build_search_strategy(tool_search_strategy)
+      @post_processing = Array(post_processing)
+      @manual_call_templates = Array(manual_call_templates).map { |template| CallTemplate.from_h(template) }
+    end
+
+    def to_h
+      {
+        "variables" => Utils.deep_copy(variables),
+        "load_variables_from" => load_variables_from.map { |loader| loader.respond_to?(:to_h) ? loader.to_h : loader },
+        "tool_repository" => tool_repository.respond_to?(:to_h) ? tool_repository.to_h : tool_repository,
+        "tool_search_strategy" => tool_search_strategy.respond_to?(:to_h) ? tool_search_strategy.to_h : tool_search_strategy,
+        "post_processing" => post_processing.map { |processor| processor.respond_to?(:to_h) ? processor.to_h : processor },
+        "manual_call_templates" => manual_call_templates.map(&:to_h)
+      }
+    end
+
+    private
+
+    def build_repository(value)
+      return value if value && value.respond_to?(:save_manual)
+
+      data = Utils.stringify_keys(value || { "tool_repository_type" => "in_memory" })
+      case data["tool_repository_type"]
+      when nil, "in_memory"
+        InMemoryToolRepository.new
+      else
+        raise ValidationError, "Unsupported tool_repository_type: #{data['tool_repository_type']}"
+      end
+    end
+
+    def build_search_strategy(value)
+      return value if value && value.respond_to?(:search_tools)
+
+      data = Utils.stringify_keys(value || { "tool_search_strategy_type" => "tag_and_description_word_match" })
+      case data["tool_search_strategy_type"]
+      when nil, "tag_and_description_word_match"
+        TagSearchStrategy.new(
+          description_weight: data.fetch("description_weight", 1),
+          tag_weight: data.fetch("tag_weight", 3)
+        )
+      else
+        raise ValidationError, "Unsupported tool_search_strategy_type: #{data['tool_search_strategy_type']}"
+      end
+    end
+  end
+  UtcpClientConfig = ClientConfig
+end

data/lib/utcp/errors.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module UTCP
+  class Error < StandardError; end
+  UtcpError = Error
+  UTCPError = Error
+
+  class ValidationError < Error
+    attr_reader :path
+
+    def initialize(message, path: nil)
+      @path = path
+      super(path ? "#{path}: #{message}" : message)
+    end
+  end
+
+  class SerializerValidationError < ValidationError; end
+  UtcpSerializerValidationError = SerializerValidationError
+  class VariableNotFoundError < Error
+    attr_reader :variable_name
+
+    def initialize(variable_name)
+      @variable_name = variable_name
+      super("Required variable not found: #{variable_name}")
+    end
+  end
+  UtcpVariableNotFound = VariableNotFoundError
+  UtcpVariableNotFoundException = VariableNotFoundError
+
+  class ToolNotFoundError < Error; end
+  class ManualAlreadyRegisteredError < Error; end
+  class ProtocolNotFoundError < Error; end
+  class ProtocolNotAllowedError < Error; end
+  class MissingDependencyError < Error; end
+  class AuthenticationError < Error; end
+  class ToolCallError < Error
+    attr_reader :tool_name, :status, :response_body
+
+    def initialize(message, tool_name: nil, status: nil, response_body: nil)
+      @tool_name = tool_name
+      @status = status
+      @response_body = response_body
+      super(message)
+    end
+  end
+  class SecurityError < Error; end
+  class TimeoutError < ToolCallError; end
+end

data/lib/utcp/migration.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+module UTCP
+  module Migration
+    module_function
+
+    def config_v0_1_to_v1_1(config)
+      data = Utils.stringify_keys(Utils.deep_copy(Utils.hash!(config, "v0.1 config")))
+      providers = Array(data.delete("providers"))
+      existing = Array(data["manual_call_templates"])
+      data["manual_call_templates"] = existing + providers.map { |provider| migrate_provider(provider) }
+      data["variables"] ||= {}
+      data
+    end
+
+    def manual_v0_1_to_v1_1(manual)
+      data = Utils.stringify_keys(Utils.deep_copy(Utils.hash!(manual, "v0.1 manual")))
+      provider_info = Utils.stringify_keys(data.delete("provider_info") || {})
+      info = Utils.stringify_keys(data["info"] || {})
+      info["title"] ||= provider_info["name"] || "UTCP Manual"
+      info["version"] ||= provider_info["version"] || "1.0.0"
+      info["description"] ||= provider_info["description"] if provider_info["description"]
+
+      {
+        "manual_version" => "1.0.0",
+        "utcp_version" => VERSION,
+        "info" => info,
+        "tools" => Array(data["tools"]).map { |tool| migrate_tool(tool) }
+      }
+    end
+
+    def v0_1_manual?(value)
+      return false unless value.is_a?(Hash)
+
+      data = Utils.stringify_keys(value)
+      data.key?("provider_info") || Array(data["tools"]).any? do |tool|
+        tool.is_a?(Hash) && (tool.key?("provider") || tool.key?("parameters") || tool.key?(:provider))
+      end
+    end
+
+    def migrate_provider(provider)
+      source = Utils.stringify_keys(Utils.deep_copy(Utils.hash!(provider, "provider")))
+      type = source.delete("provider_type") || source.delete("type") || source["call_template_type"]
+      raise ValidationError, "v0.1 provider is missing provider_type" if type.nil? || type.to_s.empty?
+
+      source["call_template_type"] = type.to_s
+      source["http_method"] = source.delete("method") if source["method"] && !source["http_method"]
+      source["working_dir"] = source.delete("cwd") if source["cwd"] && !source["working_dir"]
+      source["working_dir"] = source.delete("working_directory") if source["working_directory"] && !source["working_dir"]
+
+      if type.to_s == "cli" && source["command"]
+        command = source.delete("command").to_s
+        arguments = Array(source.delete("args")).map { |argument| migrate_cli_argument(argument) }
+        source["commands"] ||= [{ "command" => ([command] + arguments).join(" "), "append_to_final_output" => true }]
+      end
+      if type.to_s == "http" && source.key?("body") && !source.key?("body_field")
+        source.delete("body")
+        source["body_field"] = "body"
+      end
+      source
+    end
+
+    def migrate_tool(tool)
+      source = Utils.stringify_keys(Utils.deep_copy(Utils.hash!(tool, "tool")))
+      source["inputs"] ||= source.delete("parameters") || {}
+      source["outputs"] ||= {}
+      source["tags"] ||= []
+      provider = source.delete("provider") || source.delete("tool_provider")
+      source["tool_call_template"] ||= migrate_provider(provider) if provider
+      source
+    end
+
+    def migrate_cli_argument(argument)
+      converted = argument.to_s.gsub(/\$\{([A-Za-z0-9_]+)\}|\$([A-Za-z0-9_]+)/) do
+        "UTCP_ARG_#{Regexp.last_match(1) || Regexp.last_match(2)}_UTCP_END"
+      end
+      converted.include?("UTCP_ARG_") ? converted : Shellwords.escape(converted)
+    end
+
+    class << self
+      alias migrate_config config_v0_1_to_v1_1
+      alias migrate_manual manual_v0_1_to_v1_1
+    end
+  end
+end
+