scout-ai 1.0.0 → 1.1.0

data/lib/scout/llm/tools/call.rb

@@ -0,0 +1,76 @@
+module LLM
+  def self.call_id_name_and_arguments(tool_call)
+    tool_call_id = tool_call.dig("call_id") || tool_call.dig("id")
+    if tool_call['function']
+      function_name = tool_call.dig("function", "name")
+      function_arguments = tool_call.dig("function", "arguments")
+    else
+      function_name = tool_call.dig("name")
+      function_arguments = tool_call.dig("arguments")
+    end
+
+    function_arguments = JSON.parse(function_arguments, { symbolize_names: true }) if String === function_arguments
+
+    [tool_call_id, function_name, function_arguments]
+  end
+
+  def self.process_calls(tools, calls, &block)
+    IndiferentHash.setup tools
+    calls.collect do |tool_call|
+      tool_call_id, function_name, function_arguments = call_id_name_and_arguments(tool_call)
+
+      function_arguments = IndiferentHash.setup function_arguments
+
+      obj, definition = tools[function_name]
+
+      definition = obj if Hash === obj
+
+      defaults = definition[:parameters][:defaults] if definition && definition[:parameters]
+      function_arguments = function_arguments.merge(defaults) if defaults
+
+      Log.high "Calling #{function_name} (#{Log.fingerprint function_arguments}): "
+      function_response = case obj
+                          when Proc
+                            obj.call function_name, function_arguments
+                          when Workflow
+                            call_workflow(obj, function_name, function_arguments)
+                          when KnowledgeBase
+                            call_knowledge_base(obj, function_name, function_arguments)
+                          else
+                            if block_given?
+                              block.call function_name, function_arguments
+                            else
+                              ParameterException.new "Tool or function not found '#{function_name}'. Called with parameters #{Log.fingerprint function_arguments}" if obj.nil? && definition.nil?
+                            end
+                          end
+
+      content = case function_response
+                when String
+                  function_response
+                when nil
+                  "success"
+                when Exception
+                  {exception: function_response.message, stack: function_response.backtrace }.to_json
+                else
+                  function_response.to_json
+                end
+      content = content.to_s if Numeric === content
+
+      Log.high "Called #{function_name}: " + Log.fingerprint(content)
+
+      response_message = {
+        id: tool_call_id,
+        role: "tool",
+        content: content
+      }
+
+      function_call = tool_call.dup
+
+      function_call['id'] = function_call.delete('call_id') if function_call.dig('call_id')
+      [
+        {role: "function_call", content: function_call.to_json},
+        {role: "function_call_output", content: response_message.to_json},
+      ]
+    end.flatten
+  end
+end

data/lib/scout/llm/tools/knowledge_base.rb

@@ -0,0 +1,159 @@
+require 'scout/knowledge_base'
+
+module LLM
+  def self.database_tool_definition(database, undirected = false, database_description = nil)
+
+    if undirected
+      properties = {
+        entities: {
+          type: "array",
+          items: { type: :string },
+          description: "Entities for which to find associations"
+        },
+      }
+    else
+      properties = {
+        entities: {
+          type: "array",
+          items: { type: :string },
+          description: "Source entities in the association, or target entities if 'reverse' is 'true'"
+        },
+        reverse: {
+          type: "boolean",
+          description: "Look for targets instead of sources, defaults to 'false'"
+        }
+      }
+    end
+
+    if database_description and not database_description.strip.empty?
+      description = <<-EOF
+Find associations for a list of entities in database #{database}: #{database_description}
+      EOF
+    else
+      description = <<-EOF
+Find associations for a list of entities in database #{database}.
+      EOF
+    end
+
+    if undirected
+      description += <<-EOF
+Returns a list in the format entity~partner.
+      EOF
+    else
+      description += <<-EOF
+Returns a list in the format source~target.
+      EOF
+    end
+
+    function = {
+        name: database,
+        description: description,
+        parameters: {
+          type: "object",
+          properties: properties,
+          required: ['entities']
+        }
+    }
+
+    IndiferentHash.setup function.merge(type: 'function', function: function)
+  end
+
+  def self.database_details_tool_definition(database, undirected, fields)
+
+    if undirected
+      properties = {
+        associations: {
+          type: "array",
+          items: { type: :string },
+          description: "Associations in the form of source~target or target~source"
+        },
+        fields: {
+          type: "array",
+          items: { type: :string },
+          description: "Limit the response to these fields"
+        },
+      }
+    else
+      properties = {
+        associations: {
+          type: "array",
+          items: { type: :string },
+          description: "Associations in the form of source~target"
+        },
+      }
+    end
+
+    if fields.length > 1
+      description = <<-EOF
+Return details of association as a dictionary object.
+Each key is an association and the value is an array with the values of the different fields you asked for, or for all fields otherwise.
+The fields are: #{fields * ', '}.
+Multiple values may be present and use the charater ';' to separate them.
+      EOF
+    else
+      properties.delete(:fields)
+      description = <<-EOF
+Return the #{fields.first} of association.
+Multiple values may be present and use the charater ';' to separate them.
+      EOF
+    end
+
+    function = {
+      name: database.to_s + '_association_details',
+      description: description,
+      parameters: {
+        type: "object",
+        properties: properties,
+        required: ['associations']
+      }
+    }
+
+    IndiferentHash.setup function
+  end
+
+
+  def self.knowledge_base_tool_definition(knowledge_base, databases = nil)
+    databases ||= knowledge_base.all_databases
+
+    databases.inject({}){|tool_definitions,database|
+      database_description = knowledge_base.description(database)
+      undirected = knowledge_base.undirected(database)
+      definition = self.database_tool_definition(database, undirected, database_description)
+      tool_definitions.merge!(database => [knowledge_base, definition])
+      if (fields = knowledge_base.get_database(database).fields).any?
+        details_definition = self.database_details_tool_definition(database, undirected, fields)
+        tool_definitions.merge!(database.to_s + '_association_details' => [knowledge_base, details_definition])
+      end
+      tool_definitions
+    }
+  end
+
+  def self.call_knowledge_base(knowledge_base, database, parameters={})
+    if database.end_with?('_association_details')
+      database = database.sub('_association_details', '')
+      associations, fields = IndiferentHash.process_options parameters, :associations, :fields
+      index = knowledge_base.get_index(database)
+      if fields
+        field_pos = fields.collect{|f| index.identify_field f }
+        associations.each_with_object({}) do |a,hash|
+          values = index[a]
+          next if values.nil?
+          hash[a] = values.values_at *field_pos
+        end
+      else
+        associations.each_with_object({}) do |a,hash|
+          values = index[a]
+          next if values.nil?
+          hash[a] = values
+        end
+      end
+    else
+      entities, reverse = IndiferentHash.process_options parameters, :entities, :reverse
+      if reverse
+        knowledge_base.parents(database, entities)
+      else
+        knowledge_base.children(database, entities)
+      end
+    end
+  end
+end

data/lib/scout/llm/tools/mcp.rb

@@ -0,0 +1,59 @@
+require_relative '../utils'
+require 'mcp_client'
+
+module LLM
+  def self.mcp_tools(url, options = {})
+    if url == 'stdio'
+      client = MCPClient.create_client(mcp_server_configs: [options.merge(type: 'stdio')])
+    else
+      type = IndiferentHash.process_options options, :type,
+        type: (Open.remote?(url) ? :http : :stdio)
+
+      if url && Open.remote?(url)
+        token ||= LLM.get_url_config(:key, url, :mcp)
+        options[:headers] = { 'Authorization' => "Bearer #{token}" }
+      end
+
+      client = MCPClient.create_client(mcp_server_configs: [options.merge(type: 'http', url: url)])
+    end
+
+    tools = client.list_tools
+
+    tool_definitions = IndiferentHash.setup({})
+    tools.each do |tool|
+      name = tool.name
+      description = tool.description
+      schema = tool.schema
+
+      function = {
+        name: name,
+        description: description,
+        parameters: schema
+      }
+
+      definition = IndiferentHash.setup function.merge(type: 'function', function: function)
+      block = Proc.new do |name,params|
+        res = tool.server.call_tool(name, params)
+        if Hash === res && res['content']
+          res = res['content']
+        end
+
+        if Array === res and res.length == 1
+          res = res.first
+        end
+
+        if Hash === res && res['content']
+          res = res['content']
+        end
+
+        if Hash === res && res['text']
+          res = res['text']
+        end
+
+        res
+      end
+      tool_definitions[name] = [block, definition]
+    end
+    tool_definitions
+  end
+end

data/lib/scout/llm/tools/workflow.rb

@@ -0,0 +1,106 @@
+require 'scout/workflow'
+module LLM
+  def self.task_tool_definition(workflow, task_name, inputs = nil)
+    task_info = workflow.task_info(task_name)
+    return nil if task_info.nil?
+
+    if inputs
+      names = []
+      defaults = {}
+
+      inputs.each do |i|
+        if String === i && i.include?('=')
+          name,_ , value = i.partition("=")
+          defaults[name] = value
+        else
+          names << i.to_sym
+        end
+      end
+
+    end
+
+    properties = task_info[:inputs].inject({}) do |acc,input|
+      next acc if names and not names.include?(input)
+      type = task_info[:input_types][input]
+      description = task_info[:input_descriptions][input]
+
+      type = :string if type == :text
+      type = :string if type == :select
+      type = :string if type == :path
+      type = :number if type == :float
+      type = :array if type.to_s.end_with?('_array')
+
+      acc[input] = {
+        "type": type,
+        "description": description
+      }
+
+      if type == :array
+        acc[input]['items'] = {type: :string}
+      end
+
+      if input_options = task_info[:input_options][input]
+        if select_options = input_options[:select_options]
+          select_options = select_options.values if Hash === select_options
+          acc[input]["enum"] = select_options
+        end
+      end
+
+      acc
+    end
+
+    required_inputs = task_info[:inputs].select do |input|
+      next if names and not names.include?(input.to_sym)
+      task_info[:input_options].include?(input) && task_info[:input_options][input][:required]
+    end
+
+    function = {
+      name: task_name,
+      description: task_info[:description],
+      parameters: {
+        type: "object",
+        properties: properties,
+        required: required_inputs,
+      }
+    }
+
+    function[:parameters][:defaults] = defaults if defaults
+
+    #IndiferentHash.setup function.merge(type: 'function', function: function)
+    IndiferentHash.setup function
+  end
+
+  def self.workflow_tools(workflow, tasks = nil)
+    if Array === workflow
+      workflow.inject({}){|tool_definitions,wf| tool_definitions.merge(workflow_tools(wf, tasks)) }
+
+    else
+      tasks = workflow.all_exports if tasks.nil?
+      tasks = workflow.all_tasks if tasks.empty?
+
+      tasks.inject({}){|tool_definitions,task_name|
+        definition = self.task_tool_definition(workflow, task_name)
+        next if definition.nil?
+        tool_definitions.merge(task_name => [workflow, definition])
+      }
+    end
+  end
+
+  def self.call_workflow(workflow, task_name, parameters={})
+    jobname = parameters.delete :jobname
+    begin
+      exec_type = parameters[:exec_type]
+      job = workflow.job(task_name, jobname, parameters)
+      if workflow.exec_exports.include?(task_name.to_sym) || parameters[:exec_type].to_s == 'exec'
+        job.exec
+      else
+        raise ScoutException, 'Potential recursive call' if parameters[:allow_recursive] != 'true' &&
+          (job.running? and job.info[:pid] == Process.pid)
+
+        job.run
+      end
+    rescue ScoutException
+      return $!
+    end
+  end
+end

data/lib/scout/llm/tools.rb

@@ -1,18 +1,48 @@
-require 'scout/workflow'
 require 'scout/knowledge_base'
+require_relative 'tools/mcp'
+require_relative 'tools/workflow'
+require_relative 'tools/knowledge_base'
+require_relative 'tools/call'
 module LLM
+  def self.call_tools(tool_calls, &block)
+    tool_calls.collect{|tool_call|
+      response_message = LLM.tool_response(tool_call, &block)
+      function_call = tool_call
+      function_call['id'] = tool_call.delete('call_id') if tool_call.dig('call_id')
+      [
+        {role: "function_call", content: tool_call.to_json},
+        {role: "function_call_output", content: response_message.to_json},
+      ]
+    }.flatten
+  end
+
   def self.tool_response(tool_call, &block)
-    tool_call_id = tool_call.dig("id")
-    function_name = tool_call.dig("function", "name")
-    function_arguments = tool_call.dig("function", "arguments")
+    tool_call_id = tool_call.dig("call_id") || tool_call.dig("id")
+    if tool_call['function']
+      function_name = tool_call.dig("function", "name")
+      function_arguments = tool_call.dig("function", "arguments")
+    else
+      function_name = tool_call.dig("name")
+      function_arguments = tool_call.dig("arguments")
+    end
+
     function_arguments = JSON.parse(function_arguments, { symbolize_names: true }) if String === function_arguments
-    function_response = block.call function_name, function_arguments
+
+    Log.high "Calling function #{function_name} with arguments #{Log.fingerprint function_arguments}"
+
+    function_response = begin
+                          block.call function_name, function_arguments
+                        rescue
+                          $!
+                        end
 
     content = case function_response
               when String
                 function_response
               when nil
                 "success"
+              when Exception
+                {exception: function_response.message, stack: function_response.backtrace }.to_json
               else
                 function_response.to_json
               end
@@ -24,118 +54,6 @@ module LLM
     }
   end
 
-  def self.task_tool_definition(workflow, task_name, inputs = nil)
-    task_info = workflow.task_info(task_name)
-
-    inputs = inputs.collect{|i| i.to_sym } if inputs
-
-    properties = task_info[:inputs].inject({}) do |acc,input|
-      next acc if inputs and not inputs.include?(input)
-      type = task_info[:input_types][input]
-      description = task_info[:input_descriptions][input]
-
-      type = :string if type == :text
-      type = :string if type == :select
-      type = :string if type == :path
-      type = :number if type == :float
-
-      acc[input] = {
-        "type": type,
-        "description": description
-      }
-
-      if input_options = task_info[:input_options][input]
-        if select_options = input_options[:select_options]
-          select_options = select_options.values if Hash === select_options
-          acc[input]["enum"] = select_options
-        end
-      end
-
-      acc
-    end
-
-    required_inputs = task_info[:inputs].select do |input|
-      next if inputs and not inputs.include?(input.to_sym)
-      task_info[:input_options].include?(input) && task_info[:input_options][input][:required]
-    end
-
-    {
-      type: "function",
-      function: {
-        name: task_name,
-        description: task_info[:description],
-        parameters: {
-          type: "object",
-          properties: properties,
-          required: required_inputs
-        }
-      }
-    }
-  end
-
-  def self.workflow_tools(workflow, tasks = nil)
-    tasks = workflow.all_exports
-    tasks.collect{|task_name| self.task_tool_definition(workflow, task_name) }
-  end
-
-  def self.knowledge_base_tool_definition(knowledge_base)
-
-    databases = knowledge_base.all_databases.collect{|d| d.to_s }
-
-    properties = {
-      database: {
-        type: "string",
-        enum: databases,
-        description: "Database to traverse"
-      },
-      entities: {
-        type: "array",
-        items: { type: :string },
-        description: "Parent entities to find children for"
-      }
-    }
-
-    [{
-      type: "function",
-      function: {
-        name: 'children',
-        description: "Find the graph children for a list of entities in a format like parent~child. Returns a list.",
-        parameters: {
-          type: "object",
-          properties: properties,
-          required: ['database', 'entities']
-        }
-      }
-    }]
-  end
-
-  def self.association_tool_definition(name)
-    properties = {
-      entities: {
-        type: "array",
-        items: { type: :string },
-        description: "Source entities in the association, or target entities if 'reverse' it true."
-      },
-      reverse: {
-        type: "boolean",
-        description: "Look for targets instead of sources, defaults to 'false'."
-      }
-    }
-
-    {
-      type: "function",
-      function: {
-        name: name,
-        description: "Find associations for a list of entities. Returns a list in the format source~target.",
-        parameters: {
-          type: "object",
-          properties: properties,
-          required: ['entities']
-        }
-      }
-    }
-  end
-  
   def self.run_tools(messages)
     messages.collect do |info|
       IndiferentHash.setup(info)
@@ -156,7 +74,7 @@ module LLM
       if message[:role] == 'function_call'
         tool_call = JSON.parse(message[:content])
         arguments = tool_call.delete('arguments') || {}
-        name = tool_call.delete('name')
+        name = tool_call[:name]
         tool_call['type'] = 'function'
         tool_call['function'] ||= {}
         tool_call['function']['name'] ||= name
@@ -164,7 +82,7 @@ module LLM
         {role: 'assistant', tool_calls: [tool_call]}
       elsif message[:role] == 'function_call_output'
         info = JSON.parse(message[:content])
-        id = info.delete('id') || ''
+        id = info.delete('call_id') || info.dig('id')
         info['role'] = 'tool'
         info['tool_call_id'] = id
         info
@@ -174,38 +92,45 @@ module LLM
     end.flatten
   end
 
-  def self.tools_to_ollama(messages)
+  def self.tools_to_anthropic(messages)
     messages.collect do |message|
       if message[:role] == 'function_call'
         tool_call = JSON.parse(message[:content])
         arguments = tool_call.delete('arguments') || {}
-        id = tool_call.delete('id')
-        name = tool_call.delete('name')
-        tool_call['type'] = 'function'
-        tool_call['function'] ||= {}
-        tool_call['function']['name'] ||= name
-        tool_call['function']['arguments'] ||= arguments
-        {role: 'assistant', tool_calls: [tool_call]}
+        name = tool_call[:name]
+        tool_call['type'] = 'tool_use'
+        tool_call['name'] ||= name
+        tool_call['input'] = arguments
+        {role: 'assistant', content: [tool_call]}
       elsif message[:role] == 'function_call_output'
         info = JSON.parse(message[:content])
-        id = info.delete('id') || ''
-        info['role'] = 'tool'
-        info
+        id = info.delete('call_id') || info.delete('id')
+        info.delete "role"
+        info['tool_use_id'] = id
+        info['type'] = 'tool_result'
+        {role: 'user', content: [info]}
       else
         message
       end
     end.flatten
   end
 
-  def self.tools_to_responses(messages)
+  def self.tools_to_ollama(messages)
     messages.collect do |message|
       if message[:role] == 'function_call'
         tool_call = JSON.parse(message[:content])
-        tool_call['function']['arguments'] = (tool_call['function']['arguments'] || {}).to_json
+        arguments = tool_call.delete('arguments') || {}
+        id = tool_call.delete('id')
+        name = tool_call.delete('name')
+        tool_call['type'] = 'function'
+        tool_call['function'] ||= {}
+        tool_call['function']['name'] ||= name
+        tool_call['function']['arguments'] ||= arguments
         {role: 'assistant', tool_calls: [tool_call]}
       elsif message[:role] == 'function_call_output'
         info = JSON.parse(message[:content])
-        info["tool_call_id"] = info['id']
+        id = info.delete('id') || ''
+        info['role'] = 'tool'
         info
       else
         message
@@ -213,13 +138,45 @@ module LLM
     end.flatten
   end
 
-  def self.call_tools(tool_calls, &block)
-    tool_calls.collect{|tool_call|
-      response_message = LLM.tool_response(tool_call, &block)
-      [
-        {role: "function_call", content: tool_call.to_json},
-        {role: "function_call_output", content: response_message.to_json},
-      ]
-    }.flatten
+  def self.tool_definitions_to_reponses(tools)
+    tools.values.collect do |obj,definition|
+      definition = obj if Hash === obj
+      definition
+
+      definition = case definition[:function]
+                   when Hash
+                     definition.merge(definition.delete :function)
+                   else
+                     definition
+                   end
+
+      definition = IndiferentHash.add_defaults definition, type: :function
+
+      definition[:parameters].delete :defaults if definition[:parameters]
+
+      definition
+    end
+  end
+
+  class << self
+    alias tool_definitions_to_openai tool_definitions_to_reponses
+  end
+
+  def self.tool_definitions_to_ollama(tools)
+    tools.values.collect do |obj,definition|
+      definition = obj if Hash === obj
+      definition = IndiferentHash.setup definition
+
+      definition = case definition[:function]
+                   when Hash
+                     definition
+                   else
+                     {type: :function, function: definition}
+                   end
+
+      definition = IndiferentHash.add_defaults definition, type: :function
+
+      definition
+    end
   end
 end