scout-ai 0.2.0 → 1.0.1

data/lib/scout/llm/mcp.rb

@@ -0,0 +1,28 @@
+require 'mcp'
+
+module Workflow
+  def mcp(*tasks)
+    tasks = tasks.flatten.compact
+    tasks = self.tasks.keys if tasks.empty?
+
+    tools = tasks.collect do |task,inputs=nil|
+      tool_definition = LLM.task_tool_definition(self, task, inputs)[:function]
+      description = tool_definition[:description]
+      input_schema = tool_definition[:parameters].slice(:properties, :required)
+      annotations = tool_definition.slice(:title)
+      annotations[:read_only_hint] = true
+      annotations[:destructive_hint] = false
+      annotations[:idempotent_hint] = true
+      annotations[:open_world_hint] = false
+      MCP::Tool.define(name:task, description: description, input_schema: input_schema, annotations:annotations) do |parameters,context|
+        self.job(name, parameters)
+      end
+    end
+
+    MCP::Server.new(
+      name: self.name,
+      version: "1.0.0",
+      tools: tools
+    )
+  end
+end

data/lib/scout/llm/parse.rb

@@ -1,4 +1,34 @@
+require 'scout/llm/utils'
 module LLM
+  def self.process_inside(inside)
+    header, content = inside.match(/([^\n]*)\n(.*)/).values_at 1, 2
+    if header.empty?
+      content
+    else
+      action, _sep, rest = header.partition /\s/
+      case action
+      when 'import'
+      when 'cmd'
+        title = rest.strip.empty? ? content : rest
+        tag('file', title, CMD.cmd(content).read)
+      when 'file'
+        file = content
+        title = rest.strip.empty? ? file : rest
+        tag(action, title, Open.read(file))
+      when 'directory'
+        directory = content
+        title = rest.strip.empty? ? directory : rest
+        directory_content = Dir.glob(File.join(directory, '**/*')).collect do |file|
+          file_title = Misc.path_relative_to(directory, file)
+          tag('file', file_title, Open.read(file) )
+        end * "\n"
+        tag(action, title, directory_content )
+      else 
+        tag(action, rest, content)
+      end
+    end
+  end
+
   def self.parse(question, role = nil)
     role = :user if role.nil?
 
@@ -12,21 +42,49 @@ module LLM
         inside = m[2]
         post = m[3]
         messages = parse(pre, role)
-        messages.last[:content] += "\n" + inside
-        messages.concat parse(post, role)
+
+        messages = [{role: role, content: ''}] if messages.empty?
+        messages.last[:content] += process_inside inside
+
+        last = parse(post, messages.last[:role])
+
+        messages.concat last
+
+        messages
+      elsif m = question.match(/(.*?)(```.*?```)(.*)/m)
+        pre = m[1]
+        inside = m[2]
+        post = m[3]
+        messages = parse(pre, role)
+
+        messages = [{role: role, content: ''}] if messages.empty?
+        messages.last[:content] += inside
+
+        last = parse(post, messages.last[:role])
+
+        if last.first[:role] == messages.last[:role]
+          m = last.shift
+          messages.last[:content] += m[:content]
+        end
+
+        messages.concat last
+
+        messages
       else
-        question.split("\n").collect do |line|
-          if line.include?("\t")
-            question_role, _sep, q = line.partition("\t")
-          elsif m = line.match(/^([^\s]*): ?(.*)/)
-            question_role, q = m.values_at 1, 2
-          else
-            question_role = role
-            q = line
+        chunks = question.scan(/(.*?)^(\w+):(.*?)(?=^\w+:|\z)/m)
+
+        if chunks.any?
+          messages = []
+          messages << {role: role, content: chunks.first.first} if chunks.first and not chunks.first.first.empty?
+          chunks.collect do |pre,role,text|
+            messages << {role: role, content: text.strip}
           end
-          next if q.empty?
-          {role: question_role, content: q}
-        end.compact
+          messages
+        elsif question.strip.empty?
+          []
+        else
+          [{role: role, content: question}]
+        end
       end
     end
   end

data/lib/scout/llm/rag.rb

@@ -12,5 +12,14 @@ module LLM
       end
       t
     end
+
+    def self.load(path, dim)
+      require 'hnswlib'
+
+      u = Hnswlib::HierarchicalNSW.new(space: 'l2', dim: dim)
+      u.load_index(path)
+
+      u
+    end
   end
 end

data/lib/scout/llm/tools/call.rb

@@ -0,0 +1,66 @@
+module LLM
+  def self.call_id_name_and_arguments(tool_call)
+    tool_call_id = tool_call.dig("call_id") || tool_call.dig("id")
+    if tool_call['function']
+      function_name = tool_call.dig("function", "name")
+      function_arguments = tool_call.dig("function", "arguments")
+    else
+      function_name = tool_call.dig("name")
+      function_arguments = tool_call.dig("arguments")
+    end
+
+    function_arguments = JSON.parse(function_arguments, { symbolize_names: true }) if String === function_arguments
+
+    [tool_call_id, function_name, function_arguments]
+  end
+
+  def self.process_calls(tools, calls, &block)
+    IndiferentHash.setup tools
+    calls.collect do |tool_call|
+      tool_call_id, function_name, function_arguments = call_id_name_and_arguments(tool_call)
+
+      obj, definition = tools[function_name]
+
+      function_response = case obj
+                          when Proc
+                            obj.call function_name, function_arguments
+                          when Workflow
+                            call_workflow(obj, function_name, function_arguments)
+                          when KnowledgeBase
+                            call_knowledge_base(obj, function_name, function_arguments)
+                          else
+                            if block_given?
+                              block.call function_name, function_arguments
+                            else
+                              raise "Unkown executor #{Log.fingerprint obj} for function #{function_name}"
+                            end
+                          end
+
+      content = case function_response
+                when String
+                  function_response
+                when nil
+                  "success"
+                when Exception
+                  {exception: function_response.message, stack: function_response.backtrace }.to_json
+                else
+                  function_response.to_json
+                end
+      content = content.to_s if Numeric === content
+
+      response_message = {
+        id: tool_call_id,
+        role: "tool",
+        content: content
+      }
+
+      function_call = tool_call.dup
+
+      function_call['id'] = function_call.delete('call_id') if function_call.dig('call_id')
+      [
+        {role: "function_call", content: function_call.to_json},
+        {role: "function_call_output", content: response_message.to_json},
+      ]
+    end.flatten
+  end
+end

data/lib/scout/llm/tools/knowledge_base.rb

@@ -0,0 +1,158 @@
+require 'scout/knowledge_base'
+
+module LLM
+  def self.database_tool_definition(database, undirected = false, database_description = nil)
+
+    if undirected
+      properties = {
+        entities: {
+          type: "array",
+          items: { type: :string },
+          description: "Entities for which to find associations"
+        },
+      }
+    else
+      properties = {
+        entities: {
+          type: "array",
+          items: { type: :string },
+          description: "Source entities in the association, or target entities if 'reverse' is 'true'"
+        },
+        reverse: {
+          type: "boolean",
+          description: "Look for targets instead of sources, defaults to 'false'"
+        }
+      }
+    end
+
+    if database_description and not database_description.strip.empty?
+      description = <<-EOF
+Find associations for a list of entities in database #{database}: #{database_description}
+      EOF
+    else
+      description = <<-EOF
+Find associations for a list of entities in database #{database}.
+      EOF
+    end
+
+    if undirected
+      description += <<-EOF
+Returns a list in the format entity~partner.
+      EOF
+    else
+      description += <<-EOF
+Returns a list in the format source~target.
+      EOF
+    end
+
+    function = {
+        name: database,
+        description: description,
+        parameters: {
+          type: "object",
+          properties: properties,
+          required: ['entities']
+        }
+    }
+
+    IndiferentHash.setup function.merge(type: 'function', function: function)
+  end
+
+  def self.database_details_tool_definition(database, undirected, fields)
+
+    if undirected
+      properties = {
+        associations: {
+          type: "array",
+          items: { type: :string },
+          description: "Associations in the form of source~target or target~source"
+        },
+        fields: {
+          type: "string",
+          enum: select_options,
+          description: "Limit the response to these detail fields fields"
+        },
+      }
+    else
+      properties = {
+        associations: {
+          type: "array",
+          items: { type: :string },
+          description: "Associations in the form of source~target"
+        },
+      }
+    end
+
+    if fields.length > 1
+      description = <<-EOF
+Return details of association as a dictionary object.
+Each key is an association and the value is an array with the values of the different fields you asked for, or for all fields otherwise.
+The fields are: #{fields * ', '}.
+Multiple values may be present and use the charater ';' to separate them.
+      EOF
+    else
+      properties.delete(:fields)
+      description = <<-EOF
+Return the #{field} of association.
+Multiple values may be present and use the charater ';' to separate them.
+      EOF
+    end
+
+    function = {
+        name: database + '_association_details',
+        description: description,
+        parameters: {
+          type: "object",
+          properties: properties,
+          required: ['associations']
+        }
+    }
+
+    IndiferentHash.setup function.merge(type: 'function', function: function)
+  end
+
+
+  def self.knowledge_base_tool_definition(knowledge_base, databases = nil)
+    databases ||= knowledge_base.all_databases
+
+    databases.inject({}){|tool_definitions,database|
+      database_description = knowledge_base.description(database)
+      undirected = knowledge_base.undirected(database)
+      definition = self.database_tool_definition(database, undirected, database_description)
+      tool_definitions.merge(database => [knowledge_base, definition])
+      if (fields = knowledge_base.get_database(database).fields).any?
+        details_definition = self.database_details_tool_definition(database, undirected, fields)
+        tool_definitions.merge(database + '_association_details' => [knowledge_base, details_definition])
+      end
+    }
+  end
+
+  def self.call_knowledge_base(knowledge_base, database, parameters={})
+    if database.end_with?('_association_details')
+      database = database.sub('_association_details', '')
+      associations, fields = IndiferentHash.process_options parameters, :associations, :fields
+      index = knowledge_base.get_index(database)
+      if fields
+        field_pos = fields.collect{|f| index.identify_field f }
+        associations.each_with_object({}) do |a,hash|
+          values = index[a]
+          next if values.nil?
+          hash[a] = values.values_at *field_pos
+        end
+      else
+        associations.each_with_object({}) do |a,hash|
+          values = index[a]
+          next if values.nil?
+          hash[a] = values
+        end
+      end
+    else
+      entities, reverse = IndiferentHash.process_options parameters, :entities, :reverse
+      if reverse
+        knowledge_base.parents(database, entities)
+      else
+        knowledge_base.children(database, entities)
+      end
+    end
+  end
+end

data/lib/scout/llm/tools/mcp.rb

@@ -0,0 +1,59 @@
+require_relative '../utils'
+require 'mcp_client'
+
+module LLM
+  def self.mcp_tools(url, options = {})
+    if url == 'stdio'
+      client = MCPClient.create_client(mcp_server_configs: [options.merge(type: 'stdio')])
+    else
+      type = IndiferentHash.process_options options, :type,
+        type: (Open.remote?(url) ? :http : :stdio)
+
+      if url && Open.remote?(url)
+        token ||= LLM.get_url_config(:key, url, :mcp)
+        options[:headers] = { 'Authorization' => "Bearer #{token}" }
+      end
+
+      client = MCPClient.create_client(mcp_server_configs: [options.merge(type: 'http', url: url)])
+    end
+
+    tools = client.list_tools
+
+    tool_definitions = IndiferentHash.setup({})
+    tools.each do |tool|
+      name = tool.name
+      description = tool.description
+      schema = tool.schema
+
+      function = {
+        name: name,
+        description: description,
+        parameters: schema
+      }
+
+      definition = IndiferentHash.setup function.merge(type: 'function', function: function)
+      block = Proc.new do |name,params|
+        res = tool.server.call_tool(name, params)
+        if Hash === res && res['content']
+          res = res['content']
+        end
+
+        if Array === res and res.length == 1
+          res = res.first
+        end
+
+        if Hash === res && res['content']
+          res = res['content']
+        end
+
+        if Hash === res && res['text']
+          res = res['text']
+        end
+
+        res
+      end
+      tool_definitions[name] = [block, definition]
+    end
+    tool_definitions
+  end
+end

data/lib/scout/llm/tools/workflow.rb

@@ -0,0 +1,69 @@
+require 'scout/workflow'
+module LLM
+  def self.task_tool_definition(workflow, task_name, inputs = nil)
+    task_info = workflow.task_info(task_name)
+
+    inputs = inputs.collect{|i| i.to_sym } if inputs
+
+    properties = task_info[:inputs].inject({}) do |acc,input|
+      next acc if inputs and not inputs.include?(input)
+      type = task_info[:input_types][input]
+      description = task_info[:input_descriptions][input]
+
+      type = :string if type == :text
+      type = :string if type == :select
+      type = :string if type == :path
+      type = :number if type == :float
+
+      acc[input] = {
+        "type": type,
+        "description": description
+      }
+
+      if input_options = task_info[:input_options][input]
+        if select_options = input_options[:select_options]
+          select_options = select_options.values if Hash === select_options
+          acc[input]["enum"] = select_options
+        end
+      end
+
+      acc
+    end
+
+    required_inputs = task_info[:inputs].select do |input|
+      next if inputs and not inputs.include?(input.to_sym)
+      task_info[:input_options].include?(input) && task_info[:input_options][input][:required]
+    end
+
+    function = {
+      name: task_name,
+      description: task_info[:description],
+      parameters: {
+        type: "object",
+        properties: properties,
+        required: required_inputs
+      }
+    }
+
+    IndiferentHash.setup function.merge(type: 'function', function: function)
+  end
+
+  def self.workflow_tools(workflow, tasks = nil)
+    tasks = workflow.all_exports if tasks.nil?
+    tasks = workflow.all_tasks if tasks.empty?
+
+    tasks.inject({}){|tool_definitions,task_name|
+      definition = self.task_tool_definition(workflow, task_name)
+      tool_definitions.merge(task_name => [workflow, definition])
+    }
+  end
+
+  def self.call_workflow(workflow, task_name, parameters={})
+    jobname = parameters.delete :jobname
+    if workflow.exec_exports.include? task_name.to_sym
+      workflow.job(task_name, jobname, parameters).exec
+    else
+      workflow.job(task_name, jobname, parameters).run
+    end
+  end
+end