scout-ai 0.2.0 → 1.0.1

data/lib/scout/llm/tools.rb

@@ -1,104 +1,140 @@
-require 'scout/workflow'
 require 'scout/knowledge_base'
+require_relative 'tools/mcp'
+require_relative 'tools/workflow'
+require_relative 'tools/knowledge_base'
+require_relative 'tools/call'
 module LLM
+  def self.call_tools(tool_calls, &block)
+    tool_calls.collect{|tool_call|
+      response_message = LLM.tool_response(tool_call, &block)
+      function_call = tool_call
+      function_call['id'] = tool_call.delete('call_id') if tool_call.dig('call_id')
+      [
+        {role: "function_call", content: tool_call.to_json},
+        {role: "function_call_output", content: response_message.to_json},
+      ]
+    }.flatten
+  end
+
   def self.tool_response(tool_call, &block)
-    tool_call_id = tool_call.dig("id")
-    function_name = tool_call.dig("function", "name")
-    function_arguments = tool_call.dig("function", "arguments")
+    tool_call_id = tool_call.dig("call_id") || tool_call.dig("id")
+    if tool_call['function']
+      function_name = tool_call.dig("function", "name")
+      function_arguments = tool_call.dig("function", "arguments")
+    else
+      function_name = tool_call.dig("name")
+      function_arguments = tool_call.dig("arguments")
+    end
+
     function_arguments = JSON.parse(function_arguments, { symbolize_names: true }) if String === function_arguments
-    function_response = block.call function_name, function_arguments
 
-    #content = String === function_response ? function_response : function_response.to_json,
+    Log.high "Calling function #{function_name} with arguments #{Log.fingerprint function_arguments}"
+
+    function_response = begin
+                          block.call function_name, function_arguments
+                        rescue
+                          $!
+                        end
+
     content = case function_response
               when String
                 function_response
               when nil
                 "success"
+              when Exception
+                {exception: function_response.message, stack: function_response.backtrace }.to_json
               else
                 function_response.to_json
               end
+    content = content.to_s if Numeric === content
     {
-      tool_call_id: tool_call_id,
+      id: tool_call_id,
       role: "tool",
       content: content
     }
   end
 
-  def self.task_tool_definition(workflow, task_name)
-    task_info = workflow.task_info(task_name)
-
-    properties = task_info[:inputs].inject({}) do |acc,input|
-      type = task_info[:input_types][input]
-      description = task_info[:input_descriptions][input]
-
-      type = :string if type == :select
-
-      acc[input] = {
-        "type": type,
-        "description": description
-      }
-
-      if input_options = task_info[:input_options][input]
-        if select_options = input_options[:select_options]
-          select_options = select_options.values if Hash === select_options
-          acc[input]["enum"] = select_options
-        end
+  def self.run_tools(messages)
+    messages.collect do |info|
+      IndiferentHash.setup(info)
+      role = info[:role]
+      if role == 'cmd'
+        {
+          role: 'tool',
+          content: CMD.cmd(info[:content]).read
+        }
+      else
+        info
       end
-
-      acc
-    end
-
-    required_inputs = task_info[:inputs].select do |input|
-      task_info[:input_options].include?(input) && task_info[:input_options][:required]
     end
-
-    {
-      type: "function",
-      function: {
-        name: task_name,
-        description: task_info[:description],
-        parameters: {
-          type: "object",
-          properties: properties,
-          required: required_inputs
-        }
-      }
-    }
   end
 
-  def self.workflow_tools(workflow, tasks = nil)
-    tasks ||= workflow.tasks.keys
-    tasks.collect{|task_name| self.task_tool_definition(workflow, task_name) }
+  def self.tools_to_openai(messages)
+    messages.collect do |message|
+      if message[:role] == 'function_call'
+        tool_call = JSON.parse(message[:content])
+        arguments = tool_call.delete('arguments') || {}
+        name = tool_call[:name]
+        tool_call['type'] = 'function'
+        tool_call['function'] ||= {}
+        tool_call['function']['name'] ||= name
+        tool_call['function']['arguments'] = arguments.to_json
+        {role: 'assistant', tool_calls: [tool_call]}
+      elsif message[:role] == 'function_call_output'
+        info = JSON.parse(message[:content])
+        id = info.delete('call_id') || info.dig('id')
+        info['role'] = 'tool'
+        info['tool_call_id'] = id
+        info
+      else
+        message
+      end
+    end.flatten
   end
 
-  def self.knowledge_base_tool_definition(knowledge_base)
-
-    databases = knowledge_base.all_databases.collect{|d| d.to_s }
-
-    properties = {
-      database: {
-        type: "string",
-        enum: databases,
-        description: "Database to traverse"
-      },
-      entities: {
-        type: "array",
-        items: { type: :string },
-        description: "Parent entities to find children for"
-      }
-    }
+  def self.tools_to_anthropic(messages)
+    messages.collect do |message|
+      if message[:role] == 'function_call'
+        tool_call = JSON.parse(message[:content])
+        arguments = tool_call.delete('arguments') || {}
+        name = tool_call[:name]
+        tool_call['type'] = 'tool_use'
+        tool_call['name'] ||= name
+        tool_call['input'] = arguments
+        {role: 'assistant', content: [tool_call]}
+      elsif message[:role] == 'function_call_output'
+        info = JSON.parse(message[:content])
+        id = info.delete('call_id') || info.delete('id')
+        info.delete "role"
+        info['tool_use_id'] = id
+        info['type'] = 'tool_result'
+        {role: 'user', content: [info]}
+      else
+        message
+      end
+    end.flatten
+  end
 
-    [{
-      type: "function",
-      function: {
-        name: 'children',
-        description: "Find the graph children for a list of entities in a format like parent~child. Returns a list.",
-        parameters: {
-          type: "object",
-          properties: properties,
-          required: ['database', 'entities']
-        }
-      }
-    }]
+  def self.tools_to_ollama(messages)
+    messages.collect do |message|
+      if message[:role] == 'function_call'
+        tool_call = JSON.parse(message[:content])
+        arguments = tool_call.delete('arguments') || {}
+        id = tool_call.delete('id')
+        name = tool_call.delete('name')
+        tool_call['type'] = 'function'
+        tool_call['function'] ||= {}
+        tool_call['function']['name'] ||= name
+        tool_call['function']['arguments'] ||= arguments
+        {role: 'assistant', tool_calls: [tool_call]}
+      elsif message[:role] == 'function_call_output'
+        info = JSON.parse(message[:content])
+        id = info.delete('id') || ''
+        info['role'] = 'tool'
+        info
+      else
+        message
+      end
+    end.flatten
   end
 end

data/lib/scout/llm/utils.rb

@@ -1,4 +1,21 @@
 module LLM
+
+  def self.tag(tag, content, name = nil)
+    if name
+      <<-EOF.strip
+<#{tag} name="#{name}">
+#{content}
+</#{tag}>
+      EOF
+    else
+      <<-EOF.strip
+<#{tag}>
+#{content}
+</#{tag}>
+      EOF
+    end
+  end
+
   def self.get_url_server_tokens(url, prefix=nil)
     return get_url_server_tokens(url).collect{|e| prefix.to_s + "." + e } if prefix
 
@@ -22,14 +39,4 @@ module LLM
     end
     Scout::Config.get(key, *all_tokens, hash)
   end
-
-  def self
-    if workflow.root.etc.AI[@model || 'default'].exists?
-      workflow.root.etc.AI[@model || 'default'].json
-    elsif Scout.etc.AI[@model || 'default'].exists?
-      Scout.etc.AI[@model || 'default'].json
-    else
-      {}
-    end
-  end
 end

data/lib/scout/model/base.rb

@@ -0,0 +1,19 @@
+require_relative 'util/save'
+require_relative 'util/run'
+
+class ScoutModel
+  attr_accessor :directory, :options, :state
+
+  def initialize(directory = nil, options={})
+    @options = options
+
+    if directory
+      directory = Path.setup directory.dup unless Path === directory
+      @directory = directory
+      restore
+    end
+
+    @features = []
+    @labels = []
+  end
+end

data/lib/scout/model/python/base.rb

@@ -0,0 +1,25 @@
+require_relative '../base'
+require 'scout/python'
+
+class PythonModel < ScoutModel
+  def initialize(dir, python_class = nil, python_module = nil, options = nil)
+    options, python_module = python_module, :model if options.nil? && Hash === python_module
+    options = {} if options.nil?
+
+    options[:python_class] = python_class if python_class
+    options[:python_module] = python_module if python_module
+
+    super(dir, options)
+
+    if options[:python_class]
+      self.init do
+        ScoutPython.add_path Scout.python.find(:lib)
+        ScoutPython.add_path directory
+        ScoutPython.init_scout
+        ScoutPython.class_new_obj(options[:python_module],
+                                  options[:python_class],
+                                  **options.except(:python_class, :python_module))
+      end
+    end
+  end
+end

data/lib/scout/model/python/huggingface/causal/next_token.rb

@@ -0,0 +1,23 @@
+require_relative '../causal'
+
+class NextTokenModel < CausalModel
+  def initialize(...)
+    super(...)
+
+    train do |texts|
+      model, tokenizer = @state
+
+      if self.directory
+        output_dir = self.directory['output'].find
+      else
+        output_dir = TmpFile.tmp_file "next_token_model"
+      end
+      dataset = ScoutPython.call_method(
+        "scout_ai.huggingface.data", :list_dataset, tokenizer, texts) 
+      ScoutPython.call_method(
+        "scout_ai.huggingface.train.next_token", :train_next_token, 
+        model:model, tokenizer:tokenizer, dataset:dataset, output_dir:output_dir, **options[:training_args]
+      )
+    end
+  end
+end

data/lib/scout/model/python/huggingface/causal.rb

@@ -0,0 +1,29 @@
+require_relative '../huggingface'
+
+class CausalModel < HuggingfaceModel
+  def initialize(...)
+    super("CausalLM", ...)
+
+    self.eval do |messages,list|
+      model, tokenizer = @state
+      ScoutPython.call_method(
+        "scout_ai.huggingface.eval", :eval_causal_lm_chat, 
+        model, tokenizer, messages, 
+        options[:chat_template],
+        options[:chat_template_kwargs], 
+        options[:generation_kwargs]
+      )
+    end
+
+    train do |pairs,labels|
+      # data: array of [response, reward] or [prompt, response, reward]
+      model, tokenizer = @state
+
+      ScoutPython.call_method(
+        "scout_ai.huggingface.rlhf", :train_rlhf, 
+        self.state_file, tokenizer, pairs, labels, options[:rlhf_config]
+      )
+      load_state
+    end
+  end
+end

data/lib/scout/model/python/huggingface/classification.rb

@@ -0,0 +1,50 @@
+require_relative '../huggingface'
+
+class SequenceClassificationModel < HuggingfaceModel
+  def initialize(...)
+    super("SequenceClassification", ...)
+
+    self.eval do |features,list|
+      model, tokenizer = @state
+      texts = list ? list : [features]
+      res = ScoutPython.call_method("scout_ai.huggingface.eval", :eval_model, model, tokenizer, texts, options[:locate_tokens])
+      list ? res : res[0]
+    end
+
+    post_process do |result,list|
+      model, tokenizer = @state
+
+      logit_list = list ? list.logits : result
+
+      res = ScoutPython.collect(logit_list) do |logits|
+        logits = ScoutPython.numpy2ruby logits
+        best_class = logits.index logits.max
+        best_class = options[:class_labels][best_class] if options[:class_labels]
+        best_class
+      end
+
+      list ? res : res[0]
+    end
+
+    train do |texts,labels| 
+      model, tokenizer = @state
+      
+      if directory
+        tsv_file = File.join(directory, 'dataset.tsv')
+        checkpoint_dir = File.join(directory, 'checkpoints')
+      else
+        tmpdir = TmpFile.tmp_file
+        Open.mkdir tmpdir
+        tsv_file = File.join(tmpdir, 'dataset.tsv')
+        checkpoint_dir = File.join(tmpdir, 'checkpoints')
+      end
+
+      training_args_obj = ScoutPython.call_method("scout_ai.huggingface.train", :training_args, checkpoint_dir, options[:training_args])
+      dataset_file = HuggingfaceModel.text_dataset(tsv_file, texts, labels, options[:class_labels])
+
+      ScoutPython.call_method("scout_ai.huggingface.train", :train_model, model, tokenizer, training_args_obj, dataset_file, options[:class_weights])
+
+      Open.rm_rf tmpdir if tmpdir
+    end
+  end
+end

data/lib/scout/model/python/huggingface.rb

@@ -0,0 +1,112 @@
+require_relative 'torch'
+
+class HuggingfaceModel < TorchModel
+
+  def fix_options
+    @options[:training_options] = @options.delete(:training_args) if @options.include?(:training_args)
+    @options[:training_options] = @options.delete(:training_kwargs) if @options.include?(:training_kwargs)
+    training_args = IndiferentHash.pull_keys(@options, :training) || {}
+
+    @options[:tokenizer_options] = @options.delete(:tokenizer_args) if @options.include?(:tokenizer_args)
+    @options[:tokenizer_options] = @options.delete(:tokenizer_kwargs) if @options.include?(:tokenizer_kwargs)
+    tokenizer_args = IndiferentHash.pull_keys(@options, :tokenizer) || {}
+
+    @options[:training_args] = training_args
+    @options[:tokenizer_args] = tokenizer_args
+  end
+
+  def initialize(task=nil, checkpoint=nil, dir = nil, options = {})
+    
+    super(dir, nil, nil, options)
+    
+    fix_options
+
+    options[:checkpoint] = checkpoint
+    options[:task] = task
+
+    init do 
+      TorchModel.init_python
+      checkpoint = state_file && File.directory?(state_file) ? state_file : self.options[:checkpoint]
+
+      model = ScoutPython.call_method("scout_ai.huggingface.model", :load_model, 
+                                      self.options[:task], checkpoint, 
+                                     **(IndiferentHash.setup(
+                                       self.options.except(
+                                         :training_args, :tokenizer_args, 
+                                         :task, :checkpoint, :class_labels, 
+                                         :model_options, :return_logits
+                                       ))))
+
+      tokenizer_checkpoint = self.options[:tokenizer_args][:checkpoint] || checkpoint
+
+      tokenizer = ScoutPython.call_method("scout_ai.huggingface.model", :load_tokenizer, 
+                                         tokenizer_checkpoint, 
+                                         **(IndiferentHash.setup(self.options[:tokenizer_args])))
+
+      [model, tokenizer]
+    end
+
+    load_state do |state_file|
+      model, tokenizer = @state
+      TorchModel.init_python
+      if state_file && Open.directory?(state_file)
+        model.from_pretrained(state_file)
+        tokenizer.from_pretrained(state_file)
+      end
+    end
+
+    save_state do |state_file,state|
+      model, tokenizer = @state
+      TorchModel.init_python
+      if state_file
+        model.save_pretrained(state_file)
+        tokenizer.save_pretrained(state_file)
+      end
+    end
+    
+    #self.eval do |features,list|
+    #  model, tokenizer = @state
+    #  res = case options[:task]
+    #        when "CausalLM"
+    #          if not list
+    #            list = [features]
+    #          end
+    #          # Allow for options :chat_template, :chat_template_kwargs, :generation_kwargs
+    #          #options[:generation_kwargs] = {max_new_tokens: 1000}
+    #          ScoutPython.call_method(
+    #            "scout_ai.huggingface.eval", :eval_causal_lm_chat, 
+    #            model, tokenizer, list, 
+    #            options[:chat_template],
+    #            options[:chat_template_kwargs], 
+    #            options[:generation_kwargs]
+    #          )
+    #        else
+    #          texts = list ? list : [features]
+    #          ScoutPython.call_method("scout_ai.huggingface.eval", :eval_model, model, tokenizer, texts, options[:locate_tokens])
+    #        end
+    #  list ? res : res[0]
+    #end
+
+    #train do |texts,labels| 
+    #  model, tokenizer = @state
+    #  
+    #  if directory
+    #    tsv_file = File.join(directory, 'dataset.tsv')
+    #    checkpoint_dir = File.join(directory, 'checkpoints')
+    #  else
+    #    tmpdir = TmpFile.tmp_file
+    #    Open.mkdir tmpdir
+    #    tsv_file = File.join(tmpdir, 'dataset.tsv')
+    #    checkpoint_dir = File.join(tmpdir, 'checkpoints')
+    #  end
+
+    #  training_args_obj = ScoutPython.call_method("scout_ai.huggingface.train", :training_args, checkpoint_dir, options[:training_args])
+    #  dataset_file = HuggingfaceModel.text_dataset(tsv_file, texts, labels, options[:class_labels])
+
+    #  ScoutPython.call_method("scout_ai.huggingface.train", :train_model, model, tokenizer, training_args_obj, dataset_file, options[:class_weights])
+
+    #  Open.rm_rf tmpdir if tmpdir
+    #end
+    
+  end
+end

data/lib/scout/model/python/torch/dataloader.rb

@@ -0,0 +1,57 @@
+class TorchModel
+  def self.feature_tsv(elements, labels = nil, class_labels = nil)
+    tsv = TSV.setup({}, :key_field => "ID", :fields => ["features"], :type => :flat)
+    if labels
+      tsv.fields = tsv.fields + ["label"]
+      labels = case class_labels
+               when Array
+                 labels.collect{|l| class_labels.index l}
+               when Hash
+                 inverse_class_labels = {}
+                 class_labels.each{|c,l| inverse_class_labels[l] = c }
+                 labels.collect{|l| inverse_class_labels[l]}
+               else
+                 labels
+               end
+      elements.zip(labels).each_with_index do |p,i|
+        features, label = p
+        id = i
+        if Array === features
+          tsv[id] = features + [label]
+        else
+          tsv[id] = [features, label]
+        end
+      end
+    else
+      elements.each_with_index do |features,i|
+        id = i
+        if Array === features
+          tsv[id] = features
+        else
+          tsv[id] = [features]
+        end
+      end
+    end
+    tsv
+  end
+
+  def self.feature_dataset(tsv_dataset_file, elements, labels = nil, class_labels = nil)
+    tsv = feature_tsv(elements, labels, class_labels)
+    Open.write(tsv_dataset_file, tsv.to_s)
+    tsv_dataset_file
+  end
+
+  def self.text_dataset(tsv_dataset_file, elements, labels = nil, class_labels = nil)
+    elements = elements.compact.collect{|e| e.gsub("\n", ' ').gsub('"', '\'') }
+    tsv = feature_tsv(elements, labels, class_labels)
+    tsv.fields[0] = "text"
+    if labels.nil?
+      tsv = tsv.to_single
+    else
+      tsv.type = :list
+    end
+    Open.write(tsv_dataset_file, tsv.to_s)
+    tsv_dataset_file
+  end
+
+end

data/lib/scout/model/python/torch/helpers.rb

@@ -0,0 +1,84 @@
+require 'scout/python'
+
+class TorchModel
+  module Tensor
+    def to_ruby
+      ScoutPython.numpy2ruby(self)
+    end
+
+    def to_ruby!
+      r = self.to_ruby
+      self.del
+      r
+    end
+
+    def length
+      PyCall.len(self)
+    end
+
+    def self.setup(obj)
+      obj.extend Tensor
+    end
+
+    def del
+      begin
+        self.to("cpu")
+        self.detach
+        self.grad = nil
+        self.untyped_storage.resize_ 0
+      rescue Exception
+        Log.exception $!
+      end
+      self
+    end
+  end
+
+  def self.init_python
+    return if defined?(@@init_python) && @@init_python
+    ScoutPython.add_path Scout.python.find(:lib)
+    ScoutPython.init_scout
+    ScoutPython.pyimport :torch
+    ScoutPython.pyimport :scout
+    ScoutPython.pyimport :scout_ai
+    ScoutPython.pyfrom :scout_ai, import: :util
+    ScoutPython.pyfrom :torch, import: :nn
+    @@init_python = true
+  end
+
+  def self.optimizer(model, training_args = {})
+    begin
+      learning_rate = training_args[:learning_rate] || 0.01
+      ScoutPython.torch.optim.SGD.new(model.parameters(), lr: learning_rate)
+    end
+  end
+
+  def self.criterion(model, training_args = {})
+    ScoutPython.torch.nn.MSELoss.new()
+  end
+
+  def self.device(model_options)
+    case model_options[:device]
+    when String, Symbol
+      ScoutPython.torch.device(model_options[:device].to_s)
+    when nil
+      ScoutPython.scout_ai.util.device()
+    else
+        model_options[:device]
+    end
+  end
+
+  def self.dtype(model_options)
+    case model_options[:dtype]
+    when String, Symbol
+      ScoutPython.torch.call(model_options[:dtype])
+    when nil
+      nil
+    else
+      model_options[:dtype]
+    end
+  end
+
+  def self.tensor(obj, device, dtype)
+    TorchModel::Tensor.setup(ScoutPython.torch.tensor(obj, dtype: dtype, device: device))
+  end
+end