scout-ai 0.2.0 → 1.0.0

data/lib/scout/llm/tools.rb

@@ -8,7 +8,6 @@ module LLM
     function_arguments = JSON.parse(function_arguments, { symbolize_names: true }) if String === function_arguments
     function_response = block.call function_name, function_arguments
 
-    #content = String === function_response ? function_response : function_response.to_json,
     content = case function_response
               when String
                 function_response
@@ -17,21 +16,28 @@ module LLM
               else
                 function_response.to_json
               end
+    content = content.to_s if Numeric === content
     {
-      tool_call_id: tool_call_id,
+      id: tool_call_id,
       role: "tool",
       content: content
     }
   end
 
-  def self.task_tool_definition(workflow, task_name)
+  def self.task_tool_definition(workflow, task_name, inputs = nil)
     task_info = workflow.task_info(task_name)
 
+    inputs = inputs.collect{|i| i.to_sym } if inputs
+
     properties = task_info[:inputs].inject({}) do |acc,input|
+      next acc if inputs and not inputs.include?(input)
       type = task_info[:input_types][input]
       description = task_info[:input_descriptions][input]
 
+      type = :string if type == :text
       type = :string if type == :select
+      type = :string if type == :path
+      type = :number if type == :float
 
       acc[input] = {
         "type": type,
@@ -49,7 +55,8 @@ module LLM
     end
 
     required_inputs = task_info[:inputs].select do |input|
-      task_info[:input_options].include?(input) && task_info[:input_options][:required]
+      next if inputs and not inputs.include?(input.to_sym)
+      task_info[:input_options].include?(input) && task_info[:input_options][input][:required]
     end
 
     {
@@ -67,7 +74,7 @@ module LLM
   end
 
   def self.workflow_tools(workflow, tasks = nil)
-    tasks ||= workflow.tasks.keys
+    tasks = workflow.all_exports
     tasks.collect{|task_name| self.task_tool_definition(workflow, task_name) }
   end
 
@@ -101,4 +108,118 @@ module LLM
       }
     }]
   end
+
+  def self.association_tool_definition(name)
+    properties = {
+      entities: {
+        type: "array",
+        items: { type: :string },
+        description: "Source entities in the association, or target entities if 'reverse' it true."
+      },
+      reverse: {
+        type: "boolean",
+        description: "Look for targets instead of sources, defaults to 'false'."
+      }
+    }
+
+    {
+      type: "function",
+      function: {
+        name: name,
+        description: "Find associations for a list of entities. Returns a list in the format source~target.",
+        parameters: {
+          type: "object",
+          properties: properties,
+          required: ['entities']
+        }
+      }
+    }
+  end
+  
+  def self.run_tools(messages)
+    messages.collect do |info|
+      IndiferentHash.setup(info)
+      role = info[:role]
+      if role == 'cmd'
+        {
+          role: 'tool',
+          content: CMD.cmd(info[:content]).read
+        }
+      else
+        info
+      end
+    end
+  end
+
+  def self.tools_to_openai(messages)
+    messages.collect do |message|
+      if message[:role] == 'function_call'
+        tool_call = JSON.parse(message[:content])
+        arguments = tool_call.delete('arguments') || {}
+        name = tool_call.delete('name')
+        tool_call['type'] = 'function'
+        tool_call['function'] ||= {}
+        tool_call['function']['name'] ||= name
+        tool_call['function']['arguments'] = arguments.to_json
+        {role: 'assistant', tool_calls: [tool_call]}
+      elsif message[:role] == 'function_call_output'
+        info = JSON.parse(message[:content])
+        id = info.delete('id') || ''
+        info['role'] = 'tool'
+        info['tool_call_id'] = id
+        info
+      else
+        message
+      end
+    end.flatten
+  end
+
+  def self.tools_to_ollama(messages)
+    messages.collect do |message|
+      if message[:role] == 'function_call'
+        tool_call = JSON.parse(message[:content])
+        arguments = tool_call.delete('arguments') || {}
+        id = tool_call.delete('id')
+        name = tool_call.delete('name')
+        tool_call['type'] = 'function'
+        tool_call['function'] ||= {}
+        tool_call['function']['name'] ||= name
+        tool_call['function']['arguments'] ||= arguments
+        {role: 'assistant', tool_calls: [tool_call]}
+      elsif message[:role] == 'function_call_output'
+        info = JSON.parse(message[:content])
+        id = info.delete('id') || ''
+        info['role'] = 'tool'
+        info
+      else
+        message
+      end
+    end.flatten
+  end
+
+  def self.tools_to_responses(messages)
+    messages.collect do |message|
+      if message[:role] == 'function_call'
+        tool_call = JSON.parse(message[:content])
+        tool_call['function']['arguments'] = (tool_call['function']['arguments'] || {}).to_json
+        {role: 'assistant', tool_calls: [tool_call]}
+      elsif message[:role] == 'function_call_output'
+        info = JSON.parse(message[:content])
+        info["tool_call_id"] = info['id']
+        info
+      else
+        message
+      end
+    end.flatten
+  end
+
+  def self.call_tools(tool_calls, &block)
+    tool_calls.collect{|tool_call|
+      response_message = LLM.tool_response(tool_call, &block)
+      [
+        {role: "function_call", content: tool_call.to_json},
+        {role: "function_call_output", content: response_message.to_json},
+      ]
+    }.flatten
+  end
 end

data/lib/scout/llm/utils.rb

@@ -1,4 +1,21 @@
 module LLM
+
+  def self.tag(tag, content, name = nil)
+    if name
+      <<-EOF.strip
+<#{tag} name="#{name}">
+#{content}
+</#{tag}>
+      EOF
+    else
+      <<-EOF.strip
+<#{tag}>
+#{content}
+</#{tag}>
+      EOF
+    end
+  end
+
   def self.get_url_server_tokens(url, prefix=nil)
     return get_url_server_tokens(url).collect{|e| prefix.to_s + "." + e } if prefix
 
@@ -22,14 +39,4 @@ module LLM
     end
     Scout::Config.get(key, *all_tokens, hash)
   end
-
-  def self
-    if workflow.root.etc.AI[@model || 'default'].exists?
-      workflow.root.etc.AI[@model || 'default'].json
-    elsif Scout.etc.AI[@model || 'default'].exists?
-      Scout.etc.AI[@model || 'default'].json
-    else
-      {}
-    end
-  end
 end

data/lib/scout/model/base.rb

@@ -0,0 +1,19 @@
+require_relative 'util/save'
+require_relative 'util/run'
+
+class ScoutModel
+  attr_accessor :directory, :options, :state
+
+  def initialize(directory = nil, options={})
+    @options = options
+
+    if directory
+      directory = Path.setup directory.dup unless Path === directory
+      @directory = directory
+      restore
+    end
+
+    @features = []
+    @labels = []
+  end
+end

data/lib/scout/model/python/base.rb

@@ -0,0 +1,25 @@
+require_relative '../base'
+require 'scout/python'
+
+class PythonModel < ScoutModel
+  def initialize(dir, python_class = nil, python_module = nil, options = nil)
+    options, python_module = python_module, :model if options.nil? && Hash === python_module
+    options = {} if options.nil?
+
+    options[:python_class] = python_class if python_class
+    options[:python_module] = python_module if python_module
+
+    super(dir, options)
+
+    if options[:python_class]
+      self.init do
+        ScoutPython.add_path Scout.python.find(:lib)
+        ScoutPython.add_path directory
+        ScoutPython.init_scout
+        ScoutPython.class_new_obj(options[:python_module],
+                                  options[:python_class],
+                                  **options.except(:python_class, :python_module))
+      end
+    end
+  end
+end

data/lib/scout/model/python/huggingface/causal/next_token.rb

@@ -0,0 +1,23 @@
+require_relative '../causal'
+
+class NextTokenModel < CausalModel
+  def initialize(...)
+    super(...)
+
+    train do |texts|
+      model, tokenizer = @state
+
+      if self.directory
+        output_dir = self.directory['output'].find
+      else
+        output_dir = TmpFile.tmp_file "next_token_model"
+      end
+      dataset = ScoutPython.call_method(
+        "scout_ai.huggingface.data", :list_dataset, tokenizer, texts) 
+      ScoutPython.call_method(
+        "scout_ai.huggingface.train.next_token", :train_next_token, 
+        model:model, tokenizer:tokenizer, dataset:dataset, output_dir:output_dir, **options[:training_args]
+      )
+    end
+  end
+end

data/lib/scout/model/python/huggingface/causal.rb

@@ -0,0 +1,29 @@
+require_relative '../huggingface'
+
+class CausalModel < HuggingfaceModel
+  def initialize(...)
+    super("CausalLM", ...)
+
+    self.eval do |messages,list|
+      model, tokenizer = @state
+      ScoutPython.call_method(
+        "scout_ai.huggingface.eval", :eval_causal_lm_chat, 
+        model, tokenizer, messages, 
+        options[:chat_template],
+        options[:chat_template_kwargs], 
+        options[:generation_kwargs]
+      )
+    end
+
+    train do |pairs,labels|
+      # data: array of [response, reward] or [prompt, response, reward]
+      model, tokenizer = @state
+
+      ScoutPython.call_method(
+        "scout_ai.huggingface.rlhf", :train_rlhf, 
+        self.state_file, tokenizer, pairs, labels, options[:rlhf_config]
+      )
+      load_state
+    end
+  end
+end

data/lib/scout/model/python/huggingface/classification.rb

@@ -0,0 +1,50 @@
+require_relative '../huggingface'
+
+class SequenceClassificationModel < HuggingfaceModel
+  def initialize(...)
+    super("SequenceClassification", ...)
+
+    self.eval do |features,list|
+      model, tokenizer = @state
+      texts = list ? list : [features]
+      res = ScoutPython.call_method("scout_ai.huggingface.eval", :eval_model, model, tokenizer, texts, options[:locate_tokens])
+      list ? res : res[0]
+    end
+
+    post_process do |result,list|
+      model, tokenizer = @state
+
+      logit_list = list ? list.logits : result
+
+      res = ScoutPython.collect(logit_list) do |logits|
+        logits = ScoutPython.numpy2ruby logits
+        best_class = logits.index logits.max
+        best_class = options[:class_labels][best_class] if options[:class_labels]
+        best_class
+      end
+
+      list ? res : res[0]
+    end
+
+    train do |texts,labels| 
+      model, tokenizer = @state
+      
+      if directory
+        tsv_file = File.join(directory, 'dataset.tsv')
+        checkpoint_dir = File.join(directory, 'checkpoints')
+      else
+        tmpdir = TmpFile.tmp_file
+        Open.mkdir tmpdir
+        tsv_file = File.join(tmpdir, 'dataset.tsv')
+        checkpoint_dir = File.join(tmpdir, 'checkpoints')
+      end
+
+      training_args_obj = ScoutPython.call_method("scout_ai.huggingface.train", :training_args, checkpoint_dir, options[:training_args])
+      dataset_file = HuggingfaceModel.text_dataset(tsv_file, texts, labels, options[:class_labels])
+
+      ScoutPython.call_method("scout_ai.huggingface.train", :train_model, model, tokenizer, training_args_obj, dataset_file, options[:class_weights])
+
+      Open.rm_rf tmpdir if tmpdir
+    end
+  end
+end

data/lib/scout/model/python/huggingface.rb

@@ -0,0 +1,112 @@
+require_relative 'torch'
+
+class HuggingfaceModel < TorchModel
+
+  def fix_options
+    @options[:training_options] = @options.delete(:training_args) if @options.include?(:training_args)
+    @options[:training_options] = @options.delete(:training_kwargs) if @options.include?(:training_kwargs)
+    training_args = IndiferentHash.pull_keys(@options, :training) || {}
+
+    @options[:tokenizer_options] = @options.delete(:tokenizer_args) if @options.include?(:tokenizer_args)
+    @options[:tokenizer_options] = @options.delete(:tokenizer_kwargs) if @options.include?(:tokenizer_kwargs)
+    tokenizer_args = IndiferentHash.pull_keys(@options, :tokenizer) || {}
+
+    @options[:training_args] = training_args
+    @options[:tokenizer_args] = tokenizer_args
+  end
+
+  def initialize(task=nil, checkpoint=nil, dir = nil, options = {})
+    
+    super(dir, nil, nil, options)
+    
+    fix_options
+
+    options[:checkpoint] = checkpoint
+    options[:task] = task
+
+    init do 
+      TorchModel.init_python
+      checkpoint = state_file && File.directory?(state_file) ? state_file : self.options[:checkpoint]
+
+      model = ScoutPython.call_method("scout_ai.huggingface.model", :load_model, 
+                                      self.options[:task], checkpoint, 
+                                     **(IndiferentHash.setup(
+                                       self.options.except(
+                                         :training_args, :tokenizer_args, 
+                                         :task, :checkpoint, :class_labels, 
+                                         :model_options, :return_logits
+                                       ))))
+
+      tokenizer_checkpoint = self.options[:tokenizer_args][:checkpoint] || checkpoint
+
+      tokenizer = ScoutPython.call_method("scout_ai.huggingface.model", :load_tokenizer, 
+                                         tokenizer_checkpoint, 
+                                         **(IndiferentHash.setup(self.options[:tokenizer_args])))
+
+      [model, tokenizer]
+    end
+
+    load_state do |state_file|
+      model, tokenizer = @state
+      TorchModel.init_python
+      if state_file && Open.directory?(state_file)
+        model.from_pretrained(state_file)
+        tokenizer.from_pretrained(state_file)
+      end
+    end
+
+    save_state do |state_file,state|
+      model, tokenizer = @state
+      TorchModel.init_python
+      if state_file
+        model.save_pretrained(state_file)
+        tokenizer.save_pretrained(state_file)
+      end
+    end
+    
+    #self.eval do |features,list|
+    #  model, tokenizer = @state
+    #  res = case options[:task]
+    #        when "CausalLM"
+    #          if not list
+    #            list = [features]
+    #          end
+    #          # Allow for options :chat_template, :chat_template_kwargs, :generation_kwargs
+    #          #options[:generation_kwargs] = {max_new_tokens: 1000}
+    #          ScoutPython.call_method(
+    #            "scout_ai.huggingface.eval", :eval_causal_lm_chat, 
+    #            model, tokenizer, list, 
+    #            options[:chat_template],
+    #            options[:chat_template_kwargs], 
+    #            options[:generation_kwargs]
+    #          )
+    #        else
+    #          texts = list ? list : [features]
+    #          ScoutPython.call_method("scout_ai.huggingface.eval", :eval_model, model, tokenizer, texts, options[:locate_tokens])
+    #        end
+    #  list ? res : res[0]
+    #end
+
+    #train do |texts,labels| 
+    #  model, tokenizer = @state
+    #  
+    #  if directory
+    #    tsv_file = File.join(directory, 'dataset.tsv')
+    #    checkpoint_dir = File.join(directory, 'checkpoints')
+    #  else
+    #    tmpdir = TmpFile.tmp_file
+    #    Open.mkdir tmpdir
+    #    tsv_file = File.join(tmpdir, 'dataset.tsv')
+    #    checkpoint_dir = File.join(tmpdir, 'checkpoints')
+    #  end
+
+    #  training_args_obj = ScoutPython.call_method("scout_ai.huggingface.train", :training_args, checkpoint_dir, options[:training_args])
+    #  dataset_file = HuggingfaceModel.text_dataset(tsv_file, texts, labels, options[:class_labels])
+
+    #  ScoutPython.call_method("scout_ai.huggingface.train", :train_model, model, tokenizer, training_args_obj, dataset_file, options[:class_weights])
+
+    #  Open.rm_rf tmpdir if tmpdir
+    #end
+    
+  end
+end

data/lib/scout/model/python/torch/dataloader.rb

@@ -0,0 +1,57 @@
+class TorchModel
+  def self.feature_tsv(elements, labels = nil, class_labels = nil)
+    tsv = TSV.setup({}, :key_field => "ID", :fields => ["features"], :type => :flat)
+    if labels
+      tsv.fields = tsv.fields + ["label"]
+      labels = case class_labels
+               when Array
+                 labels.collect{|l| class_labels.index l}
+               when Hash
+                 inverse_class_labels = {}
+                 class_labels.each{|c,l| inverse_class_labels[l] = c }
+                 labels.collect{|l| inverse_class_labels[l]}
+               else
+                 labels
+               end
+      elements.zip(labels).each_with_index do |p,i|
+        features, label = p
+        id = i
+        if Array === features
+          tsv[id] = features + [label]
+        else
+          tsv[id] = [features, label]
+        end
+      end
+    else
+      elements.each_with_index do |features,i|
+        id = i
+        if Array === features
+          tsv[id] = features
+        else
+          tsv[id] = [features]
+        end
+      end
+    end
+    tsv
+  end
+
+  def self.feature_dataset(tsv_dataset_file, elements, labels = nil, class_labels = nil)
+    tsv = feature_tsv(elements, labels, class_labels)
+    Open.write(tsv_dataset_file, tsv.to_s)
+    tsv_dataset_file
+  end
+
+  def self.text_dataset(tsv_dataset_file, elements, labels = nil, class_labels = nil)
+    elements = elements.compact.collect{|e| e.gsub("\n", ' ').gsub('"', '\'') }
+    tsv = feature_tsv(elements, labels, class_labels)
+    tsv.fields[0] = "text"
+    if labels.nil?
+      tsv = tsv.to_single
+    else
+      tsv.type = :list
+    end
+    Open.write(tsv_dataset_file, tsv.to_s)
+    tsv_dataset_file
+  end
+
+end

data/lib/scout/model/python/torch/helpers.rb

@@ -0,0 +1,84 @@
+require 'scout/python'
+
+class TorchModel
+  module Tensor
+    def to_ruby
+      ScoutPython.numpy2ruby(self)
+    end
+
+    def to_ruby!
+      r = self.to_ruby
+      self.del
+      r
+    end
+
+    def length
+      PyCall.len(self)
+    end
+
+    def self.setup(obj)
+      obj.extend Tensor
+    end
+
+    def del
+      begin
+        self.to("cpu")
+        self.detach
+        self.grad = nil
+        self.untyped_storage.resize_ 0
+      rescue Exception
+        Log.exception $!
+      end
+      self
+    end
+  end
+
+  def self.init_python
+    return if defined?(@@init_python) && @@init_python
+    ScoutPython.add_path Scout.python.find(:lib)
+    ScoutPython.init_scout
+    ScoutPython.pyimport :torch
+    ScoutPython.pyimport :scout
+    ScoutPython.pyimport :scout_ai
+    ScoutPython.pyfrom :scout_ai, import: :util
+    ScoutPython.pyfrom :torch, import: :nn
+    @@init_python = true
+  end
+
+  def self.optimizer(model, training_args = {})
+    begin
+      learning_rate = training_args[:learning_rate] || 0.01
+      ScoutPython.torch.optim.SGD.new(model.parameters(), lr: learning_rate)
+    end
+  end
+
+  def self.criterion(model, training_args = {})
+    ScoutPython.torch.nn.MSELoss.new()
+  end
+
+  def self.device(model_options)
+    case model_options[:device]
+    when String, Symbol
+      ScoutPython.torch.device(model_options[:device].to_s)
+    when nil
+      ScoutPython.scout_ai.util.device()
+    else
+        model_options[:device]
+    end
+  end
+
+  def self.dtype(model_options)
+    case model_options[:dtype]
+    when String, Symbol
+      ScoutPython.torch.call(model_options[:dtype])
+    when nil
+      nil
+    else
+      model_options[:dtype]
+    end
+  end
+
+  def self.tensor(obj, device, dtype)
+    TorchModel::Tensor.setup(ScoutPython.torch.tensor(obj, dtype: dtype, device: device))
+  end
+end

data/lib/scout/model/python/torch/introspection.rb

@@ -0,0 +1,34 @@
+require_relative 'helpers'
+class TorchModel
+  def self.get_layer(state, layer = nil)
+    state = state.first if Array === state
+    if layer.nil?
+      state
+    else
+      layer.split(".").inject(state){|acc,l| PyCall.getattr(acc, l.to_sym) }
+    end
+  end
+  def get_layer(...); TorchModel.get_layer(state, ...); end
+
+  def self.get_weights(state, layer = nil)
+    Tensor.setup PyCall.getattr(get_layer(state, layer), :weight)
+  end
+  def get_weights(...); TorchModel.get_weights(state, ...); end
+
+  def self.freeze(layer, requires_grad=false)
+    begin
+      PyCall.getattr(layer, :weight).requires_grad = requires_grad
+    rescue
+    end
+    ScoutPython.iterate(layer.children) do |layer|
+      freeze(layer, requires_grad)
+    end
+  end
+
+  def self.freeze_layer(state, layer, requires_grad = false)
+    layer = get_layer(state, layer)
+    freeze(layer, requires_grad)
+  end
+
+  def freeze_layer(...); TorchModel.freeze_layer(state, ...); end
+end