scout-ai 0.2.0 → 1.0.1

data/test/scout/llm/test_chat.rb

@@ -0,0 +1,256 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestMessages < Test::Unit::TestCase
+
+  def test_short
+
+    question =<<-EOF
+Hi
+    EOF
+
+    iii LLM.chat(question)
+  end
+
+  def test_inline
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+assistant:
+
+Here is some stuff
+
+user: feedback
+
+that continues here
+    EOF
+
+    iii LLM.chat(question)
+  end
+
+  def test_messages
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+user:
+
+What is the capital of France
+
+assistant:
+
+Paris
+
+user:
+
+is this the national anthem
+
+[[
+corous: Viva Espagna
+]]
+
+assistant:
+
+no
+
+user:
+
+import: math.system
+
+consider this file
+
+<file name=foo_bar>
+foo: bar
+</file>
+
+how many characters does it hold
+
+assistant:
+
+8
+    EOF
+
+    messages = LLM.messages question
+    refute messages.collect{|i| i[:role] }.include?("corous")
+    assert messages.collect{|i| i[:role] }.include?("import")
+  end
+
+  def test_chat_import
+    file1 =<<-EOF
+system: You are an assistant
+    EOF
+
+    file2 =<<-EOF
+import: header
+user: say something
+    EOF
+
+    TmpFile.with_path do |tmpdir|
+      tmpdir.header.write file1
+      tmpdir.chat.write file2
+
+      chat = LLM.chat tmpdir.chat
+    end
+  end
+
+  def test_clear
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+clear:
+
+user:
+
+What is the capital of France
+    EOF
+
+    TmpFile.with_file question do |file|
+      messages = LLM.chat file
+      refute messages.collect{|m| m[:role] }.include?('system')
+    end
+  end
+
+  def __test_job
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+job: Baking/bake_muffin_tray/Default_08a1812eca3a18dce2232509dabc9b41
+
+How are muffins made
+
+    EOF
+
+    TmpFile.with_file question do |file|
+      messages = LLM.chat file
+      ppp LLM.print messages
+    end
+  end
+
+
+  def test_task
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+user:
+
+task: Baking bake_muffin_tray blueberries=true title="This is a title" list=one,two,"and three"
+
+How are muffins made?
+
+    EOF
+
+    TmpFile.with_file question do |file|
+      messages = LLM.chat file
+      ppp LLM.print messages
+    end
+  end
+
+  def test_structure
+    require 'scout/llm/ask'
+    sss 0
+    question =<<-EOF
+system:
+
+Respond in json format with a hash of strings as keys and string arrays as values, at most three in length
+
+endpoint: sambanova
+
+What other movies have the protagonists of the original gost busters played on, just the top.
+
+    EOF
+
+    TmpFile.with_file question do |file|
+      ppp LLM.ask file
+    end
+  end
+
+  def test_tool
+    require 'scout/llm/ask'
+
+    sss 0
+    question =<<-EOF
+user:
+
+Use the provided tool to learn the instructions of baking a tray of muffins. Don't
+give me your own recipe, return the one provided by the tool
+
+tool: Baking
+    EOF
+
+    TmpFile.with_file question do |file|
+      ppp LLM.ask file, endpoint: :nano
+    end
+  end
+
+  def test_tools_with_task
+    require 'scout/llm/ask'
+
+    question =<<-EOF
+user:
+
+Use the provided tool to learn the instructions of baking a tray of muffins. Don't
+give me your own recipe, return the one provided by the tool
+
+tool: Baking bake_muffin_tray
+    EOF
+
+    TmpFile.with_file question do |file|
+      ppp LLM.ask file
+    end
+  end
+
+  def test_knowledge_base
+    require 'scout/llm/ask'
+    sss 0
+    question =<<-EOF
+system:
+
+Query the knowledge base of familiar relationships to answer the question
+
+user:
+
+Who is Miki's brother in law?
+
+association: brothers #{datafile_test(:person).brothers} undirected=true
+association: marriages #{datafile_test(:person).marriages} undirected=true source="=>Alias" target="=>Alias"
+    EOF
+
+    TmpFile.with_file question do |file|
+      ppp LLM.ask file
+    end
+  end
+
+  def test_previous_response
+    require 'scout/llm/ask'
+    sss 0
+    question =<<-EOF
+user:
+
+Say hi
+
+assistant:
+
+Hi
+
+previous_response_id: asdfasdfasdfasdf
+
+Bye
+
+    EOF
+
+    messages = LLM.messages question
+
+    iii messages
+
+  end
+end
+

data/test/scout/llm/test_mcp.rb

@@ -0,0 +1,29 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require "scout-ai"
+class TestMCP < Test::Unit::TestCase
+  def test_workflow_stdio
+    require "mcp/server/transports/stdio_transport"
+    wf = Module.new do
+      extend Workflow
+      self.name = "TestWorkflow"
+
+      desc "Just say hi to someone"
+      input :name, :string, "Name", nil, required: true
+      task :hi => :string do |name|
+        "Hi #{name}"
+      end
+
+      desc "Just say bye to someone"
+      input :name, :string, "Name", nil, required: true
+      task :bye => :string do |name|
+        "Bye #{name}"
+      end
+    end
+
+    transport = MCP::Server::Transports::StdioTransport.new(wf.mcp(:hi))
+    transport.open
+  end
+end
+

data/test/scout/llm/test_parse.rb

@@ -4,8 +4,9 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 class TestLLMParse < Test::Unit::TestCase
   def test_parse
     text=<<-EOF
+hi
 system: you are an asistant
-user: Given the contents of this file: [[ 
+user: Given the contents of this file:[[
 line 1: 1
 line 2: 2
 line 3: 3
@@ -13,7 +14,85 @@ line 3: 3
 Show me the lines in reverse order
     EOF
 
-    iii LLM.parse(text)
+    assert_include LLM.parse(text).first[:content], 'hi'
+    assert_include LLM.parse(text).last[:content], 'reverse'
+  end
+
+  def test_code
+    text=<<-EOF
+hi
+system: you are an asistant
+user: Given the contents of this file:
+```yaml
+key: value
+key2: value2
+```
+Show me the lines in reverse order
+    EOF
+
+    assert_include LLM.parse(text).last[:content], 'key2'
+  end
+
+  def test_lines
+    text=<<-EOF
+system: you are an asistant
+user: I have a question
+    EOF
+
+    assert_include LLM.parse(text).last[:content], 'question'
+  end
+
+  def test_blocks
+    text=<<-EOF
+system:
+
+you are an asistant
+
+user:
+
+I have a question
+
+    EOF
+
+    assert_include LLM.parse(text).last[:content], 'question'
+  end
+
+  def test_no_role
+    text=<<-EOF
+I have a question
+    EOF
+
+    assert_include LLM.parse(text).last[:content], 'question'
+  end
+
+
+  def test_cmd
+    text=<<-EOF
+How many files are there:
+
+[[cmd list of files
+echo "file1 file2"
+]]
+    EOF
+
+    assert_equal :user, LLM.parse(text).last[:role]
+    assert_include LLM.parse(text).first[:content], 'file1'
+  end
+
+  def test_directory
+    TmpFile.with_path do |tmpdir|
+      tmpdir.file1.write "foo"
+      tmpdir.file2.write "bar"
+      text=<<-EOF
+How many files are there:
+
+[[directory DIR
+#{tmpdir}
+]]
+      EOF
+
+      assert_include LLM.parse(text).first[:content], 'file1'
+    end
   end
 end
 

data/test/scout/llm/tools/test_knowledge_base.rb

@@ -0,0 +1,22 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestLLMToolKB < Test::Unit::TestCase
+  def test_knowledbase_definition
+    TmpFile.with_dir do |dir|
+      kb = KnowledgeBase.new dir
+      kb.register :brothers, datafile_test(:person).brothers, undirected: true
+      kb.register :parents, datafile_test(:person).parents
+
+      assert_include kb.all_databases, :brothers
+
+      assert_equal Person, kb.target_type(:parents)
+
+      knowledge_base_definition = LLM.knowledge_base_tool_definition(kb)
+      ppp JSON.pretty_generate knowledge_base_definition
+
+      assert_equal ['Isa~Miki', 'Miki~Isa', 'Guille~Clei'], LLM.call_knowledge_base(kb, :brothers, entities: %w(Isa Miki Guille))
+    end
+  end
+end
+

data/test/scout/llm/tools/test_mcp.rb

@@ -0,0 +1,11 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_client
+   c = LLM.mcp_tools("https://api.githubcopilot.com/mcp/")
+   assert_include c.keys, "get_me"
+  end
+end
+
+

data/test/scout/llm/tools/test_workflow.rb

@@ -0,0 +1,39 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestLLMToolWorkflow < Test::Unit::TestCase
+  def test_workflow_definition
+    m = Module.new do
+      extend Workflow
+      self.name = "RecipeWorkflow"
+
+      desc "List the steps to cook a recipe"
+      input :recipe, :string, "Recipe for which to extract steps"
+      task :recipe_steps => :array do |recipe|
+        ["prepare batter", "bake"]
+      end
+
+      desc "Calculate time spent in each step of the recipe"
+      input :step, :string, "Cooking step"
+      task :step_time => :string do |step|
+        case step 
+        when "prepare batter"
+          "2 hours"
+        when "bake"
+          "30 minutes"
+        else
+          "1 minute"
+        end
+      end
+    end
+
+    LLM.task_tool_definition(m, :recipe_steps)
+    LLM.task_tool_definition(m, :step_time)
+
+    tool_definitions = LLM.workflow_tools(m)
+    ppp JSON.pretty_generate tool_definitions
+
+    assert_equal ["prepare batter", "bake"], LLM.call_workflow(m, :recipe_steps)
+  end
+end
+

data/test/scout/model/python/huggingface/causal/test_next_token.rb

@@ -0,0 +1,59 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout-ai'
+class TestClass < Test::Unit::TestCase
+  def test_main
+    model = NextTokenModel.new 
+    train_texts = [
+        "say hi, no!",
+        "say hi, no no no",
+        "say hi, hi ",
+        "say hi, hi how are you ",
+        "say hi, hi are you good",
+    ] 
+
+    model_name = "distilgpt2"  # Replace with your local/other HF Llama checkpoint as needed
+
+    TmpFile.with_path do |tmp_dir|
+      iii tmp_dir
+
+      sss 0
+      model = NextTokenModel.new model_name, tmp_dir, training_num_train_epochs: 1000, training_learning_rate: 0.1
+
+      iii :new
+      chat = Chat.setup []
+      chat.user "say hi"
+      ppp model.eval chat
+
+      model.save
+      model = PythonModel.new tmp_dir
+
+      iii :load
+      chat = Chat.setup []
+      chat.user "say hi"
+      ppp model.eval chat
+
+      iii :training
+      state, tokenizer = model.init
+      tokenizer.pad_token = tokenizer.eos_token
+      model.add_list train_texts.shuffle
+      model.train
+
+      iii :trained
+      chat = Chat.setup []
+      chat.user "say hi"
+      ppp model.eval chat
+
+      model.save
+      model = PythonModel.new tmp_dir
+
+      iii :load_again
+      chat = Chat.setup []
+      chat.user "say hi"
+      ppp model.eval chat
+    end
+
+  end
+end
+

data/test/scout/model/python/huggingface/test_causal.rb

@@ -0,0 +1,33 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_eval_chat
+    #model = CausalModel.new 'BSC-LT/salamandra-2b-instruct'
+    model = CausalModel.new 'mistralai/Mistral-7B-Instruct-v0.3'
+
+    model.init
+
+    net, tok = model.state
+
+    iii model.eval([
+      {role: :system, content: "You are a calculator, just reply with the answer"},
+      {role: :user, content: " 1 + 2 ="}
+    ])
+  end
+
+  def test_eval_train
+    #model = CausalModel.new 'BSC-LT/salamandra-2b-instruct'
+    model = CausalModel.new 'mistralai/Mistral-7B-Instruct-v0.3'
+
+    model.init
+
+    net, tok = model.state
+
+    iii model.eval([
+      {role: :system, content: "You are a calculator, just reply with the answer"},
+      {role: :user, content: " 1 + 2 ="}
+    ])
+  end
+end
+

data/test/scout/model/python/huggingface/test_classification.rb

@@ -0,0 +1,30 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSequenceClassification < Test::Unit::TestCase
+  def _test_eval_sequence_classification
+    model = SequenceClassificationModel.new 'bert-base-uncased', nil,
+      class_labels: %w(Bad Good)
+
+    assert_include ["Bad", "Good"], model.eval("This is dog")
+    assert_include ["Bad", "Good"], model.eval_list(["This is dog", "This is cat"]).first
+  end
+
+  def test_train_sequence_classification
+    model = SequenceClassificationModel.new 'bert-base-uncased', nil,
+      class_labels: %w(Bad Good)
+
+    model.init
+
+    10.times do
+      model.add "The dog", 'Bad'
+      model.add "The cat", 'Good'
+    end
+
+    model.train
+
+    assert_equal "Bad", model.eval("This is dog")
+    assert_equal "Good", model.eval("This is cat")
+  end
+end
+

data/test/scout/model/python/test_base.rb

@@ -0,0 +1,44 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestPythonModel < Test::Unit::TestCase
+  def test_linear
+    model = nil
+
+    TmpFile.with_path do |dir|
+
+      dir['model.py'].write <<-EOF
+class TestModel:
+  def __init__(self, delta):
+    self.delta = delta
+
+  def eval(self, x):
+    return [e + self.delta for e in x]
+      EOF
+
+      model = PythonModel.new dir, 'TestModel', :model, delta: 1
+
+      model.eval do |sample,list=nil|
+        init unless state
+        if list
+          state.eval(list)
+        else
+          state.eval([sample])[0]
+        end
+      end
+
+      assert_equal 2, model.eval(1)
+      assert_equal [4, 6], model.eval_list([3, 5])
+
+      model.save
+
+      model = ScoutModel.new dir
+
+      assert_equal 2, model.eval(1)
+
+      model = ScoutModel.new dir, delta: 2
+
+      assert_equal 3, model.eval(1)
+    end
+  end
+end

data/test/scout/model/python/test_huggingface.rb

@@ -0,0 +1,9 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestHuggingface < Test::Unit::TestCase
+  def test_true
+    assert true
+  end
+end
+

data/test/scout/model/python/test_torch.rb

@@ -0,0 +1,71 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestTorch < Test::Unit::TestCase
+  def test_linear
+    model = nil
+
+    TmpFile.with_dir do |dir|
+
+      # Create model
+
+      TorchModel.init_python
+
+      model = TorchModel.new dir
+      model.state = ScoutPython.torch.nn.Linear.new(1, 1)
+      model.criterion = ScoutPython.torch.nn.MSELoss.new()
+
+      model.extract_features do |f|
+        [f]
+      end
+
+      model.post_process do |v,list|
+        list ? list.collect{|vv| vv.first } :  v.first
+      end
+
+      # Train model
+
+      model.add 5.0, [10.0]
+      model.add 10.0, [20.0]
+
+      model.options[:training_args][:epochs] = 1000
+      model.train
+
+      w = model.get_weights.to_ruby.first.first
+
+      assert w > 1.8
+      assert w < 2.2
+
+      # Load the model again
+
+      sss 0
+      model.save
+
+      model = ScoutModel.new dir
+
+      # Test model
+
+      y = model.eval_list([100.0, 200.0]).first
+
+      assert(y > 150.0)
+      assert(y < 250.0)
+
+      y = model.eval(100.0)
+
+      assert(y > 150.0)
+      assert(y < 250.0)
+
+      test = [1.0, 5.0, 10.0, 20.0]
+      input_sum = Misc.sum(test)
+      sum = Misc.sum(model.eval_list(test))
+      assert sum > 0.8 * input_sum * 2
+      assert sum < 1.2 * input_sum * 2
+
+      w = TorchModel.get_weights(model.state).to_ruby.first.first
+
+      assert w > 1.8
+      assert w < 2.2
+    end
+  end
+end
+