scout-ai 0.2.0 → 1.0.0

data/test/scout/llm/backends/test_openai.rb

@@ -2,7 +2,7 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestLLMOpenAI < Test::Unit::TestCase
-  def test_ask
+  def _test_ask
     prompt =<<-EOF
 system: you are a coding helper that only write code and comments without formatting so that it can work directly, avoid the initial and end commas ```.
 user: write a script that sorts files in a directory
@@ -11,24 +11,95 @@ user: write a script that sorts files in a directory
     ppp LLM::OpenAI.ask prompt
   end
 
-  def _test_argonne
+  def __test_embeddings
+    Log.severity = 0
+    text =<<-EOF
+Some text
+    EOF
+    emb = LLM::OpenAI.embed text, log_errors: true, model: 'embedding-model'
+
+    assert(Float === emb.first)
+  end
+
+  def _test_tool_call_output
+    Log.severity = 0
     prompt =<<-EOF
-user: write a script that sorts files in a directory
+function_call:
+
+{"type":"function","function":{"name":"Baking-bake_muffin_tray","arguments":"{}"},"id":"Baking_bake_muffin_tray_Default"}
+
+function_call_output:
+
+{"id":"Baking_bake_muffin_tray_Default","role":"tool","content":"Baking batter (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour))"}
+
+user:
+
+How do you bake muffins, according to the tool I provided you. Don't
+tell me the recipe you already know, use the tool call output. Let me
+know if you didn't get it.
     EOF
-    sss 0
+    ppp LLM::OpenAI.ask prompt, model: 'gpt-4.1-nano'
   end
 
-  def _test_embeddings
+  def _test_tool_call_output_2
     Log.severity = 0
-    text =<<-EOF
-Some text
+    prompt =<<-EOF
+function_call:
+
+{"name":"get_current_temperature", "arguments":{"location":"London","unit":"Celsius"},"id":"tNTnsQq2s6jGh0npOh43AwDD"}
+
+function_call_output:
+
+{"id":"tNTnsQq2s6jGh0npOh43AwDD", "content":"It's 15 degrees and raining."}
+
+user:
+
+should i take an umbrella?
     EOF
-    emb = LLM::OpenAI.embed text, log_errors: true
-    assert(Float === emb.first)
+    ppp LLM::OpenAI.ask prompt, model: 'gpt-4.1-nano'
   end
 
-  def _test_tool
+  def _test_tool_call_output_features
+    Log.severity = 0
     prompt =<<-EOF
+function_call:
+
+{"name":"Baking-bake_muffin_tray","arguments":{},"id":"Baking_bake_muffin_tray_Default"}
+
+function_call_output:
+
+{"id":"Baking_bake_muffin_tray_Default","content":"Baking batter (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour))"}
+
+user:
+
+How do you bake muffins, according to the tool I provided you. Don't
+tell me the recipe you already know, use the tool call output. Let me
+know if you didn't get it.
+    EOF
+    ppp LLM::OpenAI.ask prompt, model: 'gpt-4.1-nano'
+  end
+
+  def _test_tool_call_output_weather
+    Log.severity = 0
+    prompt =<<-EOF
+function_call:
+
+{"name":"get_current_temperature", "arguments":{"location":"London","unit":"Celsius"},"id":"tNTnsQq2s6jGh0npOh43AwDD"}
+
+function_call_output:
+
+{"id":"tNTnsQq2s6jGh0npOh43AwDD", "content":"It's 15 degrees and raining."}
+
+user:
+
+should i take an umbrella?
+    EOF
+    ppp LLM::OpenAI.ask prompt, model: 'gpt-4.1-nano'
+  end
+
+  def test_tool
+    prompt =<<-EOF
+user:
 What is the weather in London. Should I take my umbrella?
     EOF
 
@@ -58,11 +129,25 @@ What is the weather in London. Should I take my umbrella?
     ]
 
     sss 0
-    respose = LLM::OpenAI.ask prompt, tool_choice: 'required', tools: tools, model: "gpt-4o" do |name,arguments|
+    respose = LLM::OpenAI.ask prompt, tool_choice: 'required', tools: tools, model: "gpt-4.1-mini", log_errors: true do |name,arguments|
       "It's 15 degrees and raining."
     end
 
     ppp respose
   end
+
+  def _test_json_output
+    prompt =<<-EOF
+system:
+
+Respond in json format with a hash of strings as keys and string arrays as values, at most three in length
+
+user:
+
+What other movies have the protagonists of the original gost busters played on, just the top.
+    EOF
+    sss 0
+    ppp LLM::OpenAI.ask prompt, format: :json
+  end
 end
 

data/test/scout/llm/backends/test_responses.rb

@@ -0,0 +1,115 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestLLMResponses < Test::Unit::TestCase
+  def test_ask
+    prompt =<<-EOF
+system: you are a coding helper that only write code and comments without formatting so that it can work directly, avoid the initial and end commas ```.
+user: write a script that sorts files in a directory
+    EOF
+    sss 0
+    ppp LLM::Responses.ask prompt, model: 'gpt-4.1-nano'
+  end
+
+  def __test_embeddings
+    Log.severity = 0
+    text =<<-EOF
+Some text
+    EOF
+    emb = LLM::Responses.embed text, log_errors: true
+    assert(Float === emb.first)
+  end
+
+  def test_tool_call_output_weather
+    Log.severity = 0
+    prompt =<<-EOF
+function_call:
+
+{"name":"get_current_temperature", "arguments":{"location":"London","unit":"Celsius"},"id":"tNTnsQq2s6jGh0npOh43AwDD"}
+
+function_call_output:
+
+{"id":"tNTnsQq2s6jGh0npOh43AwDD", "content":"It's 15 degrees and raining."}
+
+user:
+
+should i take an umbrella?
+    EOF
+    ppp LLM::Responses.ask prompt, model: 'gpt-4.1-nano'
+  end
+
+  def test_tool
+    prompt =<<-EOF
+user:
+What is the weather in London. Should I take my umbrella?
+    EOF
+
+    tools = [
+      {
+        "type": "function",
+        "name": "get_current_temperature",
+        "description": "Get the current temperature and raining conditions for a specific location",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "location": {
+              "type": "string",
+              "description": "The city and state, e.g., San Francisco, CA"
+            },
+            "unit": {
+              "type": "string",
+              "enum": ["Celsius", "Fahrenheit"],
+              "description": "The temperature unit to use. Infer this from the user's location."
+            }
+          },
+          "required": ["location", "unit"]
+        }
+      },
+    ]
+
+    sss 1
+    respose = LLM::Responses.ask prompt, tool_choice: 'required', tools: tools, model: "gpt-4.1-nano", log_errors: true do |name,arguments|
+      "It's 15 degrees and raining."
+    end
+
+    ppp respose
+  end
+
+  def test_news
+    prompt =<<-EOF
+websearch: true
+
+user:
+
+What was the top new in the US today?
+    EOF
+    ppp LLM::Responses.ask prompt
+  end
+
+  def test_image
+    prompt =<<-EOF
+image: #{datafile_test 'cat.jpg'}
+
+user:
+
+What animal is represented in the image?
+    EOF
+    sss 0
+    ppp LLM::Responses.ask prompt
+  end
+
+  def test_json_output
+    prompt =<<-EOF
+system:
+
+Respond in json format with a hash of strings as keys and string arrays as values, at most three in length
+
+user:
+
+What other movies have the protagonists of the original gost busters played on, just the top.
+    EOF
+    sss 0
+    ppp LLM::Responses.ask prompt, format: :json
+  end
+end
+

data/test/scout/llm/test_ask.rb

@@ -12,6 +12,7 @@ system: you are a coding helper that only write code and comments without format
 user: write a script that sorts files in a directory 
     EOF
     ppp LLM.ask prompt
+    ppp LLM.ask prompt
   end
 
   def _test_workflow_ask

data/test/scout/llm/test_chat.rb

@@ -0,0 +1,214 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestMessages < Test::Unit::TestCase
+
+  def test_short
+
+    question =<<-EOF
+Hi
+    EOF
+
+    iii LLM.chat(question)
+  end
+
+  def test_inline
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+assistant:
+
+Here is some stuff
+
+user: feedback
+
+that continues here
+    EOF
+
+    iii LLM.chat(question)
+  end
+
+  def test_messages
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+user:
+
+What is the capital of France
+
+assistant:
+
+Paris
+
+user:
+
+is this the national anthem
+
+[[
+corous: Viva Espagna
+]]
+
+assistant:
+
+no
+
+user:
+
+import: math.system
+
+consider this file
+
+<file name=foo_bar>
+foo: bar
+</file>
+
+how many characters does it hold
+
+assistant:
+
+8
+    EOF
+
+    messages = LLM.messages question
+    refute messages.collect{|i| i[:role] }.include?("corous")
+    assert messages.collect{|i| i[:role] }.include?("import")
+  end
+
+  def test_chat_import
+    file1 =<<-EOF
+system: You are an assistant
+    EOF
+
+    file2 =<<-EOF
+import: header
+user: say something
+    EOF
+
+    TmpFile.with_path do |tmpdir|
+      tmpdir.header.write file1
+      tmpdir.chat.write file2
+
+      chat = LLM.chat tmpdir.chat
+    end
+  end
+
+  def test_clear
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+clear:
+
+user:
+
+What is the capital of France
+    EOF
+
+    TmpFile.with_file question do |file|
+      messages = LLM.chat file
+      refute messages.collect{|m| m[:role] }.include?('system')
+    end
+  end
+
+  def __test_job
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+job: Baking/bake_muffin_tray/Default_08a1812eca3a18dce2232509dabc9b41
+
+How are muffins made
+
+    EOF
+
+    TmpFile.with_file question do |file|
+      messages = LLM.chat file
+      ppp LLM.print messages
+    end
+  end
+
+
+  def test_task
+    question =<<-EOF
+system:
+
+you are a terse assistant that only write in short sentences
+
+user:
+
+task: Baking bake_muffin_tray blueberries=true title="This is a title" list=one,two,"and three"
+
+How are muffins made?
+
+    EOF
+
+    TmpFile.with_file question do |file|
+      messages = LLM.chat file
+      ppp LLM.print messages
+    end
+  end
+
+  def test_structure
+    require 'scout/llm/ask'
+    sss 0
+    question =<<-EOF
+system:
+
+Respond in json format with a hash of strings as keys and string arrays as values, at most three in length
+
+endpoint: sambanova
+
+What other movies have the protagonists of the original gost busters played on, just the top.
+
+    EOF
+
+    TmpFile.with_file question do |file|
+      ppp LLM.ask file
+    end
+  end
+
+  def test_tools
+    require 'scout/llm/ask'
+
+    question =<<-EOF
+user:
+
+Use the provided tool to learn the instructions of baking a tray of muffins. Don't
+give me your own recipe, return the one provided by the tool
+
+tool: Baking bake_muffin_tray
+    EOF
+
+    TmpFile.with_file question do |file|
+      ppp LLM.ask file
+    end
+  end
+
+  def test_knowledge_base
+    require 'scout/llm/ask'
+    sss 0
+    question =<<-EOF
+system:
+
+Query the knowledge base of familiar relationships to answer the question
+
+user:
+
+Who is Miki's brother in law?
+
+association: brothers #{datafile_test(:person).brothers} undirected=true
+association: marriages #{datafile_test(:person).marriages} undirected=true source="=>Alias" target="=>Alias"
+    EOF
+
+    TmpFile.with_file question do |file|
+      ppp LLM.ask file
+    end
+  end
+end
+

data/test/scout/llm/test_parse.rb

@@ -4,8 +4,9 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 class TestLLMParse < Test::Unit::TestCase
   def test_parse
     text=<<-EOF
+hi
 system: you are an asistant
-user: Given the contents of this file: [[ 
+user: Given the contents of this file:[[
 line 1: 1
 line 2: 2
 line 3: 3
@@ -13,7 +14,85 @@ line 3: 3
 Show me the lines in reverse order
     EOF
 
-    iii LLM.parse(text)
+    assert_include LLM.parse(text).first[:content], 'hi'
+    assert_include LLM.parse(text).last[:content], 'reverse'
+  end
+
+  def test_code
+    text=<<-EOF
+hi
+system: you are an asistant
+user: Given the contents of this file:
+```yaml
+key: value
+key2: value2
+```
+Show me the lines in reverse order
+    EOF
+
+    assert_include LLM.parse(text).last[:content], 'key2'
+  end
+
+  def test_lines
+    text=<<-EOF
+system: you are an asistant
+user: I have a question
+    EOF
+
+    assert_include LLM.parse(text).last[:content], 'question'
+  end
+
+  def test_blocks
+    text=<<-EOF
+system:
+
+you are an asistant
+
+user:
+
+I have a question
+
+    EOF
+
+    assert_include LLM.parse(text).last[:content], 'question'
+  end
+
+  def test_no_role
+    text=<<-EOF
+I have a question
+    EOF
+
+    assert_include LLM.parse(text).last[:content], 'question'
+  end
+
+
+  def test_cmd
+    text=<<-EOF
+How many files are there:
+
+[[cmd list of files
+echo "file1 file2"
+]]
+    EOF
+
+    assert_equal :user, LLM.parse(text).last[:role]
+    assert_include LLM.parse(text).first[:content], 'file1'
+  end
+
+  def test_directory
+    TmpFile.with_path do |tmpdir|
+      tmpdir.file1.write "foo"
+      tmpdir.file2.write "bar"
+      text=<<-EOF
+How many files are there:
+
+[[directory DIR
+#{tmpdir}
+]]
+      EOF
+
+      assert_include LLM.parse(text).first[:content], 'file1'
+    end
   end
 end
 

data/test/scout/model/python/huggingface/causal/test_next_token.rb

@@ -0,0 +1,59 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout-ai'
+class TestClass < Test::Unit::TestCase
+  def test_main
+    model = NextTokenModel.new 
+    train_texts = [
+        "say hi, no!",
+        "say hi, no no no",
+        "say hi, hi ",
+        "say hi, hi how are you ",
+        "say hi, hi are you good",
+    ] 
+
+    model_name = "distilgpt2"  # Replace with your local/other HF Llama checkpoint as needed
+
+    TmpFile.with_path do |tmp_dir|
+      iii tmp_dir
+
+      sss 0
+      model = NextTokenModel.new model_name, tmp_dir, training_num_train_epochs: 1000, training_learning_rate: 0.1
+
+      iii :new
+      chat = Chat.setup []
+      chat.user "say hi"
+      ppp model.eval chat
+
+      model.save
+      model = PythonModel.new tmp_dir
+
+      iii :load
+      chat = Chat.setup []
+      chat.user "say hi"
+      ppp model.eval chat
+
+      iii :training
+      state, tokenizer = model.init
+      tokenizer.pad_token = tokenizer.eos_token
+      model.add_list train_texts.shuffle
+      model.train
+
+      iii :trained
+      chat = Chat.setup []
+      chat.user "say hi"
+      ppp model.eval chat
+
+      model.save
+      model = PythonModel.new tmp_dir
+
+      iii :load_again
+      chat = Chat.setup []
+      chat.user "say hi"
+      ppp model.eval chat
+    end
+
+  end
+end
+

data/test/scout/model/python/huggingface/test_causal.rb

@@ -0,0 +1,33 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_eval_chat
+    #model = CausalModel.new 'BSC-LT/salamandra-2b-instruct'
+    model = CausalModel.new 'mistralai/Mistral-7B-Instruct-v0.3'
+
+    model.init
+
+    net, tok = model.state
+
+    iii model.eval([
+      {role: :system, content: "You are a calculator, just reply with the answer"},
+      {role: :user, content: " 1 + 2 ="}
+    ])
+  end
+
+  def test_eval_train
+    #model = CausalModel.new 'BSC-LT/salamandra-2b-instruct'
+    model = CausalModel.new 'mistralai/Mistral-7B-Instruct-v0.3'
+
+    model.init
+
+    net, tok = model.state
+
+    iii model.eval([
+      {role: :system, content: "You are a calculator, just reply with the answer"},
+      {role: :user, content: " 1 + 2 ="}
+    ])
+  end
+end
+

data/test/scout/model/python/huggingface/test_classification.rb

@@ -0,0 +1,30 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSequenceClassification < Test::Unit::TestCase
+  def _test_eval_sequence_classification
+    model = SequenceClassificationModel.new 'bert-base-uncased', nil,
+      class_labels: %w(Bad Good)
+
+    assert_include ["Bad", "Good"], model.eval("This is dog")
+    assert_include ["Bad", "Good"], model.eval_list(["This is dog", "This is cat"]).first
+  end
+
+  def test_train_sequence_classification
+    model = SequenceClassificationModel.new 'bert-base-uncased', nil,
+      class_labels: %w(Bad Good)
+
+    model.init
+
+    10.times do
+      model.add "The dog", 'Bad'
+      model.add "The cat", 'Good'
+    end
+
+    model.train
+
+    assert_equal "Bad", model.eval("This is dog")
+    assert_equal "Good", model.eval("This is cat")
+  end
+end
+