scout-ai 0.2.0 → 1.0.1

data/test/scout/llm/backends/test_bedrock.rb

@@ -0,0 +1,60 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestLLMBedrock < Test::Unit::TestCase
+  def _test_ask
+    prompt =<<-EOF
+say hi
+    EOF
+    ppp LLM::Bedrock.ask prompt, model: "anthropic.claude-3-sonnet-20240229-v1:0", model_max_tokens: 100, model_anthropic_version: 'bedrock-2023-05-31'
+  end
+
+
+  def _test_embeddings
+    Log.severity = 0
+    text =<<-EOF
+Some text
+    EOF
+    emb = LLM::Bedrock.embed text, log_errors: true
+    assert(Float === emb.first)
+  end
+
+  def test_tool
+    prompt =<<-EOF
+What is the weather in London. Should I take my umbrella? Use the provided tool
+    EOF
+
+    tools = [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_weather",
+          "description": "Get the current temperature and raining conditions for a specific location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": {
+                "type": "string",
+                "description": "The city and state, e.g., San Francisco, CA"
+              },
+              "unit": {
+                "type": "string",
+                "enum": ["Celsius", "Fahrenheit"],
+                "description": "The temperature unit to use. Infer this from the user's location."
+              }
+            },
+            "required": ["location", "unit"]
+          }
+        }
+      },
+    ]
+
+    sss 0
+    response = LLM::Bedrock.ask prompt, tool_choice: 'required', tools: tools, model: "anthropic.claude-3-sonnet-20240229-v1:0", model_max_tokens: 100, model_anthropic_version: 'bedrock-2023-05-31' do |name,arguments|
+      "It's 15 degrees and raining."
+    end
+
+    ppp response
+  end
+end
+

data/test/scout/llm/backends/test_huggingface.rb

@@ -13,7 +13,7 @@ user: write a script that sorts files in a directory
     ppp LLM::Huggingface.ask prompt, model: 'HuggingFaceTB/SmolLM2-135M-Instruct'
   end
 
-  def test_embeddings
+  def _test_embeddings
     Log.severity = 0
     text =<<-EOF
 Some text
@@ -22,7 +22,7 @@ Some text
     assert(Float === emb.first)
   end
 
-  def test_embedding_array
+  def _test_embedding_array
     Log.severity = 0
     text =<<-EOF
 Some text
@@ -31,7 +31,7 @@ Some text
     assert(Float === emb.first.first)
   end
 
-  def test_tool
+  def _test_tool
     prompt =<<-EOF
 What is the weather in London. Should I take an umbrella?
     EOF

data/test/scout/llm/backends/test_ollama.rb

@@ -13,22 +13,42 @@ user: write a script that sorts files in a directory
     ppp LLM::OLlama.ask prompt, model: 'mistral', mode: 'chat'
   end
 
-  def test_embeddings
+  def test_tool_call_output
     Log.severity = 0
-    text =<<-EOF
-Some text
+    prompt =<<-EOF
+function_call:
+
+{"type":"function","function":{"name":"Baking-bake_muffin_tray","arguments":{}},"id":"Baking_bake_muffin_tray_Default"}
+
+function_call_output:
+
+{"id":"Baking_bake_muffin_tray_Default","content":"Baking batter (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour))"}
+
+user:
+
+How do you bake muffins, according to the tool I provided you. Don't
+tell me the recipe you already know, use the tool call output. Let me
+know if you didn't get it.
     EOF
-    emb = LLM::OLlama.embed text, model: 'mistral'
-    assert(Float === emb.first)
+    ppp LLM::OLlama.ask prompt, model: 'mistral', mode: 'chat'
   end
 
-  def test_embedding_array
+  def test_tool_call_output_weather
     Log.severity = 0
-    text =<<-EOF
-Some text
+    prompt =<<-EOF
+function_call:
+
+{"name":"get_current_temperature", "arguments":{"location":"London","unit":"Celsius"},"id":"tNTnsQq2s6jGh0npOh43AwDD"}
+
+function_call_output:
+
+{"id":"tNTnsQq2s6jGh0npOh43AwDD", "content":"It's 15 degrees and raining."}
+
+user:
+
+should i take an umbrella?
     EOF
-    emb = LLM::OLlama.embed [text], model: 'mistral'
-    assert(Float === emb.first.first)
+    ppp LLM::OLlama.ask prompt, model: 'mistral'
   end
 
   def test_tool
@@ -68,5 +88,23 @@ What is the weather in London. Should I take an umbrella?
 
     ppp respose
   end
+
+  def test_embeddings
+    Log.severity = 0
+    text =<<-EOF
+Some text
+    EOF
+    emb = LLM::OLlama.embed text, model: 'mistral'
+    assert(Float === emb.first)
+  end
+
+  def test_embedding_array
+    Log.severity = 0
+    text =<<-EOF
+Some text
+    EOF
+    emb = LLM::OLlama.embed [text], model: 'mistral'
+    assert(Float === emb.first.first)
+  end
 end
 

data/test/scout/llm/backends/test_openai.rb

@@ -11,24 +11,134 @@ user: write a script that sorts files in a directory
     ppp LLM::OpenAI.ask prompt
   end
 
-  def _test_argonne
+  def __test_embeddings
+    Log.severity = 0
+    text =<<-EOF
+Some text
+    EOF
+    emb = LLM::OpenAI.embed text, log_errors: true, model: 'embedding-model'
+
+    assert(Float === emb.first)
+  end
+
+  def test_tool_call_output
+    Log.severity = 0
     prompt =<<-EOF
-user: write a script that sorts files in a directory
+function_call:
+
+{"type":"function","function":{"name":"Baking-bake_muffin_tray","arguments":"{}"},"id":"Baking_bake_muffin_tray_Default"}
+
+function_call_output:
+
+{"id":"Baking_bake_muffin_tray_Default","role":"tool","content":"Baking batter (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour))"}
+
+user:
+
+How do you bake muffins, according to the tool I provided you. Don't
+tell me the recipe you already know, use the tool call output. Let me
+know if you didn't get it.
     EOF
-    sss 0
+    ppp LLM::OpenAI.ask prompt, model: 'gpt-4.1-nano'
   end
 
-  def _test_embeddings
+  def test_tool_call_output_2
     Log.severity = 0
-    text =<<-EOF
-Some text
+    prompt =<<-EOF
+function_call:
+
+{"name":"get_current_temperature", "arguments":{"location":"London","unit":"Celsius"},"id":"tNTnsQq2s6jGh0npOh43AwDD"}
+
+function_call_output:
+
+{"id":"tNTnsQq2s6jGh0npOh43AwDD", "content":"It's 15 degrees and raining."}
+
+user:
+
+should i take an umbrella?
     EOF
-    emb = LLM::OpenAI.embed text, log_errors: true
-    assert(Float === emb.first)
+    ppp LLM::OpenAI.ask prompt, model: 'gpt-4.1-nano'
+  end
+
+  def test_tool_call_output_features
+    Log.severity = 0
+    prompt =<<-EOF
+function_call:
+
+{"name":"Baking-bake_muffin_tray","arguments":{},"id":"Baking_bake_muffin_tray_Default"}
+
+function_call_output:
+
+{"id":"Baking_bake_muffin_tray_Default","content":"Baking batter (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour))"}
+
+user:
+
+How do you bake muffins, according to the tool I provided you. Don't
+tell me the recipe you already know, use the tool call output. Let me
+know if you didn't get it.
+    EOF
+    ppp LLM::OpenAI.ask prompt, model: 'gpt-4.1-nano'
   end
 
-  def _test_tool
+  def test_tool_call_output_weather
+    Log.severity = 0
     prompt =<<-EOF
+function_call:
+
+{"name":"get_current_temperature", "arguments":{"location":"London","unit":"Celsius"},"id":"tNTnsQq2s6jGh0npOh43AwDD"}
+
+function_call_output:
+
+{"id":"tNTnsQq2s6jGh0npOh43AwDD", "content":"It's 15 degrees and raining."}
+
+user:
+
+should i take an umbrella?
+    EOF
+    ppp LLM::OpenAI.ask prompt, model: 'gpt-4.1-nano'
+  end
+
+
+  def test_tool_gpt5
+    prompt =<<-EOF
+user:
+What is the weather in London. Should I take my umbrella?
+    EOF
+
+    tools = [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_current_temperature",
+          "description": "Get the current temperature and raining conditions for a specific location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": {
+                "type": "string",
+                "description": "The city and state, e.g., San Francisco, CA"
+              },
+              "unit": {
+                "type": "string",
+                "enum": ["Celsius", "Fahrenheit"],
+                "description": "The temperature unit to use. Infer this from the user's location."
+              }
+            },
+            "required": ["location", "unit"]
+          }
+        }
+      },
+    ]
+
+    respose = LLM::OpenAI.ask prompt, tool_choice: 'required', tools: tools, model: "gpt-5", log_errors: true do |name,arguments|
+      "It's 15 degrees and raining."
+    end
+
+    ppp respose
+  end
+
+  def test_tool
+    prompt =<<-EOF
+user:
 What is the weather in London. Should I take my umbrella?
     EOF
 
@@ -58,11 +168,25 @@ What is the weather in London. Should I take my umbrella?
     ]
 
     sss 0
-    respose = LLM::OpenAI.ask prompt, tool_choice: 'required', tools: tools, model: "gpt-4o" do |name,arguments|
+    respose = LLM::OpenAI.ask prompt, tool_choice: 'required', tools: tools, model: "gpt-4.1-mini", log_errors: true do |name,arguments|
       "It's 15 degrees and raining."
     end
 
     ppp respose
   end
+
+  def test_json_output
+    prompt =<<-EOF
+system:
+
+Respond in json format with a hash of strings as keys and string arrays as values, at most three in length
+
+user:
+
+What other movies have the protagonists of the original gost busters played on, just the top.
+    EOF
+    sss 0
+    ppp LLM::OpenAI.ask prompt, format: :json
+  end
 end
 

data/test/scout/llm/backends/test_responses.rb

@@ -0,0 +1,239 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestLLMResponses < Test::Unit::TestCase
+  def test_ask
+    prompt =<<-EOF
+system: you are a coding helper that only write code and comments without formatting so that it can work directly, avoid the initial and end commas ```.
+user: write a script that sorts files in a directory
+    EOF
+    sss 0
+    ppp LLM::Responses.ask prompt, model: 'gpt-4.1-nano'
+  end
+
+  def __test_embeddings
+    Log.severity = 0
+    text =<<-EOF
+Some text
+    EOF
+    emb = LLM::Responses.embed text, log_errors: true
+    assert(Float === emb.first)
+  end
+
+  def test_tool_call_output_weather
+    Log.severity = 0
+    prompt =<<-EOF
+function_call:
+
+{"name":"get_current_temperature", "arguments":{"location":"London","unit":"Celsius"},"id":"tNTnsQq2s6jGh0npOh43AwDD"}
+
+function_call_output:
+
+{"id":"tNTnsQq2s6jGh0npOh43AwDD", "content":"It's 15 degrees and raining."}
+
+user:
+
+should i take an umbrella?
+    EOF
+    ppp LLM::Responses.ask prompt, model: 'gpt-4.1-nano'
+  end
+
+  def test_tool
+    prompt =<<-EOF
+user:
+What is the weather in London. Should I take my umbrella?
+    EOF
+
+    tools = [
+      {
+        "type": "function",
+        "name": "get_current_temperature",
+        "description": "Get the current temperature and raining conditions for a specific location",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "location": {
+              "type": "string",
+              "description": "The city and state, e.g., San Francisco, CA"
+            },
+            "unit": {
+              "type": "string",
+              "enum": ["Celsius", "Fahrenheit"],
+              "description": "The temperature unit to use. Infer this from the user's location."
+            }
+          },
+          "required": ["location", "unit"]
+        }
+      },
+    ]
+
+    sss 1
+    respose = LLM::Responses.ask prompt, tool_choice: 'required', tools: tools, model: "gpt-4.1-nano", log_errors: true do |name,arguments|
+      "It's 15 degrees and raining."
+    end
+
+    ppp respose
+  end
+
+  def test_news
+    prompt =<<-EOF
+websearch: true
+
+user:
+
+What was the top new in the US today?
+    EOF
+    ppp LLM::Responses.ask prompt
+  end
+
+  def test_image
+    prompt =<<-EOF
+image: #{datafile_test 'cat.jpg'}
+
+user:
+
+What animal is represented in the image?
+    EOF
+    sss 0
+    ppp LLM::Responses.ask prompt
+  end
+
+  def test_json_output
+    prompt =<<-EOF
+system:
+
+Respond in json format with a hash of strings as keys and string arrays as values, at most three in length
+
+user:
+
+What other movies have the protagonists of the original gost busters played on, just the top.
+    EOF
+    sss 0
+    ppp LLM::Responses.ask prompt, format: :json
+  end
+
+  def test_json_format
+    prompt =<<-EOF
+user:
+
+What other movies have the protagonists of the original gost busters played on.
+Name each actor and the top movie they took part of
+    EOF
+    sss 0
+
+    format = {
+      name: 'actors_and_top_movies',
+      type: 'object',
+      properties: {},
+      additionalProperties: {type: :string}
+    }
+    ppp LLM::Responses.ask prompt, format: format
+  end
+
+  def test_json_format_list
+    prompt =<<-EOF
+user:
+
+What other movies have the protagonists of the original gost busters played on.
+Name each actor as keys and the top 3 movies they took part of as values
+    EOF
+    sss 0
+
+    format = {
+      name: 'actors_and_top_movies',
+      type: 'object',
+      properties: {},
+      additionalProperties: {type: :array, items: {type: :string}}
+    }
+    ppp LLM::Responses.ask prompt, format: format
+  end
+
+  def test_json_format_actor_list
+    prompt =<<-EOF
+user:
+
+What other movies have the protagonists of the original gost busters played on.
+Name each actor as keys and the top 3 movies they took part of as values
+    EOF
+    sss 0
+
+    format = {
+      name: 'actors_and_top_movies',
+      type: 'object',
+      properties: {},
+      additionalProperties: false,
+      items: {
+        type: 'object', 
+        properties: {
+          name: {type: :string, description: 'actor name'}, 
+          movies: {type: :array, description: 'list of top 3 movies', items: {type: :string, description: 'movie title plus year in parenthesis'} }, 
+          additionalProperties: false
+        }
+      }
+    }
+
+    schema =  {
+      "type": "object",
+      "properties": {
+        "people": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": { "type": "string" },
+              "movies": {
+                "type": "array",
+                "items": { "type": "string" },
+                "minItems": 3,
+                "maxItems": 3
+              }
+            },
+            "required": ["name", "movies"],
+            additionalProperties: false
+          }
+        }
+      },
+      additionalProperties: false,
+      "required": ["people"]
+    }
+    ppp LLM::Responses.ask prompt, format: schema
+  end
+
+  def test_tool_gpt5
+    prompt =<<-EOF
+user:
+What is the weather in London. Should I take my umbrella?
+    EOF
+
+    tools = [
+      {
+        "type": "function",
+        "name": "get_current_temperature",
+        "description": "Get the current temperature and raining conditions for a specific location",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "location": {
+              "type": "string",
+              "description": "The city and state, e.g., San Francisco, CA"
+            },
+            "unit": {
+              "type": "string",
+              "enum": ["Celsius", "Fahrenheit"],
+              "description": "The temperature unit to use. Infer this from the user's location."
+            }
+          },
+          "required": ["location", "unit"]
+        }
+      },
+    ]
+
+    sss 0
+    respose = LLM::Responses.ask prompt, tool_choice: 'required', tools: tools, model: "gpt-5", log_errors: true do |name,arguments|
+      "It's 15 degrees and raining."
+    end
+
+    ppp respose
+  end
+end
+

data/test/scout/llm/test_agent.rb

@@ -39,76 +39,6 @@ class TestLLMAgent < Test::Unit::TestCase
     end
   end
 
-  def test_workflow
-    m = Module.new do
-      extend Workflow
-      self.name = "Registration"
-
-      desc "Register a person"
-      input :name, :string, "Last, first name"
-      input :age, :integer, "Age"
-      input :gender, :select, "Gender", nil, :select_options => %w(male female)
-      task :person => :yaml do
-        iii inputs.to_hash
-        inputs.to_hash
-      end
-    end
-
-    sss 0
-    #ppp LLM.workflow_ask(m, "Register Eduard Smith, a 25 yo male", model: "Meta-Llama-3.3-70B-Instruct")
-    ppp LLM.workflow_ask(m, "Register Eduard Smith, a 25 yo male, using a tool call to the tool provided", backend: 'ollama', model: "llama3")
-  end
-
-  def _test_openai
-    TmpFile.with_dir do |dir|
-      kb = KnowledgeBase.new dir
-      kb.format = {"Person" => "Alias"}
-      kb.register :brothers, datafile_test(:person).brothers, undirected: true
-      kb.register :marriages, datafile_test(:person).marriages, undirected: true, source: "=>Alias", target: "=>Alias"
-      kb.register :parents, datafile_test(:person).parents
-
-      sss 3
-      agent = LLM::Agent.new knowledge_base: kb, model: 'gpt-4o'
-
-      agent.system = ""
-
-      ppp agent.ask "Who is Miguel's brother-in-law"
-    end
-  end
-
-  def _test_argonne
-    TmpFile.with_dir do |dir|
-      kb = KnowledgeBase.new dir
-      kb.format = {"Person" => "Alias"}
-      kb.register :brothers, datafile_test(:person).brothers, undirected: true
-      kb.register :marriages, datafile_test(:person).marriages, undirected: true, source: "=>Alias", target: "=>Alias"
-      kb.register :parents, datafile_test(:person).parents
-
-      agent.system = ""
-
-      ppp agent.ask "Who is Miguel's brother-in-law"
-    end
-  end
-
-  def _test_nvidia
-    TmpFile.with_dir do |dir|
-      kb = KnowledgeBase.new dir
-      kb.format = {"Person" => "Alias"}
-      kb.register :brothers, datafile_test(:person).brothers, undirected: true
-      kb.register :marriages, datafile_test(:person).marriages, undirected: true, source: "=>Alias", target: "=>Alias"
-      kb.register :parents, datafile_test(:person).parents
-
-      sss 0
-
-      ppp LLM::OpenAI.ask "Say Hi", url: "https://integrate.api.nvidia.com/v1", model: "deepseek-ai/deepseek-r1"
-      exit
-
-
-      agent.system = ""
-
-      ppp agent.ask "Who is Miguel's brother-in-law. Make use of the tools using tool_calls"
-    end
-  end
 
 end
 

data/test/scout/llm/test_ask.rb

@@ -12,6 +12,7 @@ system: you are a coding helper that only write code and comments without format
 user: write a script that sorts files in a directory 
     EOF
     ppp LLM.ask prompt
+    ppp LLM.ask prompt
   end
 
   def _test_workflow_ask
@@ -37,9 +38,11 @@ user: write a script that sorts files in a directory
           "1 minute"
         end
       end
+      export :recipe_steps, :step_time
     end
 
-    ppp LLM.workflow_ask(m, "How much time does it take to prepare a cake recipe")
+    sss 0
+    ppp LLM.workflow_ask(m, "How much time does it take to prepare a 'vanilla' cake recipe, use the tools provided to find out")
   end
 
   def test_knowledbase