scout-ai 1.2.3 → 2.0.0

data/test/scout/llm/backends/test_anthropic.rb

@@ -10,22 +10,105 @@ user: say hi
     ppp LLM::Anthropic.ask prompt
   end
 
-  def _test_ask
+  # ScoutCoder: Anthropic has no embeddings endpoint; the backend raises
+  # from embed_query, so the offline contract is the exception, not a vector.
+  def test_embeddings
+    assert_raise(RuntimeError) { LLM::Anthropic.embed 'Some text', log_errors: false, model: 'embedding-model' }
+  end
+
+  def test_ask
+    client = TestFixtures.anthropic_client('backends/anthropic')
+    res = LLM::Anthropic.ask 'user: write a script that sorts files in a directory',
+                             client: client, model: 'claude-sonnet-4-5', persist: false
+
+    assert_equal 'Mock answer from Anthropic messages', res
+    assert_equal 1, client.calls.length
+    assert_equal 'claude-sonnet-4-5', client.calls.first[:model]
+    assert client.calls.first[:messages].any? { |m| m[:role].to_s == 'user' }
+  end
+
+  def test_tool_call_output_weather
+    Log.severity = 0
     prompt =<<-EOF
-user: write a script that sorts files in a directory
+function_call:
+
+{"name":"get_current_temperature", "arguments":{"location":"London","unit":"Celsius"},"id":"tNTnsQq2s6jGh0npOh43AwDD"}
+
+function_call_output:
+
+{"id":"tNTnsQq2s6jGh0npOh43AwDD", "content":"It's 15 degrees and raining."}
+
+user:
+
+should i take an umbrella?
     EOF
-    sss 0
-    ppp LLM::Anthropic.ask prompt
+    client = TestFixtures.anthropic_client('backends/anthropic')
+    res = LLM::Anthropic.ask prompt, client: client, persist: false
+
+    assert_equal 'Mock answer from Anthropic messages', res
+    # ScoutCoder: the Anthropic backend rewrites the function_call /
+    # function_call_output pair into content array items (tool_use /
+    # tool_result) instead of separate messages.
+    sent = client.calls.first[:messages]
+    assert sent.any? { |m| m[:role].to_s == 'user' }
+    assert sent.inspect.include?('tool_result')
   end
 
-  def test_embeddings
-    Log.severity = 0
-    text =<<-EOF
-Some text
+  def test_tool
+    prompt =<<-EOF
+user:
+What is the weather in London. Should I take my umbrella?
     EOF
-    emb = LLM::Anthropic.embed text, log_errors: true, model: 'embedding-model'
 
-    assert(Float === emb.first)
+    tools = [
+      {
+        "type": "custom",
+        "name": "get_current_temperature",
+        "description": "Get the current temperature and raining conditions for a specific location",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "location": {
+              "type": "string",
+              "description": "The city and state, e.g., San Francisco, CA"
+            },
+            "unit": {
+              "type": "string",
+              "enum": ["Celsius", "Fahrenheit"],
+              "description": "The temperature unit to use. Infer this from the user's location."
+            }
+          },
+          "required": ["location", "unit"]
+        }
+      },
+    ]
+
+    client = TestFixtures.anthropic_client('backends/anthropic_tool_use', 'backends/anthropic')
+    respose = LLM::Anthropic.ask prompt, tools: tools,
+                                 client: client, log_errors: true, persist: false do |name,arguments|
+      "It's 15 degrees and raining."
+    end
+
+    assert_equal 'Mock answer from Anthropic messages', respose
+    assert_equal 2, client.calls.length
+
+    # ScoutCoder: LLM::Anthropic#format_tool_definitions renames
+    # `parameters` -> `input_schema` and forces type 'custom'.
+    sent_tools = client.calls.first[:tools]
+    assert sent_tools.any? { |t| (t[:name] || t['name']) == 'get_current_temperature' }
+    assert sent_tools.all? { |t| (t[:input_schema] || t['input_schema']) }
+    assert sent_tools.all? { |t| (t[:type] || t['type']).to_s == 'custom' }
+  end
+
+  def test_json_output
+    client = TestFixtures.anthropic_client('backends/anthropic')
+    res = LLM::Anthropic.ask 'user: What other movies have the protagonists of the original gost busters played on, just the top.',
+                             format: :json, client: client, persist: false
+
+    assert_equal 'Mock answer from Anthropic messages', res
+    # ScoutCoder: Anthropic takes a response_format hash, not the string key
+    # the OpenAI backends use.
+    assert client.calls.first.inspect.include?('json_object')
   end
 
   def _test_tool_call_output_2

data/test/scout/llm/backends/test_bedrock.rb

@@ -2,15 +2,132 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestLLMBedrock < Test::Unit::TestCase
+  # Offline Bedrock coverage: LLM::Bedrock.ask/embed accept options[:client]
+  # (an object with invoke_model(model_id:, content_type:, body:) returning
+  # something whose .body.string is JSON), so FakeBedrockClient exercises the
+  # full request construction + response parsing path without AWS credentials.
+  #
+  # The old real-service versions are kept disabled below (_test_*).
+
+  # ScoutCoder: LLM::Bedrock is not required by lib/scout-ai.rb; each test has
+  # to require 'scout/llm/backends/bedrock' itself (same as LLM.ask does when
+  # dispatching to the :bedrock backend).
+  def test_ask
+    require 'scout/llm/backends/bedrock'
+    client = TestFixtures.bedrock_client('backends/bedrock')
+
+    response = LLM::Bedrock.ask 'user: say hi to bedrock',
+                                client: client,
+                                model: 'anthropic.claude-3-sonnet-20240229-v1:0',
+                                model_max_tokens: 100
+
+    assert_equal 'Mock answer from Bedrock', response
+
+    assert_equal 1, client.calls.length
+    call = client.calls.first
+    assert_equal 'anthropic.claude-3-sonnet-20240229-v1:0', call[:model_id]
+    assert_equal 'application/json', call[:content_type]
+    # ScoutCoder: IndiferentHash#pretty_print takes 0 args, so assert on
+    # individual keys instead of pp-ing the recorded parameters.
+    assert_equal 100, call[:body]['max_tokens']
+    assert call[:body]['messages'].any? { |m| m['role'] == 'user' }
+    assert call[:body]['messages'].any? { |m| m['content'].to_s.include?('say hi to bedrock') }
+  end
+
+  def test_ask_prompt_type
+    require 'scout/llm/backends/bedrock'
+    client = TestFixtures.bedrock_client('backends/bedrock')
+
+    response = LLM::Bedrock.ask 'user: say hi through the prompt endpoint',
+                                client: client, type: :prompt,
+                                model: 'meta.llama3-8b-instruct-v1:0',
+                                model_max_tokens: 100
+
+    assert_equal 'Mock answer from Bedrock', response
+
+    body = client.calls.first[:body]
+    assert body.include?('prompt')
+    assert body['prompt'].to_s.include?('say hi through the prompt endpoint')
+  end
+
+  def test_embeddings
+    require 'scout/llm/backends/bedrock'
+    client = TestFixtures.bedrock_client('backends/bedrock_embedding')
+
+    emb = LLM::Bedrock.embed 'Some text', client: client,
+                             model: 'amazon.titan-embed-text-v1'
+
+    assert(Float === emb.first)
+    assert_equal [0.1, 0.2, 0.3], emb
+    assert_equal 1, client.calls.length
+
+    call = client.calls.first
+    assert_equal 'amazon.titan-embed-text-v1', call[:model_id]
+    assert_equal 'Some text', call[:body]['inputText']
+  end
+
+  # Tool loop: the first payload carries a tool_call content entry, the block
+  # answers it, and the second payload is the final text answer.
+  def test_tool_loop
+    require 'scout/llm/backends/bedrock'
+    tools = [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_current_temperature",
+          "description": "Get the current temperature for a specific location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": { "type": "string",
+                            "description": "The city and state, e.g., San Francisco, CA" },
+              "unit": { "type": "string", "enum": ["Celsius", "Fahrenheit"],
+                        "description": "The temperature unit to use." }
+            },
+            "required": ["location", "unit"]
+          }
+        }
+      }
+    ]
+
+    client = TestFixtures.bedrock_client('backends/bedrock_tool_use', 'backends/bedrock')
+
+    calls_seen = []
+    response = LLM::Bedrock.ask 'user: What is the weather in London? Should I take an umbrella? Use the tool.',
+                                tools: tools, client: client,
+                                model: 'anthropic.claude-3-sonnet-20240229-v1:0',
+                                model_max_tokens: 100 do |name, arguments|
+      calls_seen << [name, arguments]
+      "It's 15 degrees and raining."
+    end
+
+    assert_equal 'Mock answer from Bedrock', response
+
+    # the block was reached with the unwrapped tool call
+    assert_equal 1, calls_seen.length
+    assert_equal 'get_current_temperature', calls_seen.first.first
+    assert_equal 'London', calls_seen.first.last['location']
+
+    # two invoke_model rounds, the second one carrying the tool response
+    assert_equal 2, client.calls.length
+    sent = client.calls.last[:body]['messages']
+    assert sent.any? { |m| m['role'] == 'tool' }
+    tool_messages = sent.select { |m| m['role'] == 'tool' }
+    assert_equal 'It\'s 15 degrees and raining.', tool_messages.last['content']
+    assert_equal 'call_1', tool_messages.last['id']
+  end
+
   def _test_ask
+    # Real-service version: see the offline tests above. Requires AWS
+    # credentials and network.
     prompt =<<-EOF
 say hi
     EOF
     ppp LLM::Bedrock.ask prompt, model: "anthropic.claude-3-sonnet-20240229-v1:0", model_max_tokens: 100, model_anthropic_version: 'bedrock-2023-05-31'
   end
 
-
   def _test_embeddings
+    # Real-service version: see test_embeddings above.
     Log.severity = 0
     text =<<-EOF
 Some text
@@ -57,4 +174,3 @@ What is the weather in London. Should I take my umbrella? Use the provided tool
     ppp response
   end
 end
-

data/test/scout/llm/backends/test_huggingface.rb

@@ -2,8 +2,24 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestLLMHF < Test::Unit::TestCase
+  class FakeHFClient
+    attr_reader :messages, :tools, :calls
 
-  def test_ask
+    def initialize(*responses)
+      @responses = responses
+      @calls = 0
+    end
+
+    def chat(messages, tools, parameters = {})
+      @messages = messages
+      @tools = tools
+      response = @responses[@calls] || @responses.last
+      @calls += 1
+      response
+    end
+  end
+
+  def _test_ask
     Log.severity = 0
     prompt =<<-EOF
 system: you are a coding helper that only write code and inline comments. No extra explanations or comentary
@@ -13,61 +29,140 @@ user: write a script that sorts files in a directory
     ppp LLM::Huggingface.ask prompt, model: 'HuggingFaceTB/SmolLM2-135M-Instruct'
   end
 
-  def _test_embeddings
-    Log.severity = 0
-    text =<<-EOF
-Some text
-    EOF
-    emb = LLM::Huggingface.embed text, model: 'distilbert-base-uncased-finetuned-sst-2-english'
-    assert(Float === emb.first)
+  def test_format_tool_call
+    message = {
+      role: 'function_call',
+      content: {
+        name: 'get_current_temperature',
+        arguments: { location: 'London', unit: 'Celsius' },
+        id: 'call_123'
+      }.to_json
+    }
+
+    formatted = LLM::Huggingface.format_tool_call(message)
+
+    assert_equal 'assistant', formatted[:role]
+    assert_equal 'function', formatted.dig(:tool_calls, 0, :type)
+    assert_equal 'call_123', formatted.dig(:tool_calls, 0, :id)
+    assert_equal 'get_current_temperature', formatted.dig(:tool_calls, 0, :function, :name)
+    assert_equal 'London', formatted.dig(:tool_calls, 0, :function, :arguments, :location)
   end
 
-  def _test_embedding_array
-    Log.severity = 0
-    text =<<-EOF
-Some text
-    EOF
-    emb = LLM::Huggingface.embed [text], model: 'distilbert-base-uncased-finetuned-sst-2-english'
-    assert(Float === emb.first.first)
+  def test_format_tool_output
+    message = {
+      role: 'function_call_output',
+      content: {
+        id: 'call_123',
+        name: 'get_current_temperature',
+        content: "It's 15 degrees and raining."
+      }.to_json
+    }
+
+    formatted = LLM::Huggingface.format_tool_output(message)
+
+    assert_equal 'tool', formatted[:role]
+    assert_equal 'get_current_temperature', formatted[:name]
+    assert_equal 'call_123', formatted[:tool_call_id]
+    assert_equal "It's 15 degrees and raining.", formatted[:content]
   end
 
-  def _test_tool
-    prompt =<<-EOF
-What is the weather in London. Should I take an umbrella?
-    EOF
+  def test_parse_tool_call
+    tool_call = {
+      id: 'call_123',
+      type: 'function',
+      function: {
+        name: 'get_current_temperature',
+        arguments: { location: 'London', unit: 'Celsius' }
+      }
+    }
+
+    parsed = LLM::Huggingface.parse_tool_call(tool_call)
+
+    assert_equal 'call_123', parsed[:id]
+    assert_equal 'get_current_temperature', parsed[:name]
+    assert_equal 'London', parsed.dig(:arguments, :location)
+  end
+
+  def test_ask_with_fake_client
+    client = FakeHFClient.new({ role: 'assistant', content: 'Hello from Huggingface' })
+
+    response = LLM::Huggingface.ask("user: say hi", client: client, log_response: false)
+
+    assert_equal 'Hello from Huggingface', response
+    assert_equal 1, client.calls
+  end
+
+  def test_ask_tool_loop_with_fake_client
+    client = FakeHFClient.new(
+      {
+        role: 'assistant',
+        content: '',
+        tool_calls: [
+          {
+            id: 'call_123',
+            type: 'function',
+            function: {
+              name: 'get_current_temperature',
+              arguments: { location: 'London', unit: 'Celsius' }
+            }
+          }
+        ]
+      },
+      {
+        role: 'assistant',
+        content: 'Take an umbrella.'
+      }
+    )
 
     tools = [
       {
-        "type": "function",
-        "function": {
-          "name": "get_current_temperature",
-          "description": "Get the current temperature for a specific location",
-          "parameters": {
-            "type": "object",
-            "properties": {
-              "location": {
-                "type": "string",
-                "description": "The city and state, e.g., San Francisco, CA"
-              },
-              "unit": {
-                "type": "string",
-                "enum": ["Celsius", "Fahrenheit"],
-                "description": "The temperature unit to use. Infer this from the user's location."
-              }
+        type: 'function',
+        function: {
+          name: 'get_current_temperature',
+          description: 'Get the current temperature',
+          parameters: {
+            type: 'object',
+            properties: {
+              location: { type: 'string' },
+              unit: { type: 'string' }
             },
-            "required": ["location", "unit"]
+            required: %w(location unit)
           }
         }
-      },
+      }
     ]
 
-    sss 0
-    respose = LLM::Huggingface.ask prompt, model: 'HuggingFaceTB/SmolLM2-135M-Instruct', tool_choice: 'required', tools: tools do |name,arguments|
-      "It's raining cats and dogs"
+    response = LLM::Huggingface.ask("user: What is the weather in London?", client: client, tools: tools, log_response: false) do |_name, _arguments|
+      "It's 15 degrees and raining."
     end
 
-    ppp respose
+    assert_equal 'Take an umbrella.', response
+    assert_equal 2, client.calls
   end
 
-end
+  def test_format_tool_definitions
+    tools = {
+      'get_current_temperature' => [
+        nil,
+        {
+          name: 'get_current_temperature',
+          description: 'Get the current temperature',
+          parameters: {
+            type: 'object',
+            properties: {
+              location: { type: 'string' }
+            },
+            required: ['location'],
+            defaults: { unit: 'Celsius' }
+          }
+        }
+      ]
+    }
+
+    formatted = LLM::Huggingface.format_tool_definitions(tools)
 
+    assert_equal 'function', formatted.first[:type].to_s
+    assert_equal 'get_current_temperature', formatted.first.dig(:function, :name)
+    assert_nil formatted.first.dig(:function, :parameters, :defaults)
+  end
+end

data/test/scout/llm/backends/test_ollama.rb

@@ -3,14 +3,72 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 class TestLLMOllama < Test::Unit::TestCase
 
-  def _test_ask
-    Log.severity = 0
+  def test_ask
+    client = TestFixtures.ollama_client('backends/ollama')
+    res = LLM::OLlama.ask 'user: write a script that sorts files in a directory',
+                          client: client, model: 'mistral', mode: 'chat', persist: false
+
+    assert_equal 'Mock answer from Ollama', res
+    assert_equal 1, client.calls.length
+    assert_equal 'mistral', client.calls.first[:model]
+    assert client.calls.first[:messages].any? { |m| m[:role].to_s == 'user' }
+  end
+
+  def test_tool
     prompt =<<-EOF
-system: you are a coding helper that only write code and inline comments. No extra explanations or comentary
-system: Avoid using backticks ``` to format code.
-user: write a script that sorts files in a directory 
+What is the weather in London. Should I take an umbrella?
     EOF
-    ppp LLM::OLlama.ask prompt, model: 'mistral', mode: 'chat'
+
+    tools = [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_current_temperature",
+          "description": "Get the current temperature for a specific location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": {
+                "type": "string",
+                "description": "The city and state, e.g., San Francisco, CA"
+              },
+              "unit": {
+                "type": "string",
+                "enum": ["Celsius", "Fahrenheit"],
+                "description": "The temperature unit to use. Infer this from the user's location."
+              }
+            },
+            "required": ["location", "unit"]
+          }
+        }
+      },
+    ]
+
+    client = TestFixtures.ollama_client('backends/ollama_tool_call', 'backends/ollama')
+    respose = LLM::OLlama.ask prompt, model: 'gpt-oss',
+                              client: client, tool_choice: 'required',
+                              tools: tools, persist: false do |name,arguments|
+      "It's raining cats and dogs"
+    end
+
+    assert_equal 'Mock answer from Ollama', respose
+    assert_equal 2, client.calls.length
+
+    # ScoutCoder: LLM::OLlama#query calls client.chat(parameters) positionally
+    # (no `parameters:` kwarg) and the API returns an Array of chunk hashes.
+    sent_tools = client.calls.first[:tools]
+    assert sent_tools.any? { |t| (t.dig(:function, :name) || t.dig('function', 'name')) == 'get_current_temperature' }
+  end
+
+  def test_embeddings
+    payload = [{ 'embeddings' => [[0.1, 0.2, 0.3]] }]
+    client = FakeOllamaClient.new(payload)
+
+    emb = LLM::OLlama.embed 'Some text', client: client, model: 'mxbai-embed-large'
+
+    assert(Float === emb.first)
+    assert_equal [0.1, 0.2, 0.3], emb
+    assert_equal 'Some text', client.calls.first[:input]
   end
 
   def _test_tool_call_output
@@ -89,22 +147,14 @@ What is the weather in London. Should I take an umbrella?
     ppp respose
   end
 
-  def _test_embeddings
-    Log.severity = 0
-    text =<<-EOF
-Some text
-    EOF
-    emb = LLM::OLlama.embed text, model: 'mxbai-embed-large', url: 'localhost:3331' 
-    assert(Float === emb.first)
-  end
-
   def test_embedding_array
-    Log.severity = 0
-    text =<<-EOF
-Some text
-    EOF
-    emb = LLM::OLlama.embed [text], model: 'mxbai-embed-large', url: 'localhost:3331' 
+    payload = [{ 'embeddings' => [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]] }]
+    client = FakeOllamaClient.new(payload)
+
+    emb = LLM::OLlama.embed ['Some text', 'More text'], client: client, model: 'mxbai-embed-large'
+
     assert(Float === emb.first.first)
+    assert_equal [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], emb
   end
 end
 

data/test/scout/llm/backends/test_openwebui.rb

@@ -2,13 +2,12 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestOpenWebUI < Test::Unit::TestCase
-  def test_gepeto
+  # Real https://gepeto.bsc.es/api version moved to
+  # test/integration/scout/llm/backends/test_openwebui.rb (no client seam:
+  # LLM::OpenWebUIMethods#query posts through RestClient.post with a plain
+  # Hash "client").
+  def _test_gepeto
     Log.severity = 0
-    prompt =<<-EOF
-system: you are a coding helper that only write code and comments without formatting so that it can work directly, avoid the initial and end commas ```.
-user: write a script that sorts files in a directory 
-    EOF
-    
     prompt =<<-EOF
 user: write a script that sorts files in a directory 
     EOF
@@ -16,42 +15,45 @@ user: write a script that sorts files in a directory
     ppp LLM::OpenWebUI.ask prompt, model: 'qwen3-vl:30b', url: "https://gepeto.bsc.es/api"
   end
 
-  def _test_tool
-    prompt =<<-EOF
-What is the weather in London. Should I take an umbrella?
-    EOF
+  # Offline: stub RestClient.post so only request construction and response
+  # parsing are exercised. OpenWebUI is OpenAI-compatible, so the openai_chat
+  # fixture is replayed as the response body.
+  def test_ask_request_construction
+    fixture = TestFixtures.fixture('backends/openai_chat')
+    expected_answer = fixture.dig('choices', 0, 'message', 'content')
+
+    recorded = []
+    original = RestClient.method(:post)
 
-    tools = [
-      {
-        "type": "function",
-        "function": {
-          "name": "get_current_temperature",
-          "description": "Get the current temperature for a specific location",
-          "parameters": {
-            "type": "object",
-            "properties": {
-              "location": {
-                "type": "string",
-                "description": "The city and state, e.g., San Francisco, CA"
-              },
-              "unit": {
-                "type": "string",
-                "enum": ["Celsius", "Fahrenheit"],
-                "description": "The temperature unit to use. Infer this from the user's location."
-              }
-            },
-            "required": ["location", "unit"]
-          }
-        }
-      },
-    ]
-
-    sss 0
-    respose = LLM::OpenWebUI.ask prompt, model: 'gemma2:latest', tool_choice: 'required', tools: tools do |name,arguments|
-      "It's raining cats and dogs"
+    RestClient.define_singleton_method(:post) do |url, payload, headers|
+      recorded << {url: url, payload: JSON.parse(payload), headers: headers}
+      body = Struct.new(:body).new(fixture.to_json)
+      body
     end
 
-    ppp respose
+    begin
+      answer = LLM::OpenWebUI.ask 'user: write a script that sorts files in a directory offline',
+                                  model: 'qwen3-vl:30b', url: 'https://openwebui.example/api',
+                                  key: 'test-key', persist: false
+    ensure
+      RestClient.singleton_class.send(:define_method, :post, original)
+    end
+
+    assert_equal expected_answer, answer
+
+    assert_equal 1, recorded.length
+    call = recorded.first
+
+    assert call[:url].end_with?('chat/completions')
+    assert call[:url].start_with?('https://openwebui.example/api')
+
+    payload = call[:payload]
+    assert_equal 'qwen3-vl:30b', payload['model']
+    assert payload['messages'].any? { |m| m['role'] == 'user' }
+
+    headers = call[:headers]
+    assert headers['Authorization'] || headers[:Authorization]
+    assert_equal 'Bearer test-key', (headers['Authorization'] || headers[:Authorization]).to_s
+    assert((headers['Content-Type'] || headers[:Content_Type]).to_s.include?('application/json'))
   end
 end
-