scout-ai 1.2.5 → 2.0.0

data/test/scout/llm/backends/test_bedrock.rb

@@ -2,15 +2,132 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestLLMBedrock < Test::Unit::TestCase
+  # Offline Bedrock coverage: LLM::Bedrock.ask/embed accept options[:client]
+  # (an object with invoke_model(model_id:, content_type:, body:) returning
+  # something whose .body.string is JSON), so FakeBedrockClient exercises the
+  # full request construction + response parsing path without AWS credentials.
+  #
+  # The old real-service versions are kept disabled below (_test_*).
+
+  # ScoutCoder: LLM::Bedrock is not required by lib/scout-ai.rb; each test has
+  # to require 'scout/llm/backends/bedrock' itself (same as LLM.ask does when
+  # dispatching to the :bedrock backend).
+  def test_ask
+    require 'scout/llm/backends/bedrock'
+    client = TestFixtures.bedrock_client('backends/bedrock')
+
+    response = LLM::Bedrock.ask 'user: say hi to bedrock',
+                                client: client,
+                                model: 'anthropic.claude-3-sonnet-20240229-v1:0',
+                                model_max_tokens: 100
+
+    assert_equal 'Mock answer from Bedrock', response
+
+    assert_equal 1, client.calls.length
+    call = client.calls.first
+    assert_equal 'anthropic.claude-3-sonnet-20240229-v1:0', call[:model_id]
+    assert_equal 'application/json', call[:content_type]
+    # ScoutCoder: IndiferentHash#pretty_print takes 0 args, so assert on
+    # individual keys instead of pp-ing the recorded parameters.
+    assert_equal 100, call[:body]['max_tokens']
+    assert call[:body]['messages'].any? { |m| m['role'] == 'user' }
+    assert call[:body]['messages'].any? { |m| m['content'].to_s.include?('say hi to bedrock') }
+  end
+
+  def test_ask_prompt_type
+    require 'scout/llm/backends/bedrock'
+    client = TestFixtures.bedrock_client('backends/bedrock')
+
+    response = LLM::Bedrock.ask 'user: say hi through the prompt endpoint',
+                                client: client, type: :prompt,
+                                model: 'meta.llama3-8b-instruct-v1:0',
+                                model_max_tokens: 100
+
+    assert_equal 'Mock answer from Bedrock', response
+
+    body = client.calls.first[:body]
+    assert body.include?('prompt')
+    assert body['prompt'].to_s.include?('say hi through the prompt endpoint')
+  end
+
+  def test_embeddings
+    require 'scout/llm/backends/bedrock'
+    client = TestFixtures.bedrock_client('backends/bedrock_embedding')
+
+    emb = LLM::Bedrock.embed 'Some text', client: client,
+                             model: 'amazon.titan-embed-text-v1'
+
+    assert(Float === emb.first)
+    assert_equal [0.1, 0.2, 0.3], emb
+    assert_equal 1, client.calls.length
+
+    call = client.calls.first
+    assert_equal 'amazon.titan-embed-text-v1', call[:model_id]
+    assert_equal 'Some text', call[:body]['inputText']
+  end
+
+  # Tool loop: the first payload carries a tool_call content entry, the block
+  # answers it, and the second payload is the final text answer.
+  def test_tool_loop
+    require 'scout/llm/backends/bedrock'
+    tools = [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_current_temperature",
+          "description": "Get the current temperature for a specific location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": { "type": "string",
+                            "description": "The city and state, e.g., San Francisco, CA" },
+              "unit": { "type": "string", "enum": ["Celsius", "Fahrenheit"],
+                        "description": "The temperature unit to use." }
+            },
+            "required": ["location", "unit"]
+          }
+        }
+      }
+    ]
+
+    client = TestFixtures.bedrock_client('backends/bedrock_tool_use', 'backends/bedrock')
+
+    calls_seen = []
+    response = LLM::Bedrock.ask 'user: What is the weather in London? Should I take an umbrella? Use the tool.',
+                                tools: tools, client: client,
+                                model: 'anthropic.claude-3-sonnet-20240229-v1:0',
+                                model_max_tokens: 100 do |name, arguments|
+      calls_seen << [name, arguments]
+      "It's 15 degrees and raining."
+    end
+
+    assert_equal 'Mock answer from Bedrock', response
+
+    # the block was reached with the unwrapped tool call
+    assert_equal 1, calls_seen.length
+    assert_equal 'get_current_temperature', calls_seen.first.first
+    assert_equal 'London', calls_seen.first.last['location']
+
+    # two invoke_model rounds, the second one carrying the tool response
+    assert_equal 2, client.calls.length
+    sent = client.calls.last[:body]['messages']
+    assert sent.any? { |m| m['role'] == 'tool' }
+    tool_messages = sent.select { |m| m['role'] == 'tool' }
+    assert_equal 'It\'s 15 degrees and raining.', tool_messages.last['content']
+    assert_equal 'call_1', tool_messages.last['id']
+  end
+
   def _test_ask
+    # Real-service version: see the offline tests above. Requires AWS
+    # credentials and network.
     prompt =<<-EOF
 say hi
     EOF
     ppp LLM::Bedrock.ask prompt, model: "anthropic.claude-3-sonnet-20240229-v1:0", model_max_tokens: 100, model_anthropic_version: 'bedrock-2023-05-31'
   end
 
-
   def _test_embeddings
+    # Real-service version: see test_embeddings above.
     Log.severity = 0
     text =<<-EOF
 Some text
@@ -57,4 +174,3 @@ What is the weather in London. Should I take my umbrella? Use the provided tool
     ppp response
   end
 end
-

data/test/scout/llm/backends/test_ollama.rb

@@ -3,14 +3,72 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 class TestLLMOllama < Test::Unit::TestCase
 
-  def _test_ask
-    Log.severity = 0
+  def test_ask
+    client = TestFixtures.ollama_client('backends/ollama')
+    res = LLM::OLlama.ask 'user: write a script that sorts files in a directory',
+                          client: client, model: 'mistral', mode: 'chat', persist: false
+
+    assert_equal 'Mock answer from Ollama', res
+    assert_equal 1, client.calls.length
+    assert_equal 'mistral', client.calls.first[:model]
+    assert client.calls.first[:messages].any? { |m| m[:role].to_s == 'user' }
+  end
+
+  def test_tool
     prompt =<<-EOF
-system: you are a coding helper that only write code and inline comments. No extra explanations or comentary
-system: Avoid using backticks ``` to format code.
-user: write a script that sorts files in a directory 
+What is the weather in London. Should I take an umbrella?
     EOF
-    ppp LLM::OLlama.ask prompt, model: 'mistral', mode: 'chat'
+
+    tools = [
+      {
+        "type": "function",
+        "function": {
+          "name": "get_current_temperature",
+          "description": "Get the current temperature for a specific location",
+          "parameters": {
+            "type": "object",
+            "properties": {
+              "location": {
+                "type": "string",
+                "description": "The city and state, e.g., San Francisco, CA"
+              },
+              "unit": {
+                "type": "string",
+                "enum": ["Celsius", "Fahrenheit"],
+                "description": "The temperature unit to use. Infer this from the user's location."
+              }
+            },
+            "required": ["location", "unit"]
+          }
+        }
+      },
+    ]
+
+    client = TestFixtures.ollama_client('backends/ollama_tool_call', 'backends/ollama')
+    respose = LLM::OLlama.ask prompt, model: 'gpt-oss',
+                              client: client, tool_choice: 'required',
+                              tools: tools, persist: false do |name,arguments|
+      "It's raining cats and dogs"
+    end
+
+    assert_equal 'Mock answer from Ollama', respose
+    assert_equal 2, client.calls.length
+
+    # ScoutCoder: LLM::OLlama#query calls client.chat(parameters) positionally
+    # (no `parameters:` kwarg) and the API returns an Array of chunk hashes.
+    sent_tools = client.calls.first[:tools]
+    assert sent_tools.any? { |t| (t.dig(:function, :name) || t.dig('function', 'name')) == 'get_current_temperature' }
+  end
+
+  def test_embeddings
+    payload = [{ 'embeddings' => [[0.1, 0.2, 0.3]] }]
+    client = FakeOllamaClient.new(payload)
+
+    emb = LLM::OLlama.embed 'Some text', client: client, model: 'mxbai-embed-large'
+
+    assert(Float === emb.first)
+    assert_equal [0.1, 0.2, 0.3], emb
+    assert_equal 'Some text', client.calls.first[:input]
   end
 
   def _test_tool_call_output
@@ -89,22 +147,14 @@ What is the weather in London. Should I take an umbrella?
     ppp respose
   end
 
-  def _test_embeddings
-    Log.severity = 0
-    text =<<-EOF
-Some text
-    EOF
-    emb = LLM::OLlama.embed text, model: 'mxbai-embed-large', url: 'localhost:3331' 
-    assert(Float === emb.first)
-  end
-
   def test_embedding_array
-    Log.severity = 0
-    text =<<-EOF
-Some text
-    EOF
-    emb = LLM::OLlama.embed [text], model: 'mxbai-embed-large', url: 'localhost:3331' 
+    payload = [{ 'embeddings' => [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]] }]
+    client = FakeOllamaClient.new(payload)
+
+    emb = LLM::OLlama.embed ['Some text', 'More text'], client: client, model: 'mxbai-embed-large'
+
     assert(Float === emb.first.first)
+    assert_equal [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]], emb
   end
 end
 

data/test/scout/llm/backends/test_openwebui.rb

@@ -2,13 +2,12 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestOpenWebUI < Test::Unit::TestCase
-  def test_gepeto
+  # Real https://gepeto.bsc.es/api version moved to
+  # test/integration/scout/llm/backends/test_openwebui.rb (no client seam:
+  # LLM::OpenWebUIMethods#query posts through RestClient.post with a plain
+  # Hash "client").
+  def _test_gepeto
     Log.severity = 0
-    prompt =<<-EOF
-system: you are a coding helper that only write code and comments without formatting so that it can work directly, avoid the initial and end commas ```.
-user: write a script that sorts files in a directory 
-    EOF
-    
     prompt =<<-EOF
 user: write a script that sorts files in a directory 
     EOF
@@ -16,42 +15,45 @@ user: write a script that sorts files in a directory
     ppp LLM::OpenWebUI.ask prompt, model: 'qwen3-vl:30b', url: "https://gepeto.bsc.es/api"
   end
 
-  def _test_tool
-    prompt =<<-EOF
-What is the weather in London. Should I take an umbrella?
-    EOF
+  # Offline: stub RestClient.post so only request construction and response
+  # parsing are exercised. OpenWebUI is OpenAI-compatible, so the openai_chat
+  # fixture is replayed as the response body.
+  def test_ask_request_construction
+    fixture = TestFixtures.fixture('backends/openai_chat')
+    expected_answer = fixture.dig('choices', 0, 'message', 'content')
+
+    recorded = []
+    original = RestClient.method(:post)
 
-    tools = [
-      {
-        "type": "function",
-        "function": {
-          "name": "get_current_temperature",
-          "description": "Get the current temperature for a specific location",
-          "parameters": {
-            "type": "object",
-            "properties": {
-              "location": {
-                "type": "string",
-                "description": "The city and state, e.g., San Francisco, CA"
-              },
-              "unit": {
-                "type": "string",
-                "enum": ["Celsius", "Fahrenheit"],
-                "description": "The temperature unit to use. Infer this from the user's location."
-              }
-            },
-            "required": ["location", "unit"]
-          }
-        }
-      },
-    ]
-
-    sss 0
-    respose = LLM::OpenWebUI.ask prompt, model: 'gemma2:latest', tool_choice: 'required', tools: tools do |name,arguments|
-      "It's raining cats and dogs"
+    RestClient.define_singleton_method(:post) do |url, payload, headers|
+      recorded << {url: url, payload: JSON.parse(payload), headers: headers}
+      body = Struct.new(:body).new(fixture.to_json)
+      body
     end
 
-    ppp respose
+    begin
+      answer = LLM::OpenWebUI.ask 'user: write a script that sorts files in a directory offline',
+                                  model: 'qwen3-vl:30b', url: 'https://openwebui.example/api',
+                                  key: 'test-key', persist: false
+    ensure
+      RestClient.singleton_class.send(:define_method, :post, original)
+    end
+
+    assert_equal expected_answer, answer
+
+    assert_equal 1, recorded.length
+    call = recorded.first
+
+    assert call[:url].end_with?('chat/completions')
+    assert call[:url].start_with?('https://openwebui.example/api')
+
+    payload = call[:payload]
+    assert_equal 'qwen3-vl:30b', payload['model']
+    assert payload['messages'].any? { |m| m['role'] == 'user' }
+
+    headers = call[:headers]
+    assert headers['Authorization'] || headers[:Authorization]
+    assert_equal 'Bearer test-key', (headers['Authorization'] || headers[:Authorization]).to_s
+    assert((headers['Content-Type'] || headers[:Content_Type]).to_s.include?('application/json'))
   end
 end
-

data/test/scout/llm/backends/test_relay.rb

@@ -2,9 +2,11 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestRelay < Test::Unit::TestCase
-  def test_ask
+  # Real scp/ssh relay version moved to
+  # test/integration/scout/llm/backends/test_relay.rb: LLM::Relay shells out
+  # to `scp` and has no client seam, so the unit copy stays disabled.
+  def _test_ask
     Scout::Config.set(:server, 'localhost', :relay)
     ppp LLM::Relay.ask 'Say hi', model: 'gemma2'
   end
 end
-

data/test/scout/llm/chat/agent_meta_fixtures.rb

@@ -0,0 +1,131 @@
+# Reusable offline fixtures for agent_meta provenance tests (plan fixtures C,
+# D, E; also intended for the later token-collector rounds).  Everything runs
+# on TmpFile.with_dir directories plus persisted-style chat text; no providers.
+#
+# Helpers provided:
+#   write_chat(dir, name, text)                         -> chat file path
+#   meta_receipt(content)                               -> agent_meta entry Hash
+#   receipt_output(call_id, agent_meta, ...)            -> JSON envelope String
+#   receipt_chat_text(receipts, extra: nil)             -> persisted chat text
+#   plain_delegation_chat(job_path)                     -> persisted chat text
+#   make_job(dir, ref, dependencies:, logs:)            -> job path
+#   visit_signature(visits)                             -> comparable Array
+#   fixture_c(dir)                                      -> C/D layout paths
+#   truncated_receipt_chat(call_id, agent_meta)          -> fixture H text
+require 'fileutils'
+require 'json'
+
+module AgentMetaFixtures
+  # Write persisted-style chat text and return its absolute path.
+  def write_chat(dir, name, text)
+    path = File.expand_path(File.join(dir, name))
+    FileUtils.mkdir_p(File.dirname(path))
+    File.write(path, text)
+    path
+  end
+
+  # One agent_meta receipt entry with direct meta content, e.g.
+  # meta_receipt('pt=10 ct=4 tt=14 inference_id=w1') or
+  # meta_receipt('job=Worker/ask/Default_w').
+  def meta_receipt(content)
+    {role: 'meta', content: content}
+  end
+
+  # JSON payload of a function_call_output envelope carrying agent_meta.
+  def receipt_output(call_id, agent_meta, name: 'ask', content: 'child answer')
+    {name: name, content: content, id: call_id, agent_meta: agent_meta}.to_json
+  end
+
+  # Persisted chat text with one paired ask call per receipt entry.  Hash keys
+  # are call ids, values are the agent_meta payloads (Arrays, Strings, ...).
+  # `extra` lines are appended after the receipts (e.g. local meta lines).
+  #
+  # Message indexes produced by Chat.parse (single user turn, no leading
+  # empty user message since 49c0d20):
+  #   0 user, then per receipt: function_call, function_call_output.
+  def receipt_chat_text(receipts, extra: nil)
+    lines = ['user: Run the worker']
+    receipts.each do |call_id, agent_meta|
+      lines << 'function_call: ' + %({"name":"ask","arguments":{},"id":"#{call_id}"})
+      lines << 'function_call_output: ' + receipt_output(call_id, agent_meta)
+    end
+    lines.concat(Array(extra)) if extra
+    lines << 'assistant: done'
+    lines * "\n" + "\n"
+  end
+
+  # Persisted chat text delegating to `job_path` through an ordinary local
+  # `meta: job=` message (no receipts at all).
+  def plain_delegation_chat(job_path)
+    "user: Run the worker\nmeta: job=#{job_path}\nassistant: done\n"
+  end
+
+  # Create a job layout under `dir` for the relative reference `ref`
+  # (e.g. 'Worker/ask/Default_w'): the result file, a .info sidecar with
+  # `dependencies` (always written, matching scout-gear Step), and log chats
+  # written under `<job>.files/log/<name>` (Hash name -> chat text).  Returns
+  # the job path.
+  def make_job(dir, ref, result: 'answer', dependencies: [], logs: {})
+    path = File.expand_path(File.join(dir, ref))
+    FileUtils.mkdir_p(File.dirname(path))
+    File.write(path, result)
+    File.write(path + '.info', {dependencies: dependencies}.to_json)
+    logs.each do |name, text|
+      log_path = File.join(path + '.files', 'log', name)
+      FileUtils.mkdir_p(File.dirname(log_path))
+      File.write(log_path, text)
+    end
+    path
+  end
+
+  # Comparable signature of traverse_provenance visits:
+  # [kind, path, relation, first_visit].
+  def visit_signature(visits)
+    visits.collect do |kind, object, _parent_kind, _parent, relation, first|
+      [kind, object.respond_to?(:path) ? object.path.to_s : object.to_s, relation, first]
+    end
+  end
+
+  # Fixture C layout (plan fixtures C/D): a parent receipt points at a Worker
+  # job whose log chat has two direct metas (w1, w2), a nested receipt for a
+  # Critic job (with one direct meta c1), and a dependency job.  The parent
+  # chat also carries an ordinary local `meta: job=` reference to the Critic so
+  # both discovery paths coexist, plus one receipt-only direct meta (shadow).
+  # Returns [parent_chat_path, worker_job_path, critic_job_path, dep_job_path].
+  def fixture_c(dir)
+    critic = make_job(dir, 'Critic/ask/Default_c')
+    dep = make_job(dir, 'Dep/load/Default_1')
+
+    worker_log = receipt_chat_text(
+      {'wc1' => [meta_receipt('pt=30 ct=10 tt=40 inference_id=c1'),
+                 meta_receipt("job=#{critic}")]},
+      extra: ['meta: pt=100 ct=50 tt=150 inference_id=w1',
+              'meta: pt=20 ct=10 tt=30 inference_id=w2']
+    )
+    worker = make_job(dir, 'Worker/ask/Default_w',
+                      dependencies: [dep], logs: {'agent.chat' => worker_log})
+
+    parent = write_chat(dir, 'parent.chat',
+                        receipt_chat_text(
+                          {'p1' => [meta_receipt('pt=10 tt=12 inference_id=shadow'),
+                                    meta_receipt("job=#{worker}")]},
+                          extra: ["meta: job=#{critic}"]
+                        ))
+
+    [parent, worker, critic, dep]
+  end
+
+  # Fixture H: the output content is the standard truncation exception JSON
+  # (error: :truncated) exactly as LLM.process_calls serializes it, while the
+  # agent_meta receipts survive in the same envelope.
+  def truncated_receipt_chat(call_id, agent_meta, name: 'ask', characters: 90_000)
+    exception_msg = "Function #{name} #{call_id} was executed successfully, but it returned #{characters} characters, which is more than the maximum of 30000. To protect the model context window this result was not returned."
+    content = {exception: exception_msg, stack: ['a', 'b']}.to_json
+    payload = {name: name, content: content, id: call_id, error: :truncated,
+               agent_meta: agent_meta}.to_json
+    "user: Run\n" +
+      %({"name":"#{name}","arguments":{},"id":"#{call_id}"}).sub(/^/, 'function_call: ') + "\n" +
+      "function_call_output: #{payload}\n" +
+      "assistant: done\n"
+  end
+end