scout-ai 1.2.3 → 2.0.0

data/test/scout/model/python/huggingface/test_causal.rb

@@ -2,9 +2,15 @@ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 class TestClass < Test::Unit::TestCase
+  MODEL = 'mistralai/Mistral-7B-Instruct-v0.3'
+
+  # Conditional omission: runs only when the model is already in the local
+  # huggingface cache. The probe never downloads (see
+  # test/support/availability.rb).
   def test_eval_chat
+    omit "huggingface model #{MODEL}: #{Availability.hf_model_reason(MODEL)}" unless Availability.hf_model_cached?(MODEL)
     #model = CausalModel.new 'BSC-LT/salamandra-2b-instruct'
-    model = CausalModel.new 'mistralai/Mistral-7B-Instruct-v0.3'
+    model = CausalModel.new MODEL
 
     model.init
 
@@ -17,8 +23,9 @@ class TestClass < Test::Unit::TestCase
   end
 
   def test_eval_train
+    omit "huggingface model #{MODEL}: #{Availability.hf_model_reason(MODEL)}" unless Availability.hf_model_cached?(MODEL)
     #model = CausalModel.new 'BSC-LT/salamandra-2b-instruct'
-    model = CausalModel.new 'mistralai/Mistral-7B-Instruct-v0.3'
+    model = CausalModel.new MODEL
 
     model.init
 
@@ -30,4 +37,3 @@ class TestClass < Test::Unit::TestCase
     ])
   end
 end
-

data/test/scout/model/python/huggingface/test_classification.rb

@@ -1,17 +1,26 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
+MODEL = 'bert-base-uncased'
+
 class TestSequenceClassification < Test::Unit::TestCase
   def _test_eval_sequence_classification
-    model = SequenceClassificationModel.new 'bert-base-uncased', nil,
+    model = SequenceClassificationModel.new MODEL, nil,
       class_labels: %w(Bad Good)
 
     assert_include ["Bad", "Good"], model.eval("This is dog")
     assert_include ["Bad", "Good"], model.eval_list(["This is dog", "This is cat"]).first
   end
 
+  # Conditional omission: the probe only checks the local huggingface cache
+  # (never downloads) and that transformers/datasets are importable, so this
+  # runs whenever the environment can actually serve it.
   def test_train_sequence_classification
-    model = SequenceClassificationModel.new 'bert-base-uncased', nil,
+    reason = Availability.python_modules_reason('transformers', 'datasets')
+    omit "python infrastructure missing: #{reason}" if reason
+    omit "huggingface model #{MODEL}: #{Availability.hf_model_reason(MODEL)}" unless Availability.hf_model_cached?(MODEL)
+
+    model = SequenceClassificationModel.new MODEL, nil,
       class_labels: %w(Bad Good)
 
     model.init

data/test/scout/model/python/test_torch.rb

@@ -3,6 +3,8 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 class TestTorch < Test::Unit::TestCase
   def test_linear
+    omit "No python environment" unless Availability.python?
+    omit "Torch not installed" unless Availability.python_modules?(:torch)
     model = nil
 
     TmpFile.with_dir do |dir|

data/test/scout/model/python/torch/test_helpers.rb

@@ -3,7 +3,11 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/model/python/base'
 class TestTorchHelpers < Test::Unit::TestCase
+  # Conditional omission: the CUDA probe imports torch in a bounded
+  # subprocess and checks torch.cuda.is_available(), so this runs whenever a
+  # GPU-backed torch is actually present.
   def test_del
+    omit 'torch CUDA unavailable (probe: torch.cuda.is_available)' unless Availability.cuda?
     ScoutPython.init_scout
     ScoutPython.pyimport :torch
     batch = [[100.0]]

data/test/scout/model/test_base.rb

@@ -34,7 +34,10 @@ class TestClass < Test::Unit::TestCase
     assert_equal [4, 8], model.eval_list([1, 2])
   end
 
-  def test_R_model
+  # Real R + e1071 version moved to test/integration/scout/model/test_base.rb
+  # (installs the R package online); the trivial ScoutModel tests above stay
+  # unit-side.
+  def _test_R_model
     require 'rbbt-util'
     require 'rbbt/util/R'
 
@@ -114,4 +117,3 @@ cat(label, file="#{results}");
     end
   end
 end
-

data/test/support/availability.rb

@@ -0,0 +1,231 @@
+# Test support: bounded availability probes for conditional test omissions.
+#
+# Not named test_*.rb on purpose: the Rakefile test pattern
+# ('test/**/test_*.rb') must not collect this file.
+#
+# Every probe returns true/false (never raises, never downloads, never hangs).
+# Intended usage in a test:
+#
+#   omit 'torch CUDA unavailable' unless Availability.cuda?
+#
+# Guidelines honoured here:
+#   * Huggingface model presence is checked against the local HF cache only
+#     (HF_HOME/hub or ~/.cache/huggingface/hub), NEVER through the
+#     transformers API: importing transformers or building a tokenizer would
+#     start a multi-GB download for anything that is not already cached.
+#   * Python module probes run `python -c "import ..."` in a fresh process
+#     with a hard timeout, so a hanging import cannot block the suite.
+#   * Results are memoized per process: probes are cheap but not free.
+module Availability
+  DEFAULT_TIMEOUT = 60
+
+  class << self
+    def memo(name)
+      cache = (@cache ||= {})
+      return cache[name] if cache.key?(name)
+      cache[name] = begin
+        yield
+      rescue Exception
+        false
+      end
+    end
+
+    def python_executable
+      ENV['SCOUT_TEST_PYTHON'] || 'python'
+    end
+
+    # python itself is usable (python executable present and runnable)
+    def python?
+      memo(:python) do
+        run_bounded([python_executable, '-c', 'import sys; sys.exit(0)'], 30)
+        true
+      end
+    end
+
+    # all listed python modules import cleanly (fresh process, bounded)
+    def python_modules?(*mods)
+      memo(:"python_modules_#{mods.join(',')}") do
+        raise Errno::ENOENT unless python?
+        code = mods.collect { |m| "import #{m}" } * ';'
+        _out, _err, status = run_bounded([python_executable, '-c', code], DEFAULT_TIMEOUT, capture: true)
+        status == 0
+      end
+    end
+
+    # informative reason string when a python module is missing, nil otherwise
+    def python_modules_reason(*mods)
+      return 'python unavailable' unless python?
+      code = mods.collect { |m| "import #{m}" } * ';'
+      out, _err, status = run_bounded([python_executable, '-c', code], DEFAULT_TIMEOUT, capture: true)
+      return nil if status == 0
+      "python modules missing (#{mods * ','}): #{out.split("\n").last.to_s.strip}"
+    end
+
+    # torch is importable (does NOT imply CUDA)
+    def torch?
+      python_modules?('torch')
+    end
+
+    # torch is importable AND reports a usable CUDA device
+    def cuda?
+      memo(:cuda) do
+        return false unless torch?
+        code = 'import sys, torch; sys.exit(0 if torch.cuda.is_available() else 1)'
+        _out, _err, status = run_bounded([python_executable, '-c', code], DEFAULT_TIMEOUT, capture: true)
+        status == 0
+      end
+    end
+
+    # A Huggingface model is already in the local cache; this NEVER triggers a
+    # download (see the note at the top of the file).
+    def hf_model_cached?(repo)
+      memo(:"hf_model_#{repo}") do
+        hub = ENV['HF_HOME'] ? File.join(ENV['HF_HOME'], 'hub') : File.join(Dir.home, '.cache', 'huggingface', 'hub')
+        repo_dir = 'models--' + repo.to_s.tr('/', '--')
+        path = File.join(hub, repo_dir)
+        Dir.exist?(path) && Dir.glob(File.join(path, 'snapshots', '*')).any?
+      end
+    end
+
+    # Reason for a missing huggingface model
+    def hf_model_reason(repo)
+      return "huggingface model #{repo} not in local cache (HF_HOME=#{ENV['HF_HOME'] || '~/.cache/huggingface'}); probing would download it" unless hf_model_cached?(repo)
+      nil
+    end
+
+    # Rscript is present
+    def rscript?
+      memo(:rscript) { which?('Rscript') }
+    end
+
+    # R is present and the listed packages are installed (bounded Rscript -e)
+    def r_packages?(*pkgs)
+      memo(:"r_packages_#{pkgs.join(',')}") do
+        return false unless rscript?
+        code = "cat(if (all(c(#{pkgs.collect { |p| "'#{p}'" } * ', '}) %in% rownames(installed.packages()))) 'yes' else 'no')"
+        out, _err, status = run_bounded(['Rscript', '-e', code], DEFAULT_TIMEOUT, capture: true)
+        status == 0 && out.strip == 'yes'
+      end
+    end
+
+    def r_packages_reason(*pkgs)
+      return 'Rscript not found' unless rscript?
+      return nil if r_packages?(*pkgs)
+      "R packages missing: #{pkgs * ','}"
+    end
+
+    #{{{ endpoints
+
+    # An endpoint yaml exists in the Scout configuration paths (test/etc/AI,
+    # ~/.scout/etc/AI, ...). Purely local: no network access.
+    def endpoint_configured?(endpoint)
+      memo(:"endpoint_configured_#{endpoint}") do
+        Scout.etc.AI[endpoint.to_s].find_with_extension(:yaml).exists?
+      end
+    end
+
+    # The endpoint points to a host:port that accepts a TCP connection within
+    # the timeout. Only probed for endpoints whose config carries a url;
+    # endpoints without a url (mock, relay through scp, ...) count as
+    # reachable once configured.
+    def endpoint_reachable?(endpoint, timeout = 5)
+      memo(:"endpoint_reachable_#{endpoint}") do
+        path = Scout.etc.AI[endpoint.to_s].find_with_extension(:yaml)
+        return false unless path.exists?
+        url = path.yaml[:url] || path.yaml['url']
+        return true if url.nil?
+
+        require 'uri'
+        require 'socket'
+        uri = URI.parse(url.to_s)
+        port = uri.port || (uri.scheme == 'https' ? 443 : 80)
+        begin
+          Timeout.timeout(timeout) do
+            TCPSocket.open(uri.host, port) { |s| s.close }
+          end
+          true
+        rescue Exception
+          false
+        end
+      end
+    end
+
+    # Combined: configured AND (no url OR reachable)
+    def endpoint_available?(endpoint)
+      return false unless endpoint_configured?(endpoint)
+      endpoint_reachable?(endpoint)
+    end
+
+    def endpoint_reason(endpoint)
+      return "endpoint #{endpoint} not configured (no yaml in the Scout AI paths)" unless endpoint_configured?(endpoint)
+      return "endpoint #{endpoint} configured but not reachable" unless endpoint_reachable?(endpoint)
+      nil
+    end
+
+    # All endpoint names defined in Scout.etc.AI across the search paths
+    # (test/etc/AI plus the user configuration). Local filesystem only.
+    def configured_endpoints
+      memo(:configured_endpoints) do
+        Scout.etc.AI.glob('*.yaml').collect { |f| File.basename(f, '.yaml') }.uniq
+      end
+    end
+
+    # Endpoints that are real inference services (anything but the offline
+    # mock one defined by this test suite).
+    def real_endpoints
+      memo(:real_endpoints) do
+        configured_endpoints.reject { |e| e == 'mock' }
+      end
+    end
+
+    #{{{ internals
+
+    def which?(cmd)
+      ENV['PATH'].to_s.split(File::PATH_SEPARATOR).any? do |dir|
+        path = File.join(dir, cmd.to_s)
+        File.executable?(path)
+      end
+    end
+
+    # Runs cmd with a hard timeout. Returns [stdout, stderr, exit_status].
+    # ScoutCoder: Open3.capture3 threads cannot be killed, so Timeout.timeout
+    # only breaks the wait; the child is detached with Process.kill(:KILL) to
+    # guarantee the probe returns even when the command hangs (e.g. a python
+    # import that blocks on a broken environment).
+    def run_bounded(cmd, timeout = DEFAULT_TIMEOUT, capture: true)
+      out_r, out_w = IO.pipe
+      err_r, err_w = IO.pipe
+
+      pid = Process.spawn(*cmd, out: out_w, err: err_w)
+      out_w.close
+      err_w.close
+
+      status = nil
+      begin
+        Timeout.timeout(timeout) do
+          out = capture ? out_r.read : nil
+          err = capture ? err_r.read : nil
+          _pid, status = Process.wait2(pid)
+          return [out.to_s, err.to_s, status.exitstatus]
+        end
+      rescue Timeout::Error
+        begin
+          Process.kill(:KILL, pid)
+        rescue Errno::ESRCH
+        end
+        Process.detach(pid)
+        return ['', 'timeout', -1]
+      rescue Errno::ENOENT
+        begin
+          Process.kill(:KILL, pid)
+        rescue Errno::ESRCH, Errno::EPERM
+        end
+        return ['', 'not found', -1]
+      ensure
+        out_r.close rescue nil
+        err_r.close rescue nil
+      end
+      ['', 'unknown', -1]
+    end
+  end
+end

data/test/support/fake_clients.rb

@@ -0,0 +1,138 @@
+# Test support: offline stand-ins for the provider HTTP clients.
+#
+# Not named test_*.rb on purpose: the Rakefile test pattern
+# ('test/**/test_*.rb') must not collect this file.
+#
+# Each fake wraps one provider SDK client class. They all share:
+#   * a response list replayed in order (last entry repeats when exhausted)
+#   * `.calls` recording every parameters hash for assertions
+#   * optional failure injection: an Exception instance entry is raised
+#
+# Recorded parameters are plain Hash/IndiferentHash values so assertions can
+# deep-compare them against expected payloads without type surprises.
+class FakeResponseClientBase
+  attr_reader :calls, :responses
+
+  # NOTE: entries are NOT flattened. A single entry may itself be an Array
+  # (e.g. the Ollama API returns an array of chunk hashes per call, and
+  # OLlama#process_response iterates the returned value).
+  def initialize(*responses)
+    @responses = responses
+    @calls = []
+    @index = 0
+  end
+
+  # Replays the next scripted response. Exception entries are raised so tests
+  # can exercise provider error paths.
+  def next_response
+    entry = @responses[@index] || @responses.last
+    @index += 1
+    raise entry if Exception === entry
+    entry
+  end
+
+  def record(parameters)
+    @calls << IndiferentHash.setup(parameters.dup)
+  end
+end
+
+# ruby-openai Chat Completions + embeddings:
+#   client.chat(parameters:), client.embeddings(parameters:)
+class FakeOpenAIChatClient < FakeResponseClientBase
+  def chat(parameters: {})
+    record(parameters)
+    next_response
+  end
+
+  def embeddings(parameters: {})
+    record(parameters)
+    next_response
+  end
+end
+
+# ruby-openai Responses API: client.responses.create(parameters:)
+class FakeResponsesClient < FakeResponseClientBase
+  ResponseStub = Struct.new(:create)
+
+  def responses
+    self
+  end
+
+  def create(parameters: {})
+    record(parameters)
+    next_response
+  end
+end
+
+# anthropic gem: client.messages(parameters:)
+class FakeAnthropicClient < FakeResponseClientBase
+  def messages(parameters: {})
+    record(parameters)
+    next_response
+  end
+end
+
+# ollama-ai gem: client.chat(parameters) (positional),
+# client.request(path, parameters)
+class FakeOllamaClient < FakeResponseClientBase
+  def chat(parameters = {})
+    record(parameters)
+    next_response
+  end
+
+  def request(path, parameters = {})
+    record(parameters.merge(path: path))
+    next_response
+  end
+end
+
+# aws-sdk-bedrockruntime: client.invoke_model(model_id:, content_type:, body:)
+# returning an object whose .body.string is the JSON payload.
+class FakeBedrockClient
+  BodyStub = Struct.new(:string)
+  ResponseStub = Struct.new(:body)
+
+  attr_reader :calls
+
+  def initialize(*payloads)
+    @payloads = payloads
+    @calls = []
+    @index = 0
+  end
+
+  # Records the request (body parsed back to a Hash for assertions) and returns
+  # the next scripted payload wrapped as invoke_model does.
+  def invoke_model(model_id:, content_type:, body:)
+    payload = @payloads[@index] || @payloads.last
+    @index += 1
+    @calls << IndiferentHash.setup({model_id: model_id, content_type: content_type, body: JSON.parse(body)})
+    ResponseStub.new(BodyStub.new(payload.to_json))
+  end
+end
+
+module TestFixtures
+  class << self
+    # Build a fake client pre-loaded with the named fixtures, e.g.
+    # TestFixtures.openai_client('backends/openai_chat_tool_call',
+    #                            'backends/openai_chat')
+    def openai_client(*names)
+      FakeOpenAIChatClient.new(*names.collect { |n| fixture(n) })
+    end
+
+    def responses_client(*names)
+      FakeResponsesClient.new(*names.collect { |n| fixture(n) })
+    end
+
+    def anthropic_client(*names)
+      FakeAnthropicClient.new(*names.collect { |n| fixture(n) })
+    end
+
+    def ollama_client(*names)
+      FakeOllamaClient.new(*names.collect { |n| fixture(n) })
+    end
+
+    def bedrock_client(*names)
+      FakeBedrockClient.new(*names.collect { |n| fixture(n) })
+    end
+  end
+end

data/test/support/fixtures.rb

@@ -0,0 +1,21 @@
+# Test support: fixture loading for recorded LLM provider payloads.
+#
+# Not named test_*.rb on purpose: the Rakefile test pattern
+# ('test/**/test_*.rb') must not collect this file.
+module TestFixtures
+  FIXTURES_DIR = File.expand_path('../fixtures', __dir__)
+
+  class << self
+    # TestFixtures.fixture('backends/openai_chat')
+    #   -> parsed JSON from test/fixtures/backends/openai_chat.json
+    def fixture(name)
+      path = fixture_path(name)
+      content = Open.read(path)
+      JSON.parse(content)
+    end
+
+    def fixture_path(name)
+      Path.setup(File.join(FIXTURES_DIR, name.to_s + '.json')).find
+    end
+  end
+end

data/test/support/infrastructure_probes.rb

@@ -0,0 +1,136 @@
+# Test support: infrastructure probe collector shared by the endpoint suites.
+#
+# Not named test_*.rb on purpose: the Rakefile test pattern must not collect
+# this file.
+#
+# Runs a fixed set of probes against one inference target and reports the
+# outcome (including latency and token usage) as a summary instead of raising:
+#
+#   trivial  : answer a trivial question
+#   tool_call: one weather-style tool call round (a block answers the tool)
+#   kb_query : the Miki brother-in-law knowledge base question, a two-hop
+#              tool chain (marriages then brothers)
+#
+# Failures are recorded as :fail rows, never raised, so one broken endpoint
+# cannot break the run; the summary table is printed and written to
+# results/infrastructure_summary.md by whichever suite runs last.
+module InfrastructureProbes
+  TIMEOUT = 300
+
+  WEATHER_TOOL = {
+    "type": "function",
+    "function": {
+      "name": "get_current_temperature",
+      "description": "Get the current temperature and raining conditions for a specific location",
+      "parameters": {
+        "type": "object",
+        "properties": {
+          "location": { "type": "string", "description": "The city and state, e.g., San Francisco, CA" },
+          "unit": { "type": "string", "enum": ["Celsius", "Fahrenheit"],
+                    "description": "The temperature unit to use. Infer this from the user's location." }
+        },
+        "required": ["location", "unit"]
+      }
+    }
+  }
+
+  class << self
+    attr_accessor :results
+  end
+  self.results ||= []
+
+  module_function
+
+  def record(target, probe, status, reason: nil, answer: nil, duration: nil, tokens: nil)
+    InfrastructureProbes.results << {target: target, probe: probe, status: status, reason: reason,
+                                     answer: answer, duration: duration, tokens: tokens}
+  end
+
+  # Run the three probes against `target` (a display name), using
+  # `ask_options` (already carrying :endpoint or nothing, i.e. the default).
+  # Nothing raises: each probe is rescued and recorded.
+  def run_probes(target, ask_options = {})
+    nonce = Time.now.to_i
+
+    run_probe(target, :trivial, ask_options) do
+      answer = LLM.ask "user: Reply with the single word OK and nothing else (nonce #{nonce})",
+                       ask_options.merge(persist: false)
+      raise 'no answer' if answer.to_s.strip.empty?
+      answer
+    end
+
+    run_probe(target, :tool_call, ask_options) do
+      answer = LLM.ask "user: What is the weather in London? Use the provided tool and then answer in one sentence (nonce #{nonce}).",
+                       ask_options.merge(tools: [WEATHER_TOOL], tool_choice: 'required', persist: false) do |_name, _arguments|
+        "It's 15 degrees and raining."
+      end
+      raise 'no answer' if answer.to_s.strip.empty?
+      answer
+    end
+
+    run_probe(target, :kb_query, ask_options) do
+      TmpFile.with_dir do |dir|
+        kb = KnowledgeBase.new dir
+        kb.register :brothers, Test::Unit::TestCase.datafile_test(:person).brothers, undirected: true
+        kb.register :marriages, Test::Unit::TestCase.datafile_test(:person).marriages,
+                    undirected: true, source: "=>Alias", target: "=>Alias"
+
+        text = LLM.knowledge_base_ask(kb,
+          "Who is Miki's brother in law? The brother in law is your spouse's sibling. Use the marriages and brothers tools to find out (nonce #{nonce}).",
+          ask_options.merge(persist: false))
+        raise "answer did not mention Guille: #{text.to_s[0, 200]}" unless text.to_s.include?('Guille')
+        text
+      end
+    end
+  end
+
+  def run_probe(target, probe, ask_options)
+    start = Time.now
+    answer = Timeout.timeout(TIMEOUT) { yield }
+    record(target, probe, :ok, answer: answer, duration: Time.now - start, tokens: nil)
+  rescue Exception => e
+    record(target, probe, :fail,
+           reason: "#{e.class}: #{e.message.to_s.lines.first}",
+           duration: Time.now - start)
+  end
+
+  def summary_lines
+    lines = []
+    lines << 'Infrastructure endpoint summary'
+    lines << '=' * 96
+    InfrastructureProbes.results.each do |r|
+      line = [r[:target].to_s.ljust(14), r[:probe].to_s.ljust(11)]
+      case r[:status]
+      when :ok
+        line << 'OK'.ljust(5)
+        line << (r[:duration] ? "#{r[:duration].round(1)}s" : '').ljust(8)
+        line << " #{r[:answer].to_s.gsub(/\s+/, ' ')[0, 100]}"
+      when :fail
+        line << 'FAIL'.ljust(5)
+        line << (r[:duration] ? "#{r[:duration].round(1)}s" : '').ljust(8)
+        line << " #{r[:reason].to_s.gsub(/\s+/, ' ')[0, 140]}"
+      else
+        line << 'OMIT'.ljust(5)
+        line << " #{r[:reason].to_s.gsub(/\s+/, ' ')[0, 140]}"
+      end
+      lines << line * ' | '
+    end
+    lines << '=' * 96
+    lines << "#{InfrastructureProbes.results.count { |r| r[:status] == :ok }} ok, " +
+             "#{InfrastructureProbes.results.count { |r| r[:status] == :fail }} failed, " +
+             "#{InfrastructureProbes.results.count { |r| r[:status] == :omit }} omitted"
+    lines
+  end
+
+  def write_summary
+    return if InfrastructureProbes.results.empty?
+    content = summary_lines * "\n"
+    $stdout.puts "\n" + content
+    begin
+      Open.mkdir 'results'
+      Open.write 'results/infrastructure_summary.md', content + "\n"
+    rescue Exception
+      $stderr.puts "Could not write results/infrastructure_summary.md: #{$!.message}"
+    end
+  end
+end