scout-ai 1.2.5 → 2.0.0

data/test/support/mock_backend.rb

@@ -0,0 +1,215 @@
+# Test support: LLM::Mock, an offline scripted backend.
+#
+# Not named test_*.rb on purpose: the Rakefile test pattern
+# ('test/**/test_*.rb') must not collect this file.
+#
+# Registered into LLM::BACKENDS so it is reachable from LLM.ask / LLM.embed
+# once the endpoint yaml selects `backend: mock`.
+#
+# Scripting model
+# ---------------
+#   LLM::Mock.responses = ['an answer']
+#   LLM::Mock.responses = [{role: 'assistant', content: 'an answer'}]
+#   LLM::Mock.responses = ['first', {tool_calls: [{name: 'f', arguments: {}}]}, 'final']
+#   LLM::Mock.script('a') { ... }  # same, resetting first
+#
+# Each response entry may be:
+#   * a String                          -> plain answer
+#   * a Hash message (role/content)     -> one message
+#   * an Array of Hash messages         -> several messages at once
+#   * a Hash with :tool_calls           -> tool call round: the calls are
+#     delegated to LLM.process_calls (using options[:tools]) so the
+#     function_call / function_call_output messages match the real backends
+#   * a Proc called with (messages, options) returning any of the above
+#   * an Exception instance             -> raised
+#
+# Entries replay in order; when exhausted the last entry repeats.
+#
+# Tool loops: the real backends return output ending in a
+# function_call_output message and then re-enter `ask` with
+# messages + output (Backend#chain_tools). LLM.ask dispatches to the backend
+# only once, so the mock drives the same rounds internally; to keep tests
+# able to assert on what the model would see next, every round is recorded
+# in `.calls` with the messages accumulated so far (round 1 = the original
+# messages, round 2 = messages + marriages output, ...), matching the shape
+# of the repeated asks of the real backends.
+#
+# ScoutCoder: IndiferentHash#pretty_print takes 0 args while Ruby's PP passes
+# 1, so pp-ing one of these hashes inside a test failure (assertion diffs call
+# PP) raises ArgumentError and hides the real failure. Use Log.fingerprint or
+# .inspect when debugging IndiferentHash values in tests instead of pp.
+module LLM
+  module Mock
+    DIMENSIONS = 64
+
+    class << self
+      attr_accessor :responses
+
+      def script(*responses, &block)
+        self.reset!
+        @responses = responses.flatten
+        @setup = block if block_given?
+        self
+      end
+
+      def reset!
+        @responses = []
+        @index = 0
+        @calls = []
+        @setup = nil
+        @tool_definitions = {}
+        self
+      end
+
+      def calls
+        @calls ||= []
+      end
+
+      def next_response(messages = nil, options = {})
+        @index ||= 0
+        @responses = [@responses].flatten.compact
+        entry = @responses[@index] || @responses.last
+        @index += 1
+
+        if Proc === entry
+          entry = entry.call(messages, options)
+        end
+        entry
+      end
+
+      def index
+        @index ||= 0
+      end
+
+      #{{{ ask
+
+      # Tool definitions resolved for the last ask (same {name => [obj, def]}
+      # shape the real backends build), for assertions.
+      def tool_definitions
+        @tool_definitions ||= {}
+      end
+
+      def ask(messages, options = {}, &block)
+        messages = Chat.setup(Chat.prepare_prompt(messages)) unless Array === messages
+        options = IndiferentHash.setup(options.dup)
+
+        # Mirror Backend::ClassMethods: resolve tool:/kb:/association:
+        # directives (and explicit options[:tools]) once per ask. LLM.tools
+        # consumes the directive messages, so resolve before recording.
+        @tool_definitions = LLM::Mock.tools(messages, options)
+
+        tool_calls, response = [], []
+        round_messages = messages
+
+        loop do
+          calls << [round_messages, options]
+
+          entry = next_response(round_messages, options)
+
+          case entry
+          when Hash
+            if entry[:tool_calls]
+              script_calls = entry[:tool_calls].collect do |info|
+                info = IndiferentHash.setup(info.dup)
+                { name: info[:name] || info.dig(:function, :name),
+                  arguments: info[:arguments] || info.dig(:function, :arguments) || {},
+                  id: info[:id] || info[:call_id] || "call_#{@index}" }
+              end
+
+              new_calls = LLM.process_calls(tool_definitions, script_calls, &block).flatten
+              tool_calls.concat new_calls
+              # next round sees the original messages plus the tool call
+              # round, exactly like Backend#chain_tools does
+              round_messages = Chat.setup(messages + tool_calls)
+              next
+            else
+              response << IndiferentHash.setup(entry.dup)
+            end
+          when Array
+            entry.each { |m| response << IndiferentHash.setup(m.dup) }
+          when String
+            response << IndiferentHash.setup(role: :assistant, content: entry)
+          when nil
+            response << IndiferentHash.setup(role: :assistant, content: '')
+          else
+            raise Exception, entry if Exception === entry
+            response << IndiferentHash.setup(role: :assistant, content: entry.to_s)
+          end
+
+          break
+        end
+
+        output = (tool_calls + response).flatten
+
+        if options[:return_messages]
+          Chat.setup output
+        else
+          Chat.setup(output)
+          cleaned = Chat.clean(output)
+          return '' if cleaned.nil? || cleaned.empty?
+          cleaned.last[:content]
+        end
+      end
+
+      # Mirrors Backend::ClassMethods#tools: explicit options[:tools] plus the
+      # tool:/kb:/association: directives resolved from the messages
+      # (LLM.tools / LLM.associations -> Chat.tools / Chat.associations), so a
+      # scripted tool-call round exercises the same object/definition pairing
+      # the real backends use ({name => [obj, definition]}).
+      #
+      # ScoutCoder: 'tool:'/'association:'/'kb:' chat directives are NOT
+      # resolved by LLM.ask itself; each backend's #tools(messages, options)
+      # does it, and it *deletes* options[:tools] while at it. A custom backend
+      # registered in LLM::BACKENDS therefore has to call LLM.tools /
+      # LLM.associations itself or the directives are silently dropped.
+      def tools(messages, options)
+        tools = options[:tools]
+
+        case tools
+        when Array
+          tools = tools.inject({}) do |acc, definition|
+            definition = definition[:function] if Hash === definition && definition[:function] && definition[:name].nil?
+            acc.merge(definition[:name] => [nil, definition])
+          end
+        when nil
+          tools = {}
+        end
+
+        tools.merge!(LLM.tools(messages))
+        tools.merge!(LLM.associations(messages))
+        tools
+      end
+
+      #{{{ embed
+
+      # Deterministic bag-of-words embedding: each downcased word is hashed to
+      # a dimension and counted, then L2-normalized, so identical texts yield
+      # identical vectors and texts sharing words land near each other
+      # (cosine) for RAG nearest-neighbour assertions.
+      def embed(text, _options = {})
+        case text
+        when Array
+          text.collect { |t| embed_one(t) }
+        else
+          embed_one(text)
+        end
+      end
+
+      def embed_one(text)
+        vector = Array.new(DIMENSIONS, 0.0)
+
+        text.to_s.downcase.split(/\W+/).each do |word|
+          next if word.empty?
+          position = word.sum % DIMENSIONS
+          vector[position] += 1.0
+        end
+
+        norm = Math.sqrt(vector.inject(0.0) { |acc, v| v * v })
+        return vector if norm.zero?
+        vector.collect { |v| v / norm }
+      end
+    end
+  end
+end
+
+LLM::BACKENDS[:mock] = LLM::Mock

data/test/test_helper.rb

@@ -2,6 +2,36 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib')))
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
 require 'scout'
+require 'scout-ai'
+
+# Offline test endpoints/fixtures support. Files under test/support are
+# deliberately NOT named test_*.rb so the Rakefile pattern
+# ('test/**/test_*.rb') does not collect them as tests.
+require File.join(File.dirname(__FILE__), 'support', 'fixtures')
+require File.join(File.dirname(__FILE__), 'support', 'mock_backend')
+require File.join(File.dirname(__FILE__), 'support', 'fake_clients')
+require File.join(File.dirname(__FILE__), 'support', 'availability')
+
+# ScoutCoder: Scout.etc.AI['name'] resolves through the Scout path maps, and
+# the first map in Scout.map_order wins. Scout.prepend_path(:name, map) adds
+# the map AND unshifts it into map_order, which is what makes the repo-local
+# test/etc tree take precedence over ~/.scout/etc. Note the map must include
+# the literal '{TOPLEVEL}/{SUBPATH}' suffix and, because the path is 'etc/AI'
+# (TOPLEVEL 'etc', SUBPATH 'AI'), the map root must be the test directory so
+# it expands to <repo>/test/etc/AI/... .
+#
+# Only the offline 'mock' endpoint lives here. In particular the repo does
+# NOT define a 'test' endpoint: 'test' is expected to be provided at
+# installation level (~/.scout/etc/AI/test.yaml) and is consumed by the
+# infrastructure suite (`rake test_infrastructure`), never by `rake test`.
+Scout.prepend_path :test_etc, File.join(File.expand_path(File.dirname(__FILE__)), '{TOPLEVEL}/{SUBPATH}')
+
+# Default offline LLM configuration for the unit tests: backend is the
+# registered LLM::Mock (test/support/mock_backend.rb). No endpoint is set
+# at all, so no endpoint yaml is ever resolved on the unit path and no real
+# inference service can be reached from `rake test`.
+Scout::Config.set({backend: :mock}, :ask, :llm)
+Scout::Config.set({backend: :mock}, :embed, :llm)
 
 class Test::Unit::TestCase
 
@@ -27,8 +57,9 @@ class Test::Unit::TestCase
     Workflow.directory = tmpdir.var.jobs
     Workflow.workflows.each{|wf| wf.directory = Workflow.directory[wf.name] }
     Entity.entity_property_cache = tmpdir.entity_properties if defined?(Entity)
+    LLM::Mock.reset! if defined?(LLM::Mock)
   end
-  
+
   teardown do
     Open.rm_rf tmpdir
   end
@@ -51,7 +82,6 @@ class Test::Unit::TestCase
 
   def agent(name = nil, options = {})
     require 'scout/llm/agent'
-    options[:endpoint] = Scout::Config.get(:endpoint, :test)
     if name.nil?
       LLM::Agent.new(**options)
     else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-ai
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 2.0.0
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -97,22 +97,39 @@ files:
 - Rakefile
 - VERSION
 - bin/scout-ai
-- doc/Agent.md
-- doc/Chat.md
-- doc/LLM.md
+- doc/Improvements.md
 - doc/Model.md
-- doc/PythonAgentTasks.md
-- doc/RAG.md
-- doc/USER_GUIDE.md
+- doc/StartHere.md
+- doc/developer/Architecture.md
+- doc/developer/Backends.md
+- doc/developer/ChatLifecycle.md
+- doc/developer/DelegationInternals.md
+- doc/developer/DesignPrinciples.md
+- doc/developer/PromptProcessing.md
+- doc/developer/Provenance.md
+- doc/user/BuildingAgents.md
+- doc/user/Cookbook.md
+- doc/user/CoreConcepts.md
+- doc/user/Delegation.md
+- doc/user/GettingStarted.md
+- doc/user/ManagingContext.md
+- doc/user/MultiAgentWorkflows.md
+- doc/user/Python.md
+- doc/user/RunningInference.md
+- doc/user/ToolCalling.md
+- doc/user/WritingChats.md
 - lib/scout-ai.rb
 - lib/scout/llm/agent.rb
 - lib/scout/llm/agent/chat.rb
 - lib/scout/llm/agent/delegate.rb
 - lib/scout/llm/agent/iterate.rb
+- lib/scout/llm/agent/save.rb
+- lib/scout/llm/agent/workflow.rb
 - lib/scout/llm/ask.rb
 - lib/scout/llm/backends/anthropic.rb
 - lib/scout/llm/backends/bedrock.rb
 - lib/scout/llm/backends/default.rb
+- lib/scout/llm/backends/glm.rb
 - lib/scout/llm/backends/huggingface.rb
 - lib/scout/llm/backends/ollama.rb
 - lib/scout/llm/backends/openai.rb
@@ -121,15 +138,23 @@ files:
 - lib/scout/llm/backends/responses.rb
 - lib/scout/llm/backends/vllm.rb
 - lib/scout/llm/chat.rb
+- lib/scout/llm/chat/agent_meta.rb
 - lib/scout/llm/chat/annotation.rb
 - lib/scout/llm/chat/parse.rb
+- lib/scout/llm/chat/persist.rb
 - lib/scout/llm/chat/process.rb
 - lib/scout/llm/chat/process/clear.rb
 - lib/scout/llm/chat/process/files.rb
 - lib/scout/llm/chat/process/meta.rb
 - lib/scout/llm/chat/process/options.rb
 - lib/scout/llm/chat/process/tools.rb
+- lib/scout/llm/chat/prompt.rb
+- lib/scout/llm/chat/prompt/shorten_tools.rb
+- lib/scout/llm/chat/prompt/shorten_tools_epoch.rb
+- lib/scout/llm/chat/provenance.rb
+- lib/scout/llm/chat/tool_calls.rb
 - lib/scout/llm/embed.rb
+- lib/scout/llm/image.rb
 - lib/scout/llm/mcp.rb
 - lib/scout/llm/rag.rb
 - lib/scout/llm/tools.rb
@@ -174,42 +199,96 @@ files:
 - python/tests/test_chat_agent.py
 - python/tests/test_huggingface_eval.py
 - python/tests/test_runner.py
+- research/ChatAnalyst-required-changes.md
+- research/agent-delegation-analysis.md
+- research/agent-meta-provenance-integration-plan.md
+- research/agent-workflow-analysis.md
+- research/backends-analysis.md
+- research/chat-core-analysis.md
+- research/chatanalyst-provenance/00-baseline.md
+- research/chatanalyst-provenance/01-repo-map.md
+- research/chatanalyst-provenance/02-event-reconstruction.md
+- research/chatanalyst-provenance/03-duplication-evidence.md
+- research/chatanalyst-provenance/04-tooling-root-cause.md
+- research/chatanalyst-provenance/05-fix-plan.md
+- research/chatanalyst-provenance/07-critic-review.md
+- research/chatanalyst-provenance/final-report.md
+- research/chatanalyst-provenance/resumption.md
+- research/coding-philosophy-analysis.md
+- research/commands-analysis.md
+- research/multi-agent-patterns-analysis.md
+- research/prompt-strategies-analysis.md
+- research/prov-verbosity-fix-notes.md
+- research/provenance-analysis.md
+- research/provenance-navigation-design.md
+- research/synthesis-report.md
+- research/tools-system-analysis.md
 - scout-ai.gemspec
 - scout_commands/agent/ask
 - scout_commands/agent/find
 - scout_commands/agent/kb
-- scout_commands/documenter
 - scout_commands/llm/ask
 - scout_commands/llm/json
+- scout_commands/llm/md
 - scout_commands/llm/process
 - scout_commands/llm/process_queries
+- scout_commands/llm/prov
 - scout_commands/llm/server
 - scout_commands/llm/template
+- scout_commands/llm/word
 - scout_commands/workflow/mcp
 - share/server/chat.html
 - share/server/chat.js
+- share/word/reference.docx
 - test/data/cat.jpg
 - test/data/person/brothers
 - test/data/person/identifiers
 - test/data/person/marriages
 - test/data/person/parents
+- test/etc/AI/mock.yaml
+- test/fixtures/backends/anthropic.json
+- test/fixtures/backends/anthropic_tool_use.json
+- test/fixtures/backends/bedrock.json
+- test/fixtures/backends/bedrock_embedding.json
+- test/fixtures/backends/bedrock_tool_use.json
+- test/fixtures/backends/ollama.json
+- test/fixtures/backends/ollama_tool_call.json
+- test/fixtures/backends/openai_chat.json
+- test/fixtures/backends/openai_chat_tool_call.json
+- test/fixtures/backends/responses.json
+- test/fixtures/backends/responses_tool_call.json
+- test/integration/README.md
+- test/integration/scout/llm/backends/test_endpoints.rb
+- test/integration/scout/llm/backends/test_openwebui.rb
+- test/integration/scout/llm/backends/test_relay.rb
+- test/integration/scout/llm/test_infrastructure.rb
+- test/integration/scout/llm/test_mcp.rb
+- test/integration/scout/llm/tools/test_mcp.rb
+- test/integration/scout/model/test_base.rb
 - test/scout/llm/agent/test_chat.rb
+- test/scout/llm/agent/test_save.rb
+- test/scout/llm/agent/test_workflow.rb
 - test/scout/llm/backends/test_anthropic.rb
 - test/scout/llm/backends/test_bedrock.rb
 - test/scout/llm/backends/test_huggingface.rb
 - test/scout/llm/backends/test_ollama.rb
-- test/scout/llm/backends/test_openai.rb
 - test/scout/llm/backends/test_openwebui.rb
 - test/scout/llm/backends/test_relay.rb
-- test/scout/llm/backends/test_responses.rb
+- test/scout/llm/chat/agent_meta_fixtures.rb
+- test/scout/llm/chat/process/test_meta.rb
+- test/scout/llm/chat/process/test_normalize_usage.rb
+- test/scout/llm/chat/test_agent_meta.rb
+- test/scout/llm/chat/test_agent_meta_provenance.rb
+- test/scout/llm/chat/test_agent_meta_tokens.rb
 - test/scout/llm/chat/test_parse.rb
 - test/scout/llm/chat/test_process.rb
+- test/scout/llm/chat/test_prov_cli.rb
+- test/scout/llm/chat/test_provenance.rb
+- test/scout/llm/chat/test_tool_calls.rb
 - test/scout/llm/test_agent.rb
 - test/scout/llm/test_ask.rb
 - test/scout/llm/test_chat.rb
 - test/scout/llm/test_embed.rb
-- test/scout/llm/test_mcp.rb
-- test/scout/llm/test_parse.rb
 - test/scout/llm/test_rag.rb
 - test/scout/llm/test_tools.rb
 - test/scout/llm/test_utils.rb
@@ -229,6 +308,11 @@ files:
 - test/scout/network/test_entity.rb
 - test/scout/network/test_knowledge_base.rb
 - test/scout/network/test_paths.rb
+- test/support/availability.rb
+- test/support/fake_clients.rb
+- test/support/fixtures.rb
+- test/support/infrastructure_probes.rb
+- test/support/mock_backend.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/scout-ai
 licenses: