scout-ai 1.2.3 → 2.0.0

data/test/integration/scout/model/test_base.rb

@@ -0,0 +1,91 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+
+# Integration (real infrastructure) copy of test/scout/model/test_base.rb:
+# test_R_model runs R code and needs the e1071 package. The trivial
+# ScoutModel tests stay unit-side in test/scout/model/test_base.rb.
+# Run with `rake test_integration`.
+#
+# Conditional omission: the probe checks for Rscript and the installed R
+# packages before running, and omits with the detected reason otherwise.
+class TestClass < Test::Unit::TestCase
+  def test_R_model
+    omit Availability.r_packages_reason('e1071') unless Availability.r_packages?('e1071')
+    require 'rbbt-util'
+    require 'rbbt/util/R'
+
+    text =<<-EOF
+1 0;1;1
+1 1;0;1
+1 1;1;1
+1 0;1;1
+1 1;;1
+0 0;1;0
+0 1;0;0
+0 0;1;0
+0 1;0;0
+    EOF
+
+    TmpFile.with_file do |dir|
+      Open.mkdir dir
+      model = ScoutModel.new dir
+
+      model.extract_features do |sample|
+        sample.split(";")
+      end
+
+      model.train do |list,labels|
+        TmpFile.with_file do |feature_file|
+          Open.write(feature_file, list.collect{|feats| feats * "\t"} * "\n")
+          Open.write(feature_file + '.class', labels * "\n")
+          R.run <<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+labels = scan("#{ feature_file }.class", what=numeric());
+features = cbind(features, class = labels);
+rbbt.require('e1071')
+model = svm(class ~ ., data = features)
+save(model, file="#{ state_file }");
+          EOF
+        end
+      end
+
+      model.eval do |features|
+        TmpFile.with_file do |feature_file|
+          TmpFile.with_file do |results|
+            Open.write(feature_file, features * "\t")
+            R.run <<-EOF
+features = read.table("#{ feature_file }", sep ="\\t", stringsAsFactors=FALSE);
+library(e1071)
+load(file="#{ state_file }")
+label = predict(model, features);
+cat(label, file="#{results}");
+            EOF
+
+            Open.read(results)
+          end
+        end
+      end
+
+      text.split(/\n/).each do |line|
+        label, sample = line.split(" ")
+        model.add(sample, label)
+      end
+
+      model.train
+
+      assert model.eval("1;1;1").to_f > 0.5
+      assert model.eval("0;0;0").to_f < 0.5
+
+      model.save
+
+      model = ScoutModel.new dir
+      assert model.eval("1;1;1").to_f > 0.5
+      assert model.eval("0;0;0").to_f < 0.5
+
+      model.post_process do |result|
+        result.to_f < 0.5 ? :bad : :good
+      end
+
+      assert_equal :bad, model.eval("0;0;0")
+    end
+  end
+end

data/test/scout/llm/agent/test_chat.rb

@@ -8,7 +8,13 @@ class TestAgent < Test::Unit::TestCase
     a = LLM::Agent.new
     a.start_chat.system 'you are a robot'
     a.user "hi"
-    ppp a.print
+
+    output = a.print
+    assert_kind_of String, output
+
+    # ScoutCoder: Chat.print renders each message as "role:\n\ncontent" (roles in
+    # the meta list are inlined instead); blocks are joined by a blank line and
+    # the whole string starts with a newline.
+    assert_equal "\nsystem:\n\nyou are a robot\n\nuser:\n\nhi", output
   end
 end
-

data/test/scout/llm/agent/test_save.rb

@@ -0,0 +1,413 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require 'scout/llm/agent'
+
+class TestLLMAgentSave < Test::Unit::TestCase
+  def setup
+    super
+    @dir = File.join(tmpdir, 'save_tests')
+    FileUtils.mkdir_p @dir
+  end
+
+  def simple_agent(content = 'you are a robot')
+    agent = LLM::Agent.new
+    agent.start_chat.system content
+    agent
+  end
+
+  def chat_files_under(dir)
+    Dir.glob(File.join(dir, '**', '*.chat')).sort
+  end
+
+  def test_society_dir_for
+    assert_equal 'a/b/c.society',
+                 LLM::Agent.society_dir_for('a/b/c.chat')
+    assert_equal 'a/b/worker.society',
+                 LLM::Agent.society_dir_for('a/b/worker.chat')
+    assert_equal './c.society',
+                 LLM::Agent.society_dir_for('c.chat')
+    agent = LLM::Agent.new
+    assert_equal 'a/b/c.society',
+                 agent.society_dir_for('a/b/c.chat')
+  end
+
+  def test_society_dir_by_location
+    agent = LLM::Agent.new
+
+    nested = 'a/society/Worker/w_1/agent.chat'
+    # A nested agent.chat nests its society as a SIBLING directory, never a
+    # second `.files` tree of its own, however the save was triggered.
+    assert_equal 'a/society/Worker/w_1/society', agent.society_dir(nested)
+    assert agent.nested_save?(nested)
+
+    assert_equal 'a/b/c.society', agent.society_dir('a/b/c.chat')
+    assert !agent.nested_save?('a/b/c.chat')
+  end
+
+  def test_save_explicit_path_writes_only_the_chat_file
+    TmpFile.with_dir do |dir|
+      agent = simple_agent
+      agent.user 'hello'
+
+      chat_file = File.join(dir, 'conversation.chat')
+      written = agent.save(chat_file)
+
+      assert_equal [File.expand_path(chat_file)], written
+      assert Open.exist?(chat_file)
+      assert_include Open.read(chat_file), 'hello'
+
+      # Lazy: no society, so no .files sidecar tree at all
+      assert !Open.exist?(chat_file + '.files')
+    end
+  end
+
+  def test_save_without_path_uses_save_file
+    TmpFile.with_dir do |dir|
+      agent = simple_agent
+      agent.user 'stored'
+
+      chat_file = File.join(dir, 'auto.chat')
+      agent.save_file = chat_file
+
+      assert_equal [File.expand_path(chat_file)], agent.save
+      assert_include Open.read(chat_file), 'stored'
+    end
+  end
+
+  def test_save_without_path_raises_when_unconfigured
+    agent = simple_agent
+    agent.user 'orphan'
+
+    assert_raise ScoutException do
+      agent.save
+    end
+  end
+
+  def test_save_nested_society
+    TmpFile.with_dir do |dir|
+      root = simple_agent('root')
+      root.user 'root question'
+
+      child = simple_agent('child')
+      child.user 'child question'
+      root.chats = { 'Worker/w_A' => child }
+
+      grandchild = simple_agent('grandchild')
+      grandchild.user 'grandchild question'
+      child.chats = { 'Critic/c_1' => grandchild }
+
+      chat_file = File.join(dir, 'root.chat')
+      written = root.save(chat_file)
+
+      expected_child = File.join(chat_file.sub(/\.chat\z/, '.society'), 'Worker', 'w_A', 'agent.chat')
+      expected_grandchild = File.join(chat_file.sub(/\.chat\z/, '.society'), 'Worker', 'w_A', 'society', 'Critic', 'c_1', 'agent.chat')
+
+      assert_equal [chat_file, expected_child, expected_grandchild].collect { |p| File.expand_path(p) }.sort, written
+
+      assert_include Open.read(expected_child), 'child question'
+      assert_include Open.read(expected_grandchild), 'grandchild question'
+
+      # child learned its own save_file for later independent auto-saves
+      assert_equal expected_child, child.save_file
+      assert_equal expected_grandchild, grandchild.save_file
+    end
+  end
+
+  def test_save_cyclic_society_terminates
+    TmpFile.with_dir do |dir|
+      a = simple_agent('a')
+      a.user 'a question'
+      b = simple_agent('b')
+      b.user 'b question'
+
+      a.chats = { 'Other/current' => b }
+      b.chats = { 'First/current' => a }
+
+      chat_file = File.join(dir, 'cycle.chat')
+      written = a.save(chat_file)
+
+      expected_b = File.join(chat_file.sub(/\.chat\z/, '.society'), 'Other', 'current', 'agent.chat')
+      assert_equal [chat_file, expected_b].collect { |p| File.expand_path(p) }.sort, written
+
+      # The tree is finite: b was saved once and going back to a was cut by
+      # the seen-agent check, so nothing was written twice.
+      assert_equal 1, chat_files_under(chat_file.sub(/\.chat\z/, '.society')).length
+    end
+  end
+
+  def test_save_skips_dead_entries
+    TmpFile.with_dir do |dir|
+      agent = simple_agent
+      agent.chats = { 'Worker/w_1' => nil, 'no-slash' => simple_agent, '/lead' => simple_agent }
+
+      chat_file = File.join(dir, 'dead.chat')
+      written = agent.save(chat_file)
+
+      assert_equal [File.expand_path(chat_file)], written
+    end
+  end
+
+  def test_save_sanitizes_weird_names
+    TmpFile.with_dir do |dir|
+      agent = simple_agent
+      bad = simple_agent('weird')
+      bad.user 'weird question'
+
+      agent.chats = { '../evil/name' => bad }
+
+      chat_file = File.join(dir, 'sanitized.chat')
+      written = agent.save(chat_file)
+
+      # '..' collapses to '_' and the '/' inside the conversation is
+      # replaced, so the file stays inside the society directory
+      weird_file = File.join(chat_file.sub(/\.chat\z/, '.society'), '__', 'evil_name', 'agent.chat')
+      assert_include written.collect { |p| File.expand_path(p) },
+                     File.expand_path(weird_file)
+      assert !Open.exist?(File.join(dir, 'name', 'agent.chat'))
+    end
+  end
+
+  def test_auto_save_after_ask
+    TmpFile.with_dir do |dir|
+      LLM::Mock.script('mock answer')
+
+      agent = simple_agent
+      chat_file = File.join(dir, 'auto_save.chat')
+      agent.save_file = chat_file
+
+      agent.user 'auto question'
+      res = agent.chat persist: false, endpoint: 'mock'
+
+      assert_equal 'mock answer', res
+      content = Open.read(chat_file)
+      assert_include content, 'auto question'
+      assert_include content, 'mock answer'
+    end
+  end
+
+  def test_child_auto_save_keeps_canonical_layout
+    TmpFile.with_dir do |dir|
+      LLM::Mock.script('mock answer')
+
+      root = simple_agent('root')
+      root.user 'root question'
+
+      child = simple_agent('child')
+      child.user 'child question'
+      root.chats = { 'Worker/w_A' => child }
+
+      chat_file = File.join(dir, 'root.chat')
+      root.save(chat_file)
+
+      child_file = File.join(chat_file.sub(/\.chat\z/, '.society'), 'Worker', 'w_A', 'agent.chat')
+      assert_equal child_file, child.save_file
+
+      # The child runs its own turn later (its own auto-save, no parent save):
+      # the answer must land in the child chat file and the grandchild must
+      # go to the SIBLING society dir, never to <child_file>.files/...
+      grandchild = simple_agent('grandchild')
+      grandchild.user 'grandchild question'
+      child.chats = { 'Critic/c_1' => grandchild }
+
+      child.chat persist: false, endpoint: 'mock'
+
+      assert_include Open.read(child_file), 'mock answer'
+
+      assert Open.exist?(File.join(File.dirname(child_file), 'society', 'Critic', 'c_1', 'agent.chat'))
+      assert !Open.exist?(child_file + '.files')
+      assert !Dir.glob(File.join(chat_file + '.files', '**', 'agent.chat.files')).any?
+    end
+  end
+
+  # A bare agent.ask round does not grow current_chat, so it must NOT
+  # auto-save: the auto-save hook fires on Agent#chat, not on ask.
+  def test_no_auto_save_after_bare_ask
+    TmpFile.with_dir do |dir|
+      # Plain-string script: no tool calls, so the round is a single
+      # assistant answer that never enters Agent#chat.
+      LLM::Mock.script('mock answer')
+
+      agent = simple_agent
+      chat_file = File.join(dir, 'ask_save.chat')
+      agent.save_file = chat_file
+      agent.user 'bare question'
+
+      assert_include agent.ask(agent.current_chat, persist: false, endpoint: :mock), 'mock answer'
+
+      # Bare ask: the chat was not grown by Agent#chat, so no file anywhere
+      assert !Open.exist?(chat_file)
+      assert_equal [], Dir.glob(File.join(dir, '**', '*.chat'))
+    end
+  end
+
+  # The chat path does persist growth: a round with a delegated tool call
+  # grows current_chat inside Agent#chat (function_call /
+  # function_call_output plus the answer), so the auto-save fires, and the
+  # delegated child conversation is saved with it into the society tree.
+  def test_auto_save_after_tool_round_via_chat
+    TmpFile.with_dir do |dir|
+      LLM::Mock.script([
+        {tool_calls: [{name: 'ask', arguments: {agent: 'Worker', prompt: 'hi'}}]},
+        'final answer'
+      ])
+
+      agent = simple_agent('root')
+      worker = simple_agent('worker')
+      agent.society = { 'Worker' => worker }
+      agent.socialize
+
+      chat_file = File.join(dir, 'tool_round.chat')
+      agent.save_file = chat_file
+      agent.user 'tool question'
+
+      assert_equal 'final answer', agent.chat(persist: false, endpoint: 'mock')
+
+      # The parent chat file holds the tool round and its answer
+      content = Open.read(chat_file)
+      assert_include content, 'tool question'
+      assert_include content, 'function_call'
+      assert_include content, 'final answer'
+
+      # The delegated child conversation was saved with it, lazily, under the
+      # canonical society layout
+      child_file = File.join(chat_file.sub(/\.chat\z/, '.society'), 'Worker', 'default', 'agent.chat')
+      assert Open.exist?(child_file)
+      assert_include Open.read(child_file), 'hi'
+    end
+  end
+
+  def test_no_auto_save_without_save_file
+    TmpFile.with_dir do |dir|
+      LLM::Mock.script('mock answer')
+
+      agent = simple_agent
+      agent.user 'volatile question'
+      agent.chat persist: false, endpoint: 'mock'
+
+      assert !Open.exist?(File.join(dir, 'never.chat'))
+      assert_equal [], Dir.glob(File.join(dir, '**', '*.chat'))
+    end
+  end
+
+  def test_restart_snapshot
+    TmpFile.with_dir do |dir|
+      agent = simple_agent
+      chat_file = File.join(dir, 'restarts.chat')
+      agent.save_file = chat_file
+
+      agent.user 'before restart'
+      agent.save
+
+      resets_dir = chat_file + '.files/resets'
+      assert !Open.exist?(resets_dir)
+
+      agent.start
+
+      assert Open.exist?(resets_dir)
+      snapshots = Dir.glob(File.join(resets_dir, '*.chat'))
+      assert_equal 1, snapshots.length
+      assert_include File.basename(snapshots.first), '.chat'
+      assert_include Open.read(snapshots.first), 'before restart'
+    end
+  end
+
+  def test_no_restart_snapshot_without_prior_chat
+    TmpFile.with_dir do |dir|
+      agent = simple_agent
+      chat_file = File.join(dir, 'fresh.chat')
+      agent.save_file = chat_file
+
+      agent.start
+
+      assert !Open.exist?(chat_file + '.files/resets')
+      assert !Open.exist?(chat_file + '.files')
+    end
+  end
+end
+
+# Step 3b: Agent#society_save_file (private, delegate.rb) must derive the
+# society directory with the CANONICAL rule (Agent.society_dir_for /
+# Agent#society_dir), never by unanchored string substitution on save_file.
+#
+# The bug: `"cli.chat.files/agent.chat".sub(/\.chat/, '.society')` replaces
+# the FIRST '.chat' and yields `cli.society.files/agent.chat/...`, growing a
+# second `.files` tree instead of `cli.chat.files/agent.society/...`.
+class TestLLMAgentSocietySaveFile < Test::Unit::TestCase
+  def simple_agent(content = 'you are a robot')
+    agent = LLM::Agent.new
+    agent.start_chat.system content
+    agent
+  end
+
+  # --- direct (private method, called through send) -----------------------
+
+  def test_society_save_file_is_nil_without_save_file
+    agent = LLM::Agent.new
+    # save.rb simply does not auto-save without a save_file; the delegation
+    # must mirror that instead of raising on nil (assigning nil to
+    # agent.save_file is harmless: it means "no auto-save").
+    assert_nil agent.send(:society_save_file, 'Direct', 'harness_test')
+  end
+
+  def test_society_save_file_cli_shaped_save_file
+    agent = LLM::Agent.new
+    agent.save_file = 'dir/cli.chat.files/agent.chat'
+    assert_equal 'dir/cli.chat.files/agent.society/Direct/harness_test/agent.chat',
+                 agent.send(:society_save_file, 'Direct', 'harness_test')
+  end
+
+  def test_society_save_file_cli_shaped_named_agent
+    agent = LLM::Agent.new
+    agent.save_file = 'dir/job.chat.files/worker.chat'
+    assert_equal 'dir/job.chat.files/worker.society/Direct/harness_test/agent.chat',
+                 agent.send(:society_save_file, 'Direct', 'harness_test')
+  end
+
+  def test_society_save_file_plain_root_chat
+    agent = LLM::Agent.new
+    agent.save_file = 'dir/root.chat'
+    assert_equal 'dir/root.society/Direct/harness_test/agent.chat',
+                 agent.send(:society_save_file, 'Direct', 'harness_test')
+  end
+
+  def test_society_save_file_nested_child_save_file
+    agent = LLM::Agent.new
+    # A child chat already living inside a society tree resolves through the
+    # location rule: the society of a nested chat is a SIBLING 'society' dir.
+    agent.save_file = 'dir/root.chat.files/root.society/Direct/harness_test/agent.chat'
+    assert_equal 'dir/root.chat.files/root.society/Direct/harness_test/society/Nested/default/agent.chat',
+                 agent.send(:society_save_file, 'Nested', 'default')
+  end
+
+  # --- public behavior ----------------------------------------------------
+
+  # load_chat hands the specialist its save_file derived from the PARENT's
+  # save_file: this is the path the CLI and chat_task jobs take.
+  def test_load_chat_assigns_canonical_child_save_file
+    TmpFile.with_dir do |dir|
+      root = simple_agent('root')
+      worker = simple_agent('worker')
+      root.society = {'Worker' => worker}
+      root.socialize
+
+      root.save_file = File.join(dir, 'cli.chat.files', 'agent.chat')
+      child = root.load_chat('Worker', {}, 'harness_test')
+
+      expected = File.join(dir, 'cli.chat.files', 'agent.society',
+                           'Worker', 'harness_test', 'agent.chat')
+      assert_equal expected, child.save_file
+
+      # Writing through the child must not grow any second `.files` tree and
+      # must not leave a stray `cli.society.files` directory behind.
+      LLM::Mock.script('mock answer')
+      child.user 'child question'
+      child.chat persist: false, endpoint: 'mock'
+
+      assert_include Open.read(expected), 'mock answer'
+      assert_empty Dir.glob(File.join(dir, '**', '*.society.files*'))
+      assert_empty Dir.glob(File.join(dir, '**', 'cli.society*'))
+      # The child auto-save is the only writer here (the root itself never
+      # ran a round), so the canonical child file is the ONLY chat file.
+      assert_equal [expected], Dir.glob(File.join(dir, '**', '*.chat')).sort
+    end
+  end
+end

data/test/scout/llm/agent/test_workflow.rb

@@ -0,0 +1,110 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestWorkflowChatTaskSave < Test::Unit::TestCase
+  # NOTE on setup order: test_helper's `setup do` callback runs AFTER this
+  # `setup` method and does `Open.rm_rf tmpdir`, i.e. it deletes every
+  # directory under tmp/test_tmpdir — including any cwd we adopted here. So
+  # we must NOT Dir.chdir into a tmpdir-based directory, or every later
+  # FileUtils.pwd / Path.follow call raises Errno::ENOENT (getcwd) once the
+  # hook has removed it. The workflow does not depend on a particular cwd,
+  # so we only point the module's job directory at a fresh work dir (created
+  # with Open.mkdir so it exists regardless of Path semantics) and keep the
+  # process cwd untouched.
+  def setup
+    @tmp_work = Path.setup(File.join(TmpFile.tmpdir, 'workflow_chat_task_save'))
+    Open.mkdir @tmp_work
+    TestWorkflowChatTaskSaveWF.directory = @tmp_work
+  end
+
+  def teardown
+    FileUtils.rm_rf @tmp_work if File.exist?(@tmp_work)
+  end
+
+  def run_job(dir, task)
+    TestWorkflowChatTaskSaveWF.directory = Path.setup(dir)
+    job = TestWorkflowChatTaskSaveWF.job(task, chat: '')
+    job.produce
+    job
+  end
+
+  module TestWorkflowChatTaskSaveWF
+    extend Workflow
+    # `extend AgentWorkflow` alone does NOT copy the helper blocks into this
+    # module's helpers hash, so Workflow#job would extend the Step with an
+    # empty step_module and `log_agent` would be missing at run time.
+    # include_workflow merges the helpers (and tasks) properly.
+    include_workflow AgentWorkflow
+
+    self.name = 'TestWorkflowChatTaskSaveWF'
+
+    chat_task :plain do
+      LLM::Agent.new(start_chat: Chat.setup([{role: :system, content: 'You are a helper.'}, {role: :user, content: 'Say hi'}]))
+    end
+
+    chat_task :with_chat do
+      agent = LLM::Agent.new(start_chat: Chat.setup([{role: :system, content: 'You are a helper.'}]))
+      agent.user 'question one'
+      agent.current_chat.push({role: :assistant, content: 'answer one'})
+      agent
+    end
+  end
+
+  def test_no_files_without_society
+    job = run_job(TmpFile.tmpdir, :plain)
+    path = job.path
+    # The unified provenance save ALWAYS writes this job agent's own chat at
+    # `<path>.files/log/agent.chat` (that is the point of the mechanism);
+    # only the society subtree stays lazy and appears with a live society.
+    assert File.exist?("#{path}.files/agent.chat")
+    assert !File.exist?("#{path}.files/agent.society")
+  end
+
+  def test_canonical_layout_no_society
+    job = run_job(TmpFile.tmpdir, :with_chat)
+    path = job.path
+    # files_dir is the SUFFIXED sibling `<path>.files`, not a subdirectory
+    agent_chat = File.join("#{path}.files", 'agent.chat')
+    assert Open.exist?(agent_chat)
+    saved = LLM.chat(agent_chat)
+    assert_equal 'You are a helper.', saved.first[:content]
+    assert saved.any? { |m| m[:role] == 'assistant' && m[:content] == 'answer one' }
+    assert !File.exist?(File.join("#{path}.files", 'agent.society'))
+  end
+
+  def test_nested_society_layout
+    TmpFile.with_dir do |dir|
+      # ask_agent builds the specialist from this agent's society when the
+      # name is already there (LLM.load_agent would otherwise fail: there is
+      # no 'Worker' installed anywhere for these tests).
+      worker = LLM::Agent.new(start_chat: Chat.setup([{role: :system, content: 'child helper'}]))
+      LLM::Mock.script('child answer')
+
+      agent = LLM::Agent.new(start_chat: Chat.setup([{role: :system, content: 'parent'}]))
+      agent.society = { 'Worker' => worker }
+      agent.save_file = File.join(dir, 'agent.chat')
+      agent.user 'question'
+      child = agent.ask_agent('Worker', 'child prompt', options: {endpoint: 'mock'})
+      agent.current_chat.push({role: :assistant, content: 'final answer'})
+
+      written = agent.save
+      # ask_agent without a conversation argument lands in the 'default'
+      # conversation slot
+      nested = File.join(dir, 'agent.society', 'Worker', 'default', 'agent.chat')
+      assert_include written, File.expand_path(nested)
+      assert Open.exist?(nested)
+      assert !File.exist?(File.join(dir, 'nested.files'))
+    end
+  end
+
+  def test_chat_task_result_delta_preserved
+    job = run_job(TmpFile.tmpdir, :with_chat)
+    result = LLM.chat(job.path)
+    # The job result is a Chat projection: it legitimately starts with the
+    # provenance meta header (job=...). The delta assertions apply to the
+    # conversation messages themselves.
+    delta = result.reject { |m| m[:role].to_s == 'meta' }
+    assert_include delta.first[:content], 'question one'
+    assert_include delta.last[:content], 'answer one'
+  end
+end