scout-ai 1.0.1 → 1.1.1

data/test/scout/llm/chat/test_parse.rb

@@ -0,0 +1,126 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestParse < Test::Unit::TestCase
+  def test_parse_simple_text
+    text = "Hello\nWorld"
+    msgs = Chat.parse(text)
+    assert_equal 1, msgs.size
+    assert_equal 'user', msgs[0][:role]
+    assert_equal "Hello\nWorld", msgs[0][:content]
+  end
+
+  def test_parse_block_and_inline_headers
+    text = <<~TXT
+    assistant:
+    This is a block
+    with lines
+    user: inline reply
+    another line
+    TXT
+
+    msgs = Chat.parse(text)
+
+    # Expect a few messages: initial empty user, assistant block, inline user, and final user block
+    assert_equal 'user', msgs[0][:role]
+    assert_equal '', msgs[0][:content]
+
+    assert_equal 'assistant', msgs[1][:role]
+    assert_equal "This is a block\nwith lines", msgs[1][:content]
+
+    assert_equal 'user', msgs[2][:role]
+    assert_equal 'inline reply', msgs[2][:content]
+
+    assert_equal 'assistant', msgs[3][:role]
+    assert_equal 'another line', msgs[3][:content]
+  end
+
+  def test_parse_code_fence_protection
+    text = <<~TXT
+    assistant:
+    Here is code:
+    ```
+    def foo
+    end
+    ```
+    Done
+    TXT
+
+    msgs = Chat.parse(text)
+    assert_equal 2, msgs.size # initial empty + assistant
+
+    assistant_msg = msgs[1]
+    assert_equal 'assistant', assistant_msg[:role]
+
+    expected = "Here is code:\n```\ndef foo\nend\n```\nDone"
+    assert_equal expected, assistant_msg[:content]
+  end
+
+  def test_parse_xml_protection
+    text = <<~TXT
+    assistant:
+    Before xml
+    <note>
+    This is protected
+    </note>
+    After
+    TXT
+
+    msgs = Chat.parse(text)
+    assistant_msg = msgs.find { |m| m[:role] == 'assistant' }
+    assert assistant_msg
+    assert_equal "Before xml\n<note>\nThis is protected\n</note>\nAfter", assistant_msg[:content]
+  end
+
+  def test_parse_square_brackets_protection
+    text = <<~TXT
+    assistant:
+    Start
+    [[This: has colon
+    and lines]]
+    End
+    TXT
+
+    msgs = Chat.parse(text)
+    assistant_msg = msgs.find { |m| m[:role] == 'assistant' }
+    assert assistant_msg
+    assert_equal "Start\nThis: has colon\nand lines\nEnd", assistant_msg[:content]
+  end
+
+  def test_parse_cmd_output_protection
+    text = <<~TXT
+    assistant:
+    Before
+    shell:-- ls {{{
+    file1
+    shell:-- ls }}}
+    After
+    TXT
+
+    msgs = Chat.parse(text)
+    assistant_msg = msgs.find { |m| m[:role] == 'assistant' }
+    assert assistant_msg
+
+    expected = "Before\n<cmd_output cmd=\"ls\">\nfile1\n</cmd_output>\nAfter"
+    assert_equal expected, assistant_msg[:content]
+  end
+
+  def test_previous_response_id_behavior
+    text = <<~TXT
+    previous_response_id:abc123
+    Some block
+    assistant: Got it
+    TXT
+
+    msgs = Chat.parse(text)
+
+    # Find the previous_response_id message
+    idx = msgs.index { |m| m[:role] == 'previous_response_id' }
+    assert idx, 'previous_response_id message not found'
+    assert_equal 'abc123', msgs[idx][:content]
+
+    # The message after previous_response_id should be a user block containing "Some block"
+    assert_equal 'user', msgs[idx + 1][:role]
+    assert_equal 'Some block', msgs[idx + 1][:content]
+  end
+end

data/test/scout/llm/chat/test_process.rb

@@ -0,0 +1,123 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/llm/chat'
+class TestProcess < Test::Unit::TestCase
+  def setup
+    super
+    @tmp = tmpdir
+  end
+
+  def _test_imports_basic_and_continue_last
+    TmpFile.with_file do |file|
+      Open.write(file, "assistant: hello\nuser: from_import\n")
+
+      messages = [{role: 'import', content: file}]
+      out = Chat.imports(messages)
+
+      # Should have replaced import with the messages from the file
+      roles = out.collect{|m| m[:role]}
+      assert_includes roles, 'assistant'
+      assert_includes roles, 'user'
+
+      # Test continue: only last non-empty message
+      messages = [{role: 'continue', content: file}]
+      out = Chat.imports(messages)
+      assert_equal 1, out.size
+      assert_equal 'user', out[0][:role]
+      assert_equal 'from_import', out[0][:content].strip
+
+      # Test last: should behave similarly but using purge
+      messages = [{role: 'last', content: file}]
+      out = Chat.imports(messages)
+      assert_equal 1, out.size
+    end
+  end
+
+  def _test_files_file_reads_and_tags_content
+    TmpFile.with_file do |tmp|
+      file = File.join(tmp, 'afile.txt')
+      Open.write(file, "SOME_UNIQUE_CONTENT_12345")
+
+      messages = [{role: 'file', content: file}]
+      out = Chat.files(messages)
+
+      assert_equal 1, out.size
+      msg = out[0]
+      assert_equal 'user', msg[:role]
+      # content should include the file content and the filename
+      assert_match /SOME_UNIQUE_CONTENT_12345/, msg[:content]
+      assert_match /afile.txt/, msg[:content]
+    end
+  end
+
+  def _test_options_extracts_and_resets
+    chat = [
+      {role: 'endpoint', content: 'http://api.example'},
+      {role: 'option', content: 'k1 v1'},
+      {role: 'sticky_option', content: 'sk sv'},
+      {role: 'assistant', content: 'ok'},
+      {role: 'option', content: 'k2 v2'},
+      {role: 'user', content: 'do something'}
+    ]
+
+    opts = Chat.options(chat)
+
+    # endpoint should be sticky
+    assert_equal 'http://api.example', opts['endpoint']
+    # sticky_option should be present
+    assert_equal 'sv', opts['sk']
+    # first option k1 should have been cleared after assistant
+    assert_nil opts['k1']
+    # second option should remain
+    assert_equal 'v2', opts['k2']
+
+    # chat should have been replaced and should not include option messages
+    roles = chat.collect{|m| m[:role]}
+    assert_includes roles, 'assistant'
+    assert_includes roles, 'user'
+    assert_not_includes roles, 'option'
+    assert_not_includes roles, 'sticky_option'
+  end
+
+  def test_tasks_creates_jobs_and_calls_workflow_produce
+    # define a minimal workflow class to be resolved by Kernel.const_get
+    klass = Class.new do
+      def self.job(task_name, jobname=nil, options={})
+        # return a simple object with a path that responds to find
+        path = Struct.new(:p) do
+          def find; p; end
+        end
+        job = Struct.new(:path).new(path.new("/tmp/fake_job_#{task_name}"))
+        job
+      end
+    end
+
+    Object.const_set('TestWorkflow', klass)
+
+    produced = nil
+    # stub Workflow.produce to capture
+    orig = Workflow.method(:produce)
+    Workflow.define_singleton_method(:produce) do |jobs|
+      produced = jobs
+    end
+
+    begin
+      messages = [ {role: 'task', content: 'TestWorkflow mytask jobname=jn param=1'} ]
+      out = Chat.tasks(messages)
+
+      # Should have returned a job message pointing to our fake path
+      assert_equal 1, out.size
+      assert_equal 'job', out[0][:role]
+      assert_match /fake_job_mytask/, out[0][:content]
+
+      # produce should have been called with the job
+      assert_not_nil produced
+      assert_equal 1, produced.size
+    ensure
+      # restore original
+      Workflow.define_singleton_method(:produce, orig)
+      Object.send(:remove_const, 'TestWorkflow') rescue nil
+    end
+  end
+end

data/test/scout/llm/test_agent.rb

@@ -1,9 +1,9 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
 require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
-require 'rbbt-util'
+require 'scout/knowledge_base'
 class TestLLMAgent < Test::Unit::TestCase
-  def _test_system
+  def test_system
     TmpFile.with_dir do |dir|
       kb = KnowledgeBase.new dir
       kb.format = {"Person" => "Alias"}
@@ -13,32 +13,9 @@ class TestLLMAgent < Test::Unit::TestCase
 
       agent = LLM::Agent.new knowledge_base: kb
 
-      agent.system = ""
-
       sss 0
       ppp agent.ask "Who is Miguel's brother-in-law. Brother in law is your spouses sibling or your sibling's spouse"
-      #ppp agent.ask "Who is Guille's brother-in-law. Brother in law is your spouses sibling or your sibling's spouse"
-    end
-  end
-
-  def _test_system_gepeto
-    TmpFile.with_dir do |dir|
-      Scout::Config.set(:backend, :ollama, :ask)
-      kb = KnowledgeBase.new dir
-      kb.format = {"Person" => "Alias"}
-      kb.register :brothers, datafile_test(:person).brothers, undirected: true
-      kb.register :marriages, datafile_test(:person).marriages, undirected: true, source: "=>Alias", target: "=>Alias"
-      kb.register :parents, datafile_test(:person).parents
-
-      agent = LLM::Agent.new knowledge_base: kb, model: 'mistral', url: "https://gepeto.bsc.es/"
-
-      agent.system = ""
-
-      sss 0
-      ppp agent.ask "Who is Guille's brother-in-law. Brother in law is your spouses sibling or your sibling's spouse"
     end
   end
-
-
 end
 

data/test/scout/llm/test_chat.rb

@@ -3,143 +3,9 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 class TestMessages < Test::Unit::TestCase
 
-  def test_short
-
-    question =<<-EOF
-Hi
-    EOF
-
-    iii LLM.chat(question)
-  end
-
-  def test_inline
-    question =<<-EOF
-system:
-
-you are a terse assistant that only write in short sentences
-
-assistant:
-
-Here is some stuff
-
-user: feedback
-
-that continues here
-    EOF
-
-    iii LLM.chat(question)
-  end
-
-  def test_messages
-    question =<<-EOF
-system:
-
-you are a terse assistant that only write in short sentences
-
-user:
-
-What is the capital of France
-
-assistant:
-
-Paris
-
-user:
-
-is this the national anthem
-
-[[
-corous: Viva Espagna
-]]
-
-assistant:
-
-no
-
-user:
-
-import: math.system
-
-consider this file
-
-<file name=foo_bar>
-foo: bar
-</file>
-
-how many characters does it hold
-
-assistant:
-
-8
-    EOF
-
-    messages = LLM.messages question
-    refute messages.collect{|i| i[:role] }.include?("corous")
-    assert messages.collect{|i| i[:role] }.include?("import")
-  end
-
-  def test_chat_import
-    file1 =<<-EOF
-system: You are an assistant
-    EOF
-
-    file2 =<<-EOF
-import: header
-user: say something
-    EOF
-
-    TmpFile.with_path do |tmpdir|
-      tmpdir.header.write file1
-      tmpdir.chat.write file2
-
-      chat = LLM.chat tmpdir.chat
-    end
-  end
-
-  def test_clear
-    question =<<-EOF
-system:
-
-you are a terse assistant that only write in short sentences
-
-clear:
-
-user:
-
-What is the capital of France
-    EOF
-
-    TmpFile.with_file question do |file|
-      messages = LLM.chat file
-      refute messages.collect{|m| m[:role] }.include?('system')
-    end
-  end
-
-  def __test_job
-    question =<<-EOF
-system:
-
-you are a terse assistant that only write in short sentences
-
-job: Baking/bake_muffin_tray/Default_08a1812eca3a18dce2232509dabc9b41
-
-How are muffins made
-
-    EOF
-
-    TmpFile.with_file question do |file|
-      messages = LLM.chat file
-      ppp LLM.print messages
-    end
-  end
-
 
   def test_task
     question =<<-EOF
-system:
-
-you are a terse assistant that only write in short sentences
-
 user:
 
 task: Baking bake_muffin_tray blueberries=true title="This is a title" list=one,two,"and three"
@@ -150,26 +16,8 @@ How are muffins made?
 
     TmpFile.with_file question do |file|
       messages = LLM.chat file
-      ppp LLM.print messages
-    end
-  end
-
-  def test_structure
-    require 'scout/llm/ask'
-    sss 0
-    question =<<-EOF
-system:
-
-Respond in json format with a hash of strings as keys and string arrays as values, at most three in length
-
-endpoint: sambanova
-
-What other movies have the protagonists of the original gost busters played on, just the top.
-
-    EOF
-
-    TmpFile.with_file question do |file|
-      ppp LLM.ask file
+      assert_include messages.collect{|m| m[:role] }, 'function_call'
+      assert_include messages.find{|m| m[:role] == 'function_call_output' }[:content], 'Baking'
     end
   end
 
@@ -228,29 +76,5 @@ association: marriages #{datafile_test(:person).marriages} undirected=true sourc
       ppp LLM.ask file
     end
   end
-
-  def test_previous_response
-    require 'scout/llm/ask'
-    sss 0
-    question =<<-EOF
-user:
-
-Say hi
-
-assistant:
-
-Hi
-
-previous_response_id: asdfasdfasdfasdf
-
-Bye
-
-    EOF
-
-    messages = LLM.messages question
-
-    iii messages
-
-  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-ai
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.1
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -37,6 +37,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ollama-ai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-mcp-client
   requirement: !ruby/object:Gem::Requirement
@@ -89,9 +103,15 @@ files:
 - lib/scout/llm/backends/relay.rb
 - lib/scout/llm/backends/responses.rb
 - lib/scout/llm/chat.rb
+- lib/scout/llm/chat/annotation.rb
+- lib/scout/llm/chat/parse.rb
+- lib/scout/llm/chat/process.rb
+- lib/scout/llm/chat/process/clear.rb
+- lib/scout/llm/chat/process/files.rb
+- lib/scout/llm/chat/process/options.rb
+- lib/scout/llm/chat/process/tools.rb
 - lib/scout/llm/embed.rb
 - lib/scout/llm/mcp.rb
-- lib/scout/llm/parse.rb
 - lib/scout/llm/rag.rb
 - lib/scout/llm/tools.rb
 - lib/scout/llm/tools/call.rb
@@ -145,6 +165,8 @@ files:
 - test/scout/llm/backends/test_openwebui.rb
 - test/scout/llm/backends/test_relay.rb
 - test/scout/llm/backends/test_responses.rb
+- test/scout/llm/chat/test_parse.rb
+- test/scout/llm/chat/test_process.rb
 - test/scout/llm/test_agent.rb
 - test/scout/llm/test_ask.rb
 - test/scout/llm/test_chat.rb
@@ -186,7 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.0.dev
+rubygems_version: 3.7.2
 specification_version: 4
 summary: AI gear for scouts
 test_files: []

data/lib/scout/llm/parse.rb

@@ -1,91 +0,0 @@
-require 'scout/llm/utils'
-module LLM
-  def self.process_inside(inside)
-    header, content = inside.match(/([^\n]*)\n(.*)/).values_at 1, 2
-    if header.empty?
-      content
-    else
-      action, _sep, rest = header.partition /\s/
-      case action
-      when 'import'
-      when 'cmd'
-        title = rest.strip.empty? ? content : rest
-        tag('file', title, CMD.cmd(content).read)
-      when 'file'
-        file = content
-        title = rest.strip.empty? ? file : rest
-        tag(action, title, Open.read(file))
-      when 'directory'
-        directory = content
-        title = rest.strip.empty? ? directory : rest
-        directory_content = Dir.glob(File.join(directory, '**/*')).collect do |file|
-          file_title = Misc.path_relative_to(directory, file)
-          tag('file', file_title, Open.read(file) )
-        end * "\n"
-        tag(action, title, directory_content )
-      else 
-        tag(action, rest, content)
-      end
-    end
-  end
-
-  def self.parse(question, role = nil)
-    role = :user if role.nil?
-
-    if Array === question
-      question.collect do |q|
-        Hash === q ? q : {role: role, content: q}
-      end
-    else
-      if m = question.match(/(.*?)\[\[(.*?)\]\](.*)/m)
-        pre = m[1]
-        inside = m[2]
-        post = m[3]
-        messages = parse(pre, role)
-
-        messages = [{role: role, content: ''}] if messages.empty?
-        messages.last[:content] += process_inside inside
-
-        last = parse(post, messages.last[:role])
-
-        messages.concat last
-
-        messages
-      elsif m = question.match(/(.*?)(```.*?```)(.*)/m)
-        pre = m[1]
-        inside = m[2]
-        post = m[3]
-        messages = parse(pre, role)
-
-        messages = [{role: role, content: ''}] if messages.empty?
-        messages.last[:content] += inside
-
-        last = parse(post, messages.last[:role])
-
-        if last.first[:role] == messages.last[:role]
-          m = last.shift
-          messages.last[:content] += m[:content]
-        end
-
-        messages.concat last
-
-        messages
-      else
-        chunks = question.scan(/(.*?)^(\w+):(.*?)(?=^\w+:|\z)/m)
-
-        if chunks.any?
-          messages = []
-          messages << {role: role, content: chunks.first.first} if chunks.first and not chunks.first.first.empty?
-          chunks.collect do |pre,role,text|
-            messages << {role: role, content: text.strip}
-          end
-          messages
-        elsif question.strip.empty?
-          []
-        else
-          [{role: role, content: question}]
-        end
-      end
-    end
-  end
-end