scout-ai 1.0.1 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a13a09db3a006c0a8a2c5bc2487d5248c7b599ea9013924ab588d25540e8848d
-  data.tar.gz: 73f87320f05289b9818aef772b205b39745ee5553733664614988e3d45c66c64
+  metadata.gz: 57e9757e8effe1d42977eb5b9c5b5f3ee9d3e4603edb845682a35b4f041f3185
+  data.tar.gz: 9613eabf0c2cda6a10fa286bffe437151e8619713808fb23a030fabbf1c16fa3
 SHA512:
-  metadata.gz: 72bea9d93dc2d374580c763688d9a2b5e7a576ca5806e5169f4f88d0640a7fe29e9a7013cf55c0d89763699c630c662e918ebb9b9033948be6a70d2491eb7575
-  data.tar.gz: dfe39bc0d5f624df7ecced479f379902492088555bd6afbe140e89b94dadaa6e8fed82b8c04d6369ca8f8dac5014b8c3cb0a0d64657f6e5d37b4306435f8bbf1
+  metadata.gz: 3dd57a37318b2392655a318d6c947e8e8215bde51c46c516b3378c8e72c7475c00cd0b462a59a36ffe456fbe3099fc5b065ad8ce3929fdf186ed49fad0e9f3bf
+  data.tar.gz: 2b9e7be0db65387f18d9ce5340ec81789e852e26b67be076c10d0e6f2def94b7facf34cf64e41089bd911cb8cb25c440ed778d7f30f38ab38fa6da78dc21ac1c

data/.vimproject

@@ -6,6 +6,14 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
   scout-ai
  }
  chats=chats filter="*"{
+  parse
+
+  test_tool
+
+  ask_agent
+
+  test_ollama_tool
+
   test_github
 
   test_stdio
@@ -91,7 +99,6 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
   scout=scout{
    llm=llm{
     utils.rb
-    parse.rb
     tools.rb
     tools=tools{
      mcp.rb
@@ -100,11 +107,22 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
      call.rb
     }
     chat.rb
+    chat=chat{
+     annotation.rb
+     parse.rb
+     process.rb
+     process=process{
+      tools.rb
+      files.rb
+      clear.rb
+      options.rb
+     }
+    }
 
     backends=backends{
      openai.rb
-     anthropic.rb
      responses.rb
+     anthropic.rb
      ollama.rb
      bedrock.rb
      openwebui.rb

data/Rakefile

@@ -18,6 +18,7 @@ Juwelier::Tasks.new do |gem|
   # dependencies defined in Gemfile
   gem.add_runtime_dependency 'scout-rig', '>= 0'
   gem.add_runtime_dependency 'ruby-openai', '>= 0'
+  gem.add_runtime_dependency 'ollama-ai', '>= 0'
   gem.add_runtime_dependency 'ruby-mcp-client', '>= 0'
 end
 Juwelier::RubygemsDotOrgTasks.new

data/VERSION

@@ -1 +1 @@
-1.0.1
+1.1.1

data/bin/scout-ai

@@ -2,6 +2,52 @@
 
 $LOAD_PATH.unshift File.join(__dir__, '../lib')
 
+if _i = ARGV.index("--log")
+  require 'scout/log'
+  log = ARGV[_i+1]
+  Log.severity = log.to_i
+  ARGV.delete "--log"
+  ARGV.delete log
+end
+
+dev_dir = nil
+if _i = ARGV.index("--dev")
+  dev_dir = ARGV[_i+1]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
+if dev_dir.nil? 
+  _s = nil
+  ARGV.each_with_index do |s,i|
+    if s.match(/^--dev(?:=(.*))?/)
+      dev_dir = $1
+      _s = s
+      next
+    end
+  end
+  ARGV.delete _s if _s
+end
+
+if dev_dir.nil? && ENV["SCOUT_DEV"]
+  dev_dir = ENV["SCOUT_DEV"]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
+if dev_dir
+  ['scout-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
+  ['rbbt-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
+end
+
 require 'scout-ai'
 
 load Scout.bin.scout.find

data/lib/scout/llm/agent/chat.rb

@@ -9,6 +9,8 @@ module LLM
         (@current_chat || start_chat).annotate chat unless Chat === chat
         @current_chat = chat
       else
+        start_chat = self.start_chat
+        Chat.setup(start_chat) unless Chat === start_chat
         @current_chat = start_chat.branch
       end
     end
@@ -25,14 +27,9 @@ module LLM
       self.ask(current_chat, ...)
     end
 
-    def chat(model = nil, options = {})
-      new = self.ask(current_chat, model, options.merge(return_messages: true))
-      current_chat.concat(new)
-      new.last['content']
-    end
 
-    def chat(model = nil, options = {})
-      response = ask(current_chat, model, options.merge(return_messages: true))
+    def chat(options = {})
+      response = ask(current_chat, options.merge(return_messages: true))
       if Array === response
         current_chat.concat(response)
         current_chat.answer

data/lib/scout/llm/agent/delegate.rb

@@ -7,6 +7,13 @@ module LLM
 
       block ||= Proc.new do |name, parameters|
         message = parameters[:message]
+        new_conversation = parameters[:new_conversation]
+        Log.medium "Delegated to #{agent}: " + Log.fingerprint(message)
+        if new_conversation
+          agent.start
+        else
+          agent.purge
+        end
         agent.user message
         agent.chat
       end
@@ -15,6 +22,11 @@ module LLM
         message: {
           "type": :string,
           "description": "Message to pass to the agent"
+        },
+        new_conversation: {
+          "type": :boolean,
+          "description": "Erase conversation history and start a new conversation with this message",
+          "default": false
         }
       }
 

data/lib/scout/llm/agent.rb

@@ -49,7 +49,7 @@ You have access to the following databases associating entities:
     end
 
     # function: takes an array of messages and calls LLM.ask with them
-    def ask(messages, model = nil, options = {})
+    def ask(messages, options = {})
       messages = [messages] unless messages.is_a? Array
       model ||= @model if model
 
@@ -87,7 +87,7 @@ You have access to the following databases associating entities:
 
       workflow = Workflow.require_workflow workflow_path if workflow_path.exists?
       knowledge_base = KnowledgeBase.new knowledge_base_path if knowledge_base_path.exists?
-      chat = LLM.chat chat_path if chat_path.exists?
+      chat = Chat.setup LLM.chat(chat_path.find) if chat_path.exists?
 
       LLM::Agent.new workflow: workflow, knowledge_base: knowledge_base, start_chat: chat
     end

data/lib/scout/llm/ask.rb

@@ -34,14 +34,26 @@ module LLM
 
     endpoint, persist = IndiferentHash.process_options options, :endpoint, :persist, persist: true
 
-    endpoint ||= Scout::Config.get :endpoint, :ask, :llm, env: 'ASK_ENDPOINT,LLM_ENDPOINT'
+    endpoint ||= Scout::Config.get :endpoint, :ask, :llm, env: 'ASK_ENDPOINT,LLM_ENDPOINT,ENDPOINT,LLM,ASK'
     if endpoint && Scout.etc.AI[endpoint].exists?
       options = IndiferentHash.add_defaults options, Scout.etc.AI[endpoint].yaml
     elsif endpoint && endpoint != ""
       raise "Endpoint not found #{endpoint}"
     end
 
-    Persist.persist(endpoint, :json, prefix: "LLM ask", other: options.merge(messages: messages), persist: persist) do
+    if options[:backend].to_s == 'responses'
+      messages = Chat.clear(messages, 'previous_response_id')
+    else
+      messages = Chat.clean(messages, 'previous_response_id')
+      options.delete :previous_response_id
+    end
+
+    Log.high Log.color :green, "Asking #{endpoint || 'client'}: #{options[:previous_response_id]}\n" + LLM.print(messages)
+    tools = options[:tools]
+    Log.high "Tools: #{Log.fingerprint tools.keys}}" if tools
+    Log.debug "#{Log.fingerprint tools}}" if tools
+
+    res = Persist.persist(endpoint, :json, prefix: "LLM ask", other: options.merge(messages: messages), persist: persist) do
       backend = IndiferentHash.process_options options, :backend
       backend ||= Scout::Config.get :backend, :ask, :llm, env: 'ASK_BACKEND,LLM_BACKEND', default: :openai
 
@@ -71,6 +83,10 @@ module LLM
         raise "Unknown backend: #{backend}"
       end
     end
+
+    Log.high Log.color :blue, "Response:\n" + LLM.print(res) 
+
+    res
   end
 
   def self.workflow_ask(workflow, question, options = {})

data/lib/scout/llm/backends/huggingface.rb

@@ -1,5 +1,3 @@
-require_relative '../parse'
-require_relative '../tools'
 require_relative '../chat'
 
 module LLM

data/lib/scout/llm/backends/ollama.rb

@@ -1,7 +1,4 @@
 require 'ollama-ai'
-require_relative '../parse'
-require_relative '../tools'
-require_relative '../utils'
 require_relative '../chat'
 
 module LLM
@@ -81,10 +78,11 @@ module LLM
       tools.merge!(LLM.associations messages)
 
       if tools.any?
-        parameters[:tools] = tools.values.collect{|obj,definition| Hash === obj ? obj : definition}
+        parameters[:tools] = LLM.tool_definitions_to_ollama tools
       end
 
-      Log.low "Calling client with parameters #{Log.fingerprint parameters}\n#{LLM.print messages}"
+      Log.low "Calling ollama #{url}: #{Log.fingerprint(parameters.except(:tools))}}"
+      Log.medium "Tools: #{Log.fingerprint tools.keys}}" if tools
 
       parameters[:messages] = LLM.tools_to_ollama messages
 
@@ -93,7 +91,9 @@ module LLM
       response = self.process_response client.chat(parameters), tools, &block
 
       res = if response.last[:role] == 'function_call_output' 
-              response + self.ask(messages + response, original_options.except(:tool_choice).merge(return_messages: true, tools: tools), &block)
+              #response + self.ask(messages + response, original_options.except(:tool_choice).merge(return_messages: true, tools: tools), &block)
+              # This version seems to keep the original message from getting forgotten
+              response + self.ask(response + messages, original_options.except(:tool_choice).merge(return_messages: true, tools: tools), &block)
             else
               response
             end

data/lib/scout/llm/backends/openai.rb

@@ -58,8 +58,10 @@ module LLM
         model ||= LLM.get_url_config(:model, url, :openai_ask, :ask, :openai, env: 'OPENAI_MODEL', default: "gpt-4.1")
       end
 
-      case format.to_sym
-      when :json, :json_object
+      case format
+      when Hash
+        options[:response_format] = format
+      when 'json', 'json_object', :json, :json_object
         options[:response_format] = {type: 'json_object'}
       else
         options[:response_format] = {type: format}
@@ -84,12 +86,13 @@ module LLM
       tools.merge!(LLM.associations messages)
 
       if tools.any?
-        parameters[:tools] = tools.values.collect{|obj,definition| Hash === obj ? obj : definition}
+        parameters[:tools] = LLM.tool_definitions_to_openai tools
       end
 
       messages = self.process_input messages
 
-      Log.low "Calling openai #{url}: #{Log.fingerprint parameters}}"
+      Log.debug "Calling openai #{url}: #{Log.fingerprint(parameters.except(:tools))}}"
+      Log.high "Tools: #{Log.fingerprint tools.keys}}" if tools
 
       parameters[:messages] = LLM.tools_to_openai messages
 

data/lib/scout/llm/backends/openwebui.rb

@@ -1,9 +1,6 @@
-require 'scout'
 require 'openai'
 require 'rest-client'
-require_relative '../parse'
-require_relative '../tools'
-require_relative '../utils'
+require_relative '../chat'
 
 module LLM
   module OpenWebUI

data/lib/scout/llm/backends/relay.rb

@@ -1,7 +1,5 @@
 require 'scout'
-require 'openai'
-require_relative '../parse'
-require_relative '../tools'
+require_relative '../chat'
 
 module LLM
 	module Relay

data/lib/scout/llm/backends/responses.rb

@@ -45,13 +45,15 @@ module LLM
     #end
 
     def self.tools_to_responses(messages)
+      last_id = nil
       messages.collect do |message|
         if message[:role] == 'function_call'
           info = JSON.parse(message[:content])
           IndiferentHash.setup info
           name = info[:name] || IndiferentHash.dig(info,:function, :name)
           IndiferentHash.setup info
-          id = info[:id].sub(/^fc_/, '')
+          id = last_id = info[:id] || "fc_#{rand(1000).to_s}"
+          id = id.sub(/^fc_/, '')
           IndiferentHash.setup({
             "type" => "function_call",
             "status" => "completed",
@@ -62,7 +64,8 @@ module LLM
         elsif message[:role] == 'function_call_output'
           info = JSON.parse(message[:content])
           IndiferentHash.setup info
-          id = info[:id].sub(/^fc_/, '')
+          id = info[:id] || last_id
+          id = id.sub(/^fc_/, '')
           {                               # append result message
             "type" => "function_call_output",
             "output" => info[:content],
@@ -103,35 +106,44 @@ module LLM
     def self.process_input(messages)
       messages = self.tools_to_responses messages
 
-      messages.collect do |message|
-        IndiferentHash.setup(message)
-        if message[:role] == 'image'
+      res = []
+      messages.each do |message|
+        message = IndiferentHash.setup(message)
+
+        role = message[:role]
+
+        case role.to_s
+        when 'image'
           path = message[:content]
           path = LLM.find_file path
           if Open.remote?(path) 
-            {role: :user, content: {type: :input_image, image_url: path }}
+            res << {role: :user, content: {type: :input_image, image_url: path }}
           elsif Open.exists?(path)
             path = self.encode_image(path)
-            {role: :user, content: [{type: :input_image, image_url: path }]}
+            res << {role: :user, content: [{type: :input_image, image_url: path }]}
           else
-            raise
+            raise "Image does not exist in #{path}"
           end
-        elsif message[:role] == 'pdf'
+        when 'pdf'
           path = original_path = message[:content]
           if Open.remote?(path) 
-            {role: :user, content: {type: :input_file, file_url: path }}
+            res << {role: :user, content: {type: :input_file, file_url: path }}
           elsif Open.exists?(path)
             data = self.encode_pdf(path)
-            {role: :user, content: [{type: :input_file, file_data: data, filename: File.basename(path) }]}
+            res << {role: :user, content: [{type: :input_file, file_data: data, filename: File.basename(path) }]}
           else
-            raise
+            raise "PDF does not exist in #{path}"
           end
-        elsif message[:role] == 'websearch'
-          {role: :tool, content: {type: "web_search_preview"} }
+        when 'websearch'
+          res << {role: :tool, content: {type: "web_search_preview"} }
+        when 'previous_response_id'
+          res = []
         else
-          message
+          res << message
         end
-      end.flatten
+      end
+
+      res
     end
 
     def self.process_format(format)
@@ -187,6 +199,7 @@ module LLM
 
       messages = LLM.chat(question)
       options = options.merge LLM.options messages
+      
 
       client, url, key, model, log_errors, return_messages, format, websearch, previous_response_id, tools, = IndiferentHash.process_options options,
         :client, :url, :key, :model, :log_errors, :return_messages, :format, :websearch, :previous_response_id, :tools,
@@ -232,13 +245,16 @@ module LLM
       tools.merge!(LLM.associations messages)
 
       if tools.any?
-        parameters[:tools] = tools.values.collect{|obj,definition| Hash === obj ? obj : definition}
+        parameters[:tools] = LLM.tool_definitions_to_reponses tools
       end
 
       parameters['previous_response_id'] = previous_response_id if String === previous_response_id
-      Log.low "Calling client with parameters #{Log.fingerprint parameters}\n#{LLM.print messages}"
+
+      Log.low "Calling responses #{url}: #{Log.fingerprint(parameters.except(:tools))}}"
+      Log.medium "Tools: #{Log.fingerprint tools.keys}}" if tools
 
       messages = self.process_input messages
+
       input = []
       messages.each do |message|
         parameters[:tools] ||= []

data/lib/scout/llm/chat/annotation.rb

@@ -0,0 +1,195 @@
+require 'scout/annotation'
+module Chat
+  extend Annotation
+
+  def message(role, content)
+    self.append({role: role.to_s, content: content})
+  end
+
+  def user(content)
+    message(:user, content)
+  end
+
+  def system(content)
+    message(:system, content)
+  end
+
+  def assistant(content)
+    message(:assistant, content)
+  end
+
+  def import(file)
+    message(:import, file)
+  end
+
+  def import_last(file)
+    message(:last, file)
+  end
+
+  def file(file)
+    message(:file, file)
+  end
+
+  def directory(directory)
+    message(:directory, directory)
+  end
+
+  def continue(file)
+    message(:continue, file)
+  end
+
+  def format(format)
+    message(:format, format)
+  end
+
+  def tool(*parts)
+    content = parts * "\n"
+    message(:tool, content)
+  end
+
+  def task(workflow, task_name, inputs = {})
+    input_str = IndiferentHash.print_options inputs
+    content = [workflow, task_name, input_str]*" "
+    message(:task, content)
+  end
+
+  def inline_task(workflow, task_name, inputs = {})
+    input_str = IndiferentHash.print_options inputs
+    content = [workflow, task_name, input_str]*" "
+    message(:inline_task, content)
+  end
+
+  def job(step)
+    message(:job, step.path)
+  end
+
+  def inline_job(step)
+    message(:inline_job, step.path)
+  end
+
+
+  def association(name, path, options = {})
+    options_str = IndiferentHash.print_options options
+    content = [name, path, options_str]*" "
+    message(:association, name)
+  end
+
+  def tag(content, name=nil, tag=:file, role=:user)
+    self.message role, LLM.tag(tag, content, name)
+  end
+
+
+  def ask(...)
+    LLM.ask(LLM.chat(self), ...)
+  end
+
+  def chat(...)
+    response = ask(...)
+    if Array === response
+      current_chat.concat(response)
+      final(response)
+    else
+      current_chat.push({role: :assistant, content: response})
+      response
+    end
+  end
+
+  def json(...)
+    self.format :json
+    output = ask(...)
+    obj = JSON.parse output
+    if (Hash === obj) and obj.keys == ['content']
+      obj['content']
+    else
+      obj
+    end
+  end
+
+  def json_format(format, ...)
+    self.format format
+    output = ask(...)
+    obj = JSON.parse output
+    if (Hash === obj) and obj.keys == ['content']
+      obj['content']
+    else
+      obj
+    end
+  end
+
+  def branch
+    self.annotate self.dup
+  end
+
+  def option(name, value)
+    self.message 'option', [name, value] * " "
+  end
+
+  def endpoint(value)
+    option :endpoint, value
+  end
+
+  def model(value)
+    option :model, value
+  end
+
+  def image(file)
+    self.message :image, file
+  end
+
+  # Reporting
+
+  def print
+    LLM.print LLM.chat(self)
+  end
+
+  def final
+    LLM.purge(self).last
+  end
+
+  def purge
+    Chat.setup(LLM.purge(self))
+  end
+
+  def shed
+    self.annotate [final]
+  end
+
+  def answer
+    final[:content]
+  end
+
+  # Write and save
+
+  def save(path, force = true)
+    path = path.to_s if Symbol === path
+    if not (Open.exists?(path) || Path === path || Path.located?(path))
+      path = Scout.chats.find[path]
+    end
+    return if Open.exists?(path) && ! force
+    Open.write path, LLM.print(self)
+  end
+
+  def write(path, force = true)
+    path = path.to_s if Symbol === path
+    if not (Open.exists?(path) || Path === path || Path.located?(path))
+      path = Scout.chats.find[path]
+    end
+    return if Open.exists?(path) && ! force
+    Open.write path, self.print
+  end
+
+  def write_answer(path, force = true)
+    path = path.to_s if Symbol === path
+    if not (Open.exists?(path) || Path === path || Path.located?(path))
+      path = Scout.chats.find[path]
+    end
+    return if Open.exists?(path) && ! force
+    Open.write path, self.answer
+  end
+
+  # Image
+  def create_image(file, ...)
+    base64_image = LLM.image(LLM.chat(self), ...)
+    Open.write(file, Base64.decode(file_content), mode: 'wb')
+  end
+end