scout-ai 1.1.0 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc083461a140b5149d6965f5aa635e651015786cfe312f220a33967fa10c52cb
-  data.tar.gz: 694ac0ff0b626d99ee993c77a1dba12022f9e7a545bbab67b94627a4698ade09
+  metadata.gz: aaebf68f2d4bcbd388e292dbdd536558e0b33daf744845bc20fd7bf92e3693f4
+  data.tar.gz: 46d9aafabb314aaca7056e2982d911a69c165da1dfc668fd0b3755e43bd2b3a5
 SHA512:
-  metadata.gz: ef373e303af56478ecde91faf87093226936e3537b9cb3346192e230af43cc8350091e589080b7c4e32993bf0cbcc43ef380582a4d38b82ca5da33bc60754dfd
-  data.tar.gz: cccb6e93d1b827a02f5b9159a11eb7839c11c48b0cdd5a0096644b19479e0609d9d641efff286a545625fc17863e2f106faa2ea775e7cde121f313260fdef779
+  metadata.gz: 3ce6e03f60080d5f103eddd7e14df23fe90ee8f446563084fbfc30eebfdabbcb193cc061b22058bf9560eb335586f678cdb81954726b83523030e9cd25d0e335
+  data.tar.gz: a6d7c13bf096be0b1947d64461acfaaa38ad94bafa33112b28bd167b832fcb16fa5b913fbd16699be058d67e7ebb38efee1322237ec8c90010fef73222b3f882

data/.vimproject

@@ -6,6 +6,7 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
   scout-ai
  }
  chats=chats filter="*"{
+  parse
 
   test_tool
 
@@ -98,7 +99,6 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
   scout=scout{
    llm=llm{
     utils.rb
-    parse.rb
     tools.rb
     tools=tools{
      mcp.rb
@@ -107,6 +107,17 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
      call.rb
     }
     chat.rb
+    chat=chat{
+     annotation.rb
+     parse.rb
+     process.rb
+     process=process{
+      tools.rb
+      files.rb
+      clear.rb
+      options.rb
+     }
+    }
 
     backends=backends{
      openai.rb

data/Rakefile

@@ -18,7 +18,9 @@ Juwelier::Tasks.new do |gem|
   # dependencies defined in Gemfile
   gem.add_runtime_dependency 'scout-rig', '>= 0'
   gem.add_runtime_dependency 'ruby-openai', '>= 0'
+  gem.add_runtime_dependency 'ollama-ai', '>= 0'
   gem.add_runtime_dependency 'ruby-mcp-client', '>= 0'
+  gem.add_runtime_dependency 'hnswlib', '>= 0'
 end
 Juwelier::RubygemsDotOrgTasks.new
 require 'rake/testtask'

data/VERSION

@@ -1 +1 @@
-1.1.0
+1.1.2

data/bin/scout-ai

@@ -2,6 +2,52 @@
 
 $LOAD_PATH.unshift File.join(__dir__, '../lib')
 
+if _i = ARGV.index("--log")
+  require 'scout/log'
+  log = ARGV[_i+1]
+  Log.severity = log.to_i
+  ARGV.delete "--log"
+  ARGV.delete log
+end
+
+dev_dir = nil
+if _i = ARGV.index("--dev")
+  dev_dir = ARGV[_i+1]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
+if dev_dir.nil? 
+  _s = nil
+  ARGV.each_with_index do |s,i|
+    if s.match(/^--dev(?:=(.*))?/)
+      dev_dir = $1
+      _s = s
+      next
+    end
+  end
+  ARGV.delete _s if _s
+end
+
+if dev_dir.nil? && ENV["SCOUT_DEV"]
+  dev_dir = ENV["SCOUT_DEV"]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
+if dev_dir
+  ['scout-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
+  ['rbbt-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
+end
+
 require 'scout-ai'
 
 load Scout.bin.scout.find

data/lib/scout/llm/agent/chat.rb

@@ -28,8 +28,8 @@ module LLM
     end
 
 
-    def chat(model = nil, options = {})
-      response = ask(current_chat, model, options.merge(return_messages: true))
+    def chat(options = {})
+      response = ask(current_chat, options.merge(return_messages: true))
       if Array === response
         current_chat.concat(response)
         current_chat.answer

data/lib/scout/llm/ask.rb

@@ -34,16 +34,24 @@ module LLM
 
     endpoint, persist = IndiferentHash.process_options options, :endpoint, :persist, persist: true
 
-    endpoint ||= Scout::Config.get :endpoint, :ask, :llm, env: 'ASK_ENDPOINT,LLM_ENDPOINT'
+    endpoint ||= Scout::Config.get :endpoint, :ask, :llm, env: 'ASK_ENDPOINT,LLM_ENDPOINT,ENDPOINT,LLM,ASK'
     if endpoint && Scout.etc.AI[endpoint].exists?
       options = IndiferentHash.add_defaults options, Scout.etc.AI[endpoint].yaml
     elsif endpoint && endpoint != ""
       raise "Endpoint not found #{endpoint}"
     end
 
-    Log.high Log.color :green, "Asking #{endpoint || 'client'}:\n" + LLM.print(messages) 
+    if options[:backend].to_s == 'responses'
+      messages = Chat.clear(messages, 'previous_response_id')
+    else
+      messages = Chat.clean(messages, 'previous_response_id')
+      options.delete :previous_response_id
+    end
+
+    Log.high Log.color :green, "Asking #{endpoint || 'client'}: #{options[:previous_response_id]}\n" + LLM.print(messages)
     tools = options[:tools]
     Log.high "Tools: #{Log.fingerprint tools.keys}}" if tools
+    Log.debug "#{Log.fingerprint tools}}" if tools
 
     res = Persist.persist(endpoint, :json, prefix: "LLM ask", other: options.merge(messages: messages), persist: persist) do
       backend = IndiferentHash.process_options options, :backend

data/lib/scout/llm/backends/huggingface.rb

@@ -1,5 +1,3 @@
-require_relative '../parse'
-require_relative '../tools'
 require_relative '../chat'
 
 module LLM

data/lib/scout/llm/backends/ollama.rb

@@ -1,7 +1,4 @@
 require 'ollama-ai'
-require_relative '../parse'
-require_relative '../tools'
-require_relative '../utils'
 require_relative '../chat'
 
 module LLM

data/lib/scout/llm/backends/openai.rb

@@ -58,8 +58,10 @@ module LLM
         model ||= LLM.get_url_config(:model, url, :openai_ask, :ask, :openai, env: 'OPENAI_MODEL', default: "gpt-4.1")
       end
 
-      case format.to_sym
-      when :json, :json_object
+      case format
+      when Hash
+        options[:response_format] = format
+      when 'json', 'json_object', :json, :json_object
         options[:response_format] = {type: 'json_object'}
       else
         options[:response_format] = {type: format}

data/lib/scout/llm/backends/openwebui.rb

@@ -1,9 +1,6 @@
-require 'scout'
 require 'openai'
 require 'rest-client'
-require_relative '../parse'
-require_relative '../tools'
-require_relative '../utils'
+require_relative '../chat'
 
 module LLM
   module OpenWebUI

data/lib/scout/llm/backends/relay.rb

@@ -1,7 +1,5 @@
 require 'scout'
-require 'openai'
-require_relative '../parse'
-require_relative '../tools'
+require_relative '../chat'
 
 module LLM
 	module Relay

data/lib/scout/llm/backends/responses.rb

@@ -106,35 +106,44 @@ module LLM
     def self.process_input(messages)
       messages = self.tools_to_responses messages
 
-      messages.collect do |message|
-        IndiferentHash.setup(message)
-        if message[:role] == 'image'
+      res = []
+      messages.each do |message|
+        message = IndiferentHash.setup(message)
+
+        role = message[:role]
+
+        case role.to_s
+        when 'image'
           path = message[:content]
           path = LLM.find_file path
           if Open.remote?(path) 
-            {role: :user, content: {type: :input_image, image_url: path }}
+            res << {role: :user, content: {type: :input_image, image_url: path }}
           elsif Open.exists?(path)
             path = self.encode_image(path)
-            {role: :user, content: [{type: :input_image, image_url: path }]}
+            res << {role: :user, content: [{type: :input_image, image_url: path }]}
           else
-            raise
+            raise "Image does not exist in #{path}"
           end
-        elsif message[:role] == 'pdf'
+        when 'pdf'
           path = original_path = message[:content]
           if Open.remote?(path) 
-            {role: :user, content: {type: :input_file, file_url: path }}
+            res << {role: :user, content: {type: :input_file, file_url: path }}
           elsif Open.exists?(path)
             data = self.encode_pdf(path)
-            {role: :user, content: [{type: :input_file, file_data: data, filename: File.basename(path) }]}
+            res << {role: :user, content: [{type: :input_file, file_data: data, filename: File.basename(path) }]}
           else
-            raise
+            raise "PDF does not exist in #{path}"
           end
-        elsif message[:role] == 'websearch'
-          {role: :tool, content: {type: "web_search_preview"} }
+        when 'websearch'
+          res << {role: :tool, content: {type: "web_search_preview"} }
+        when 'previous_response_id'
+          res = []
         else
-          message
+          res << message
         end
-      end.flatten
+      end
+
+      res
     end
 
     def self.process_format(format)
@@ -190,6 +199,7 @@ module LLM
 
       messages = LLM.chat(question)
       options = options.merge LLM.options messages
+      
 
       client, url, key, model, log_errors, return_messages, format, websearch, previous_response_id, tools, = IndiferentHash.process_options options,
         :client, :url, :key, :model, :log_errors, :return_messages, :format, :websearch, :previous_response_id, :tools,
@@ -244,6 +254,7 @@ module LLM
       Log.medium "Tools: #{Log.fingerprint tools.keys}}" if tools
 
       messages = self.process_input messages
+
       input = []
       messages.each do |message|
         parameters[:tools] ||= []

data/lib/scout/llm/chat/annotation.rb

@@ -0,0 +1,195 @@
+require 'scout/annotation'
+module Chat
+  extend Annotation
+
+  def message(role, content)
+    self.append({role: role.to_s, content: content})
+  end
+
+  def user(content)
+    message(:user, content)
+  end
+
+  def system(content)
+    message(:system, content)
+  end
+
+  def assistant(content)
+    message(:assistant, content)
+  end
+
+  def import(file)
+    message(:import, file)
+  end
+
+  def import_last(file)
+    message(:last, file)
+  end
+
+  def file(file)
+    message(:file, file)
+  end
+
+  def directory(directory)
+    message(:directory, directory)
+  end
+
+  def continue(file)
+    message(:continue, file)
+  end
+
+  def format(format)
+    message(:format, format)
+  end
+
+  def tool(*parts)
+    content = parts * "\n"
+    message(:tool, content)
+  end
+
+  def task(workflow, task_name, inputs = {})
+    input_str = IndiferentHash.print_options inputs
+    content = [workflow, task_name, input_str]*" "
+    message(:task, content)
+  end
+
+  def inline_task(workflow, task_name, inputs = {})
+    input_str = IndiferentHash.print_options inputs
+    content = [workflow, task_name, input_str]*" "
+    message(:inline_task, content)
+  end
+
+  def job(step)
+    message(:job, step.path)
+  end
+
+  def inline_job(step)
+    message(:inline_job, step.path)
+  end
+
+
+  def association(name, path, options = {})
+    options_str = IndiferentHash.print_options options
+    content = [name, path, options_str]*" "
+    message(:association, name)
+  end
+
+  def tag(content, name=nil, tag=:file, role=:user)
+    self.message role, LLM.tag(tag, content, name)
+  end
+
+
+  def ask(...)
+    LLM.ask(LLM.chat(self), ...)
+  end
+
+  def chat(...)
+    response = ask(...)
+    if Array === response
+      current_chat.concat(response)
+      final(response)
+    else
+      current_chat.push({role: :assistant, content: response})
+      response
+    end
+  end
+
+  def json(...)
+    self.format :json
+    output = ask(...)
+    obj = JSON.parse output
+    if (Hash === obj) and obj.keys == ['content']
+      obj['content']
+    else
+      obj
+    end
+  end
+
+  def json_format(format, ...)
+    self.format format
+    output = ask(...)
+    obj = JSON.parse output
+    if (Hash === obj) and obj.keys == ['content']
+      obj['content']
+    else
+      obj
+    end
+  end
+
+  def branch
+    self.annotate self.dup
+  end
+
+  def option(name, value)
+    self.message 'option', [name, value] * " "
+  end
+
+  def endpoint(value)
+    option :endpoint, value
+  end
+
+  def model(value)
+    option :model, value
+  end
+
+  def image(file)
+    self.message :image, file
+  end
+
+  # Reporting
+
+  def print
+    LLM.print LLM.chat(self)
+  end
+
+  def final
+    LLM.purge(self).last
+  end
+
+  def purge
+    Chat.setup(LLM.purge(self))
+  end
+
+  def shed
+    self.annotate [final]
+  end
+
+  def answer
+    final[:content]
+  end
+
+  # Write and save
+
+  def save(path, force = true)
+    path = path.to_s if Symbol === path
+    if not (Open.exists?(path) || Path === path || Path.located?(path))
+      path = Scout.chats.find[path]
+    end
+    return if Open.exists?(path) && ! force
+    Open.write path, LLM.print(self)
+  end
+
+  def write(path, force = true)
+    path = path.to_s if Symbol === path
+    if not (Open.exists?(path) || Path === path || Path.located?(path))
+      path = Scout.chats.find[path]
+    end
+    return if Open.exists?(path) && ! force
+    Open.write path, self.print
+  end
+
+  def write_answer(path, force = true)
+    path = path.to_s if Symbol === path
+    if not (Open.exists?(path) || Path === path || Path.located?(path))
+      path = Scout.chats.find[path]
+    end
+    return if Open.exists?(path) && ! force
+    Open.write path, self.answer
+  end
+
+  # Image
+  def create_image(file, ...)
+    base64_image = LLM.image(LLM.chat(self), ...)
+    Open.write(file, Base64.decode(file_content), mode: 'wb')
+  end
+end

data/lib/scout/llm/chat/parse.rb

@@ -0,0 +1,139 @@
+module Chat
+  def self.parse(text, role = nil)
+    default_role = "user"
+
+    messages = []
+    current_role = role || default_role
+    current_content = ""
+    in_protected_block = false
+    protected_block_type = nil
+    protected_stack = []
+
+    role = default_role if role.nil?
+
+    file_lines = text.split("\n")
+
+    file_lines.each do |line|
+      stripped = line.strip
+
+      # Detect protected blocks
+      if stripped.start_with?("```")
+        if in_protected_block
+          in_protected_block = false
+          protected_block_type = nil
+          current_content << "\n" << line unless line.strip.empty?
+        else
+          in_protected_block = true
+          protected_block_type = :square
+          current_content << "\n" << line unless line.strip.empty?
+        end
+        next
+      elsif stripped.end_with?("]]") && in_protected_block && protected_block_type == :square
+        in_protected_block = false
+        protected_block_type = nil
+        line = line.sub("]]", "")
+        current_content << "\n" << line unless line.strip.empty?
+        next
+      elsif stripped.start_with?("[[")
+        in_protected_block = true
+        protected_block_type = :square
+        line = line.sub("[[", "")
+        current_content << "\n" << line unless line.strip.empty?
+        next
+      elsif stripped.end_with?("]]") && in_protected_block && protected_block_type == :square
+        in_protected_block = false
+        protected_block_type = nil
+        line = line.sub("]]", "")
+        current_content << "\n" << line unless line.strip.empty?
+        next
+      elsif stripped.match(/^.*:-- .* {{{/)
+        in_protected_block = true
+        protected_block_type = :square
+        line = line.sub(/^.*:-- (.*) {{{.*/, '<cmd_output cmd="\1">')
+        current_content << "\n" << line unless line.strip.empty?
+        next
+      elsif stripped.match(/^.*:--.* }}}/) && in_protected_block && protected_block_type == :square
+        in_protected_block = false
+        protected_block_type = nil
+        line = line.sub(/^.*:-- .* }}}.*/, "</cmd_output>")
+        current_content << "\n" << line unless line.strip.empty?
+        next
+      elsif in_protected_block
+
+        if protected_block_type == :xml
+          if stripped =~ %r{</(\w+)>}
+            closing_tag = $1
+            if protected_stack.last == closing_tag
+              protected_stack.pop
+            end
+            if protected_stack.empty?
+              in_protected_block = false
+              protected_block_type = nil
+            end
+          end
+        end
+        current_content << "\n" << line
+        next
+      end
+
+      # XML-style tag handling (protected content)
+      if stripped =~ /^<(\w+)(\s+[^>]*)?>/
+        tag = $1
+        protected_stack.push(tag)
+        in_protected_block = true
+        protected_block_type = :xml
+      end
+
+      # Match a new message header
+      if line =~ /^([a-z0-9_]+):(.*)$/
+        role = $1
+        inline_content = $2.strip
+
+        current_content = current_content.strip if current_content
+        # Save current message if any
+        messages << { role: current_role, content: current_content }
+
+        if inline_content.empty?
+          # Block message
+          current_role = role
+          current_content = ""
+        else
+          # Inline message + next block is default role
+          messages << { role: role, content: inline_content }
+          current_role = 'user' if role == 'previous_response_id'
+          current_content = ""
+        end
+      else
+        if current_content.nil?
+          current_content = line
+        else
+          current_content += "\n" + line
+        end
+      end
+    end
+
+    # Final message
+    messages << { role: current_role || default_role, content: current_content.strip }
+
+    messages
+  end
+
+  def self.print(chat)
+    return chat if String  === chat
+    "\n" + chat.collect do |message|
+      IndiferentHash.setup message
+      case message[:content]
+      when Hash, Array
+        message[:role].to_s + ":\n\n" + message[:content].to_json
+      when nil, ''
+        message[:role].to_s + ":"
+      else
+        if %w(option previous_response_id function_call function_call_output).include? message[:role].to_s
+          message[:role].to_s + ": " + message[:content].to_s
+        else
+          message[:role].to_s + ":\n\n" + message[:content].to_s
+        end
+      end
+    end * "\n\n"
+  end
+end

data/lib/scout/llm/chat/process/clear.rb

@@ -0,0 +1,29 @@
+module Chat
+  def self.clear(messages, role = 'clear')
+    new = []
+
+    messages.reverse.each do |message|
+      if message[:role].to_s == role
+        break
+      else
+        new << message
+      end
+    end
+
+    new.reverse
+  end
+
+  def self.clean(messages, role = 'skip')
+    messages.reject do |message|
+      ((String === message[:content]) && message[:content].empty?) ||
+        message[:role] == role
+    end
+  end
+
+  def self.purge(chat)
+    chat.reject do |msg|
+      IndiferentHash.setup msg
+      msg[:role].to_s == 'previous_response_id'
+    end
+  end
+end