scout-ai 1.2.3 → 2.0.0

data/lib/scout/llm/agent/workflow.rb

@@ -0,0 +1,164 @@
+require 'scout/workflow'
+
+module AgentWorkflow
+  extend Workflow
+
+  helper :chat do |chat=nil|
+    @chat ||= begin
+                chat = recursive_inputs[:chat]
+                chat = Chat.parse(chat) if String === chat
+                Chat.setup(chat)
+                chat
+              end
+  end
+
+  helper :options do
+    @options ||= LLM.options self.chat
+  end
+
+  helper :agent_options do |options|
+    IndiferentHash.setup options.except(:agent, 'agent', :chat, 'chat')
+  end
+
+  helper :tooling do
+    @tooling ||= begin
+                   chat = self.chat
+                   chat.remove_role(:tool) +
+                     chat.remove_role(:kb) +
+                     chat.remove_role(:mcp) +
+                     chat.remove_role(:introduce)
+                 end
+  end
+
+  helper :tooling_intro do
+    self.tooling.select { |msg| msg[:role] == 'introduce' }
+  end
+
+  helper :agent do |name = nil, chat: nil, options: nil, tooling: nil, files: nil, **kwargs|
+    options = self.options if options.nil?
+    tooling = self.tooling if tooling.nil?
+    options = IndiferentHash.add_defaults options, kwargs
+
+    agent = LLM.load_agent name, agent_options(options)
+    agent.job = self
+    agent.start_chat.follow tooling if tooling && !tooling.empty?
+
+    agent.start_chat.system <<-EOF
+Your current working directory is #{Dir.pwd}.
+You are working through an ask job with path #{self.path} and files_dir #{self.files_dir}.
+    EOF
+
+    if dependencies.any?
+      agent.start_chat.system <<-EOF
+This workflow job has the following depencencies:
+
+#{rec_dependencies.collect(&:path) * "\n"}
+
+Their input should already be incorporated, but they might have artifacts
+      EOF
+    end
+
+    if chat && !chat.empty?
+      chat = LLM.chat(chat.dup)
+      other_jobs = chat.jobs
+      if other_jobs.any?
+        agent.start_chat.system <<-EOF
+There are other jobs found in this chat:
+
+#{other_jobs * "\n"}
+        EOF
+      end
+    end
+
+    files.each do |path|
+      target = file(path)
+      if File.exist?(target.find)
+        agent.start_chat.file target
+      elsif File.exist?(path.find)
+        agent.start_chat.file path
+      end
+    end if files
+
+    if chat && !chat.empty?
+      chat = LLM.chat(chat)
+      chat.reject! { |msg| msg[:content].to_s.start_with? 'You have been assigned' }
+      chat.reject! { |msg| msg[:content].to_s.start_with? 'This workflow job has the following' }
+      chat.reject! { |msg| msg[:content].to_s.start_with? 'There are other jobs found in this chat' }
+      agent.start_chat.follow chat
+    end
+
+    agent
+  end
+
+  helper :log_agent do |agent, agent_name=nil|
+    # Delegate to the Agent#save machinery so every path that saves an agent
+    # (chat_task jobs, CLI, plain agent.save) writes the identical canonical
+    # layout: this agent's FULL chat at <files_dir>/<agent_name>.chat and
+    # every nested conversation under
+    # <files_dir>/<agent_name>.society/<agent>/<conversation>/agent.chat.
+    #
+    # ScoutCoder: the save_file MUST be assigned BEFORE the first chat/start
+    # call (chat_task does it for us here, but any caller must too): the
+    # Agent#chat ensure-hook and the restart-snapshot hook in Agent#start
+    # both write through save_file, so assigning it late loses the
+    # intermediate turns. `agent_name` defaults to 'agent' and names the
+    # file, never a directory: worker.chat -> worker.society, no log/ level.
+    agent.save_file = files_dir["#{agent_name || 'agent'}.chat"] if agent.save_file.nil?
+    agent.save
+
+    update_info :dependencies, dependencies.collect { |dependency| dependency.path.find }
+    agent
+  end
+end
+
+module Workflow
+  def chat_task(task_name, &block)
+    input :chat, :text, 'Chat in Scout-AI chat-file format'
+    task task_name => :chat do |chat|
+      begin
+        # No eager mkdir: directories appear lazily when the save machinery
+        # (or any other writer) actually produces a file under files_dir.
+        Chat.allow_path files_dir
+        response = self.instance_exec(&block)
+
+        result = if LLM::Agent === response
+          agent = response
+          if agent.current_chat.last[:role].to_s != 'assistant'
+            agent.chat(return_messages: true)
+          else
+            agent.current_chat - agent.start_chat
+          end.tap { log_agent(agent) }
+        elsif Hash === response
+          [response]
+        else
+          response
+        end
+
+        Chat.project(self.short_path, result)
+      rescue ScoutException
+        error = { role: :assistant, content: { exception: $!, job: self.short_path }.to_json }
+        Chat.project(self.short_path, [error])
+      end
+    end
+  end
+
+  class << self
+    alias require_workflow_old require_workflow
+  end
+
+  def self.require_workflow(name, ...)
+    begin
+      require_workflow_old(name, ...)
+    rescue => e
+      begin
+        LLM.load_agent(name).workflow
+      rescue
+        raise e
+      end
+    end
+  end
+end
+
+class LLM::Agent
+  attr_accessor :job
+end

data/lib/scout/llm/agent.rb

@@ -10,7 +10,8 @@ module LLM
   end
 
   class Agent
-    attr_accessor :workflow, :knowledge_base, :start_chat, :process_exception, :other_options, :path
+    attr_accessor :workflow, :knowledge_base, :start_chat, :process_exception, :other_options, :path, :job
+
     def initialize(workflow: nil, knowledge_base: nil, start_chat: nil, **kwargs)
       @workflow = workflow
       @workflow = Workflow.require_workflow @workflow if String === @workflow
@@ -27,8 +28,8 @@ module LLM
       else
         @workflow ||= begin
                         m = Module.new
-                        m = "ScoutAgent"
                         m.extend Workflow
+                        m.name = 'ScoutAgent'
                         m.tasks = {}
                         m
                       end
@@ -67,44 +68,56 @@ module LLM
       messages = [messages] unless messages.is_a? Array
       model ||= @model if model
 
-      messages.delete_if{|info| info[:role] == 'agent' }
-      if (list = messages.select{|info| info[:role] == 'socialize'}).any?
-        socialize = list.last[:content]
-        messages.delete_if{|info| info[:role] == 'socialize' }
-        self.socialize(options.dup) if socialize && %w(true TRUE True T 1).include?(socialize.to_s)
-      end
-
-      tools = options[:tools] || {}
-      if other_tools = @other_options[:tools]
-        other_tools = JSON.parse other_tools if String === other_tools
-        tools = tools.merge other_tools
-      end
+      no_ask_override = @other_options[:no_ask_override]
 
+      messages.delete_if{|info| info[:role] == 'agent' }
       begin
 
-        if workflow || knowledge_base
-          tools.merge!(LLM.workflow_tools(workflow)) if workflow
-          tools.merge!(LLM.knowledge_base_tool_definition(knowledge_base)) if knowledge_base and knowledge_base.all_databases.any?
-        end
+        if workflow && workflow.tasks.include?(:ask) && ! no_ask_override
+          other_options.each do |key,value|
+            messages.push(IndiferentHash.setup({role: :sticky_option, content: "#{key} #{value}"})) 
+          end
 
-        if workflow && workflow.tasks.include?(:ask)
-          options.each do |key,value|
-            messages.push(IndiferentHash.setup({role: :option, content: "#{key} #{value}"})) 
+          options.except(:return_messages).each do |key,value|
+            messages.push(IndiferentHash.setup({role: :sticky_option, content: "#{key} #{value}"})) 
           end
 
           job = workflow.job(:ask, chat: Chat.print(messages))
-          job.clean
+          self.job = job
+          Open.mkdir job.files_dir
+          Chat.allow_job job
+          job.clean if ENV['SCOUT_NO_ASK_CACHE'] == 'true'
+          job.recursive_clean if ENV['SCOUT_NO_ASK_CACHE'] == 'recursive'
           job.produce
           
-          messages = LLM.chat job.path
+          messages = Chat.project(job.short_path, LLM.chat(job.path))
           if options[:return_messages]
-            messages
+            Chat.setup(messages)
           else
-            Chat.answer messages
+            Chat.setup(messages).answer
           end
         else
+
+          if (list = messages.select{|info| info[:role] == 'socialize'}).any?
+            socialize = list.last[:content]
+            messages.delete_if{|info| info[:role] == 'socialize' }
+            self.socialize(options.dup) if socialize && %w(true TRUE True T 1).include?(socialize.to_s)
+          end
+
+          tools = options[:tools] || {}
+          if other_tools = @other_options[:tools]
+            other_tools = JSON.parse other_tools if String === other_tools
+            tools = tools.merge other_tools
+          end
+
+
+          if workflow || knowledge_base
+            tools.merge!(LLM.workflow_tools(workflow)) if workflow && workflow.tasks.any?
+            tools.merge!(LLM.knowledge_base_tool_definition(knowledge_base)) if knowledge_base and knowledge_base.all_databases.any?
+          end
+
           options[:tools] = tools
-          LLM.ask messages, @other_options.merge(log_errors: true).merge(options).merge(agent: false)
+          LLM.ask messages, @other_options.except(:no_ask_override).merge(log_errors: true).merge(options).merge(agent: false)
         end
       rescue
         exception = $!
@@ -116,7 +129,13 @@ module LLM
             raise exception
           end
         else
-          raise exception
+          begin
+            self.save if self.save_file
+          rescue
+            Log.exception $!
+          ensure
+            raise exception
+          end
         end
       end
     end
@@ -140,6 +159,7 @@ module LLM
     end
 
     def self.load_agent(agent_name = nil, options = {})
+      agent_name = agent_name.to_s if Symbol === agent_name
       if agent_name && Path.is_filename?(agent_name) 
         if File.directory?(agent_name)
           dir = Path.setup(agent_name) unless Path === agent_name
@@ -151,40 +171,43 @@ module LLM
         end
       end
 
-      agent_name ||= 'default'
-
-      workflow_path = Scout.workflows[agent_name]
-      agent_path = Scout.var.Agent[agent_name]
-      agent_path = Scout.chats[agent_name] unless agent_path.exists?
-      agent_path = Scout.chats.Agent[agent_name] unless agent_path.exists?
-
-      raise ScoutException, "No agent found with name #{agent_name}" unless workflow_path.exists? || agent_path.exists?
-
-      workflow = if workflow_path.exists?
-                   agent_path = workflow_path
-                   Workflow.require_workflow agent_name
-                 elsif agent_path.workflow.find_with_extension("rb").exists?
-                   Workflow.require_workflow_file agent_path.workflow.find_with_extension("rb")
-                 elsif agent_path.python.exists? && agent_path.python.glob('*.py').any?
-                   require 'scout/workflow/python'
-                   PythonWorkflow.load_directory agent_path.python, 'ScoutAgent'
-                 end
-
-      knowledge_base = if agent_path.knowledge_base.exists?
-                         KnowledgeBase.load agent_path.knowledge_base.find
-                       elsif workflow_path.knowledge_base.exists?
-                         KnowledgeBase.load workflow_path.knowledge_base.find
-                       end
-
-      chat = if agent_path.start_chat.exists?
-               Chat.setup LLM.chat(agent_path.start_chat.find)
-             elsif workflow_path.start_chat.exists?
-               Chat.setup LLM.chat(workflow_path.start_chat.find)
-             elsif agent_path.start_chat.exists?
-               Chat.setup LLM.chat(agent_path.start_chat.find)
-             elsif workflow && workflow.documentation[:description]
-               Chat.setup([ {role: 'introduce', content: workflow.name} ])
-             end
+      if agent_name
+
+        workflow_path = Scout.workflows[agent_name]
+        agent_path = Scout.Agent[agent_name]
+        agent_path = Scout.var.Agent[agent_name] unless agent_path.exists?
+        agent_path = Scout.chats.Agent[agent_name] unless agent_path.exists?
+        agent_path = Scout.chats[agent_name] unless agent_path.exists?
+
+        raise ScoutException, "No agent found with name #{agent_name}" unless workflow_path.exists? || agent_path.exists?
+
+        @@agent_workflow ||= {}
+        workflow = @@agent_workflow[agent_name] ||= if workflow_path.exists?
+                                                      agent_path = workflow_path
+                                                      Workflow.require_workflow agent_name
+                                                    elsif agent_path.workflow.find_with_extension("rb").exists?
+                                                      Workflow.require_workflow_file agent_path.workflow.find_with_extension("rb")
+                                                    elsif agent_path.python.exists? && agent_path.python.glob('*.py').any?
+                                                      require 'scout/workflow/python'
+                                                      PythonWorkflow.load_directory agent_path.python, 'ScoutAgent'
+                                                    end
+
+        knowledge_base = if agent_path.knowledge_base.exists?
+                           KnowledgeBase.load agent_path.knowledge_base.find
+                         elsif workflow_path.knowledge_base.exists?
+                           KnowledgeBase.load workflow_path.knowledge_base.find
+                         end
+
+        chat = if agent_path.start_chat.exists?
+                 Chat.setup LLM.chat(agent_path.start_chat.find)
+               elsif workflow_path.start_chat.exists?
+                 Chat.setup LLM.chat(workflow_path.start_chat.find)
+               elsif agent_path.start_chat.exists?
+                 Chat.setup LLM.chat(agent_path.start_chat.find)
+               elsif workflow && workflow.documentation[:description]
+                 Chat.setup([ {role: 'introduce', content: workflow.name} ])
+               end
+      end
 
       agent = LLM::Agent.new **options.merge(workflow: workflow, knowledge_base: knowledge_base, start_chat: chat)
       agent.path = agent_path.find if agent_path
@@ -196,3 +219,5 @@ end
 require_relative 'agent/chat'
 require_relative 'agent/iterate'
 require_relative 'agent/delegate'
+require_relative 'agent/save'
+require_relative 'agent/workflow'

data/lib/scout/llm/ask.rb

@@ -2,45 +2,82 @@ require 'scout'
 require_relative 'chat'
 
 module LLM
+
+  BACKENDS = IndiferentHash.setup({})
+
+  def self.register_backend(name, mod)
+    BACKENDS[name] = mod
+  end
+
   def self.ask(question, options = {}, &block)
     messages = LLM.chat(question)
-    options = IndiferentHash.add_defaults LLM.options(messages), options
+    options = IndiferentHash.add_defaults options, LLM.options(messages)
 
-    endpoint, persist = IndiferentHash.process_options options, :endpoint, :persist, persist: true
+    endpoint, persist, agent_save_file = IndiferentHash.process_options options, :endpoint, :persist, :agent_save_file, persist: true
 
+    persist ||= Scout::Config.get :persist, :ask, :llm, env: 'ASK_PERSIST,LLM_PERSIST,PERSIST'
     endpoint ||= Scout::Config.get :endpoint, :ask, :llm, env: 'ASK_ENDPOINT,LLM_ENDPOINT,ENDPOINT,LLM,ASK'
-    if endpoint && Scout.etc.AI[endpoint].exists?
+    if endpoint && Scout.etc.AI[endpoint].find_with_extension(:yaml).exists?
       options = IndiferentHash.add_defaults options, Scout.etc.AI[endpoint].yaml
     elsif endpoint && endpoint != ""
       raise "Endpoint not found #{endpoint}"
     end
 
-    if options[:backend].to_s == 'responses' && options[:previous_response].to_s != 'false'
-      messages = Chat.clear(messages, 'previous_response_id')
-    else
-      messages = Chat.clean(messages, 'previous_response_id')
-      options.delete :previous_response_id
-    end
-
     agent_name = IndiferentHash.process_options options, :agent
+    agent_name = nil if %(none false nil).include?(agent_name.to_s)
     if agent_name
+      options[:endpoint] ||= endpoint
       agent = LLM::Agent.load_agent agent_name
+      agent.save_file = agent_save_file if agent_save_file
       agent.follow messages
-      res = agent.ask options
+      res = agent.chat options
       return res
     end
 
-    options[:meta] = Chat.meta messages
+    job_paths = messages.job_paths
+    meta = Chat.meta(messages)
+    options[:current_meta] = meta if meta and meta.any?
+
+    if options[:backend].to_s == 'responses' && options[:previous_response].to_s != 'false'
+      messages = Chat.clear(messages, 'previous_response_id')
+    else
+      messages = Chat.clean(messages, 'previous_response_id')
+      options.delete :previous_response_id
+    end
 
-    Log.high Log.color :green, "Asking #{endpoint || 'client'}: #{options[:previous_response_id]}\n" + Chat.print_brief(messages)
     tools = options[:tools]
-    Log.high "Tools: #{Log.fingerprint tools.keys}" if tools
-    Log.debug "#{Log.fingerprint tools}}" if tools
+    if tools
+      # ScoutCoder: options[:tools] may be either the internal Hash shape
+      # ({name => [obj, definition]}) or a plain Array of provider-style
+      # definitions (as in the tests and InfrastructureProbes); only the
+      # Hash shape has #keys, so fingerprint accordingly.
+      tool_names = Hash === tools ? tools.keys : tools.collect { |t| t[:name] || t.dig(:function, :name) }
 
+      Log.high Log.color(:green, "Asking #{endpoint || options[:endpoint] || 'client'}: #{options[:previous_response_id]}\n" + Chat.print_brief(messages))
+      Log.medium "Tools: #{Log.fingerprint tool_names}" if tool_names&.any?
+      Log.debug "#{Log.fingerprint tools}}"
+    else
+      Log.high Log.color :green, "Asking #{endpoint || options[:endpoint] || 'client'}: #{options[:previous_response_id]}\n" + Chat.print_brief(messages)
+    end
+
+    persist = false if persist.to_s.downcase == 'false'
     res = Persist.persist(endpoint, :json, prefix: "LLM ask", other: options.merge(messages: messages), persist: persist, dir: Scout.var.cache.ask) do
       backend = IndiferentHash.process_options options, :backend
       backend ||= Scout::Config.get :backend, :ask, :llm, env: 'ASK_BACKEND,LLM_BACKEND', default: :responses
 
+      job_paths.each do |job_path|
+        begin
+          job = Step.load Path.setup(job_path)
+          jobs = [job] + job.rec_dependencies.to_a
+          jobs.each do |job|
+            Chat.allow_read_job job
+          end
+        rescue
+          Log.exception $!
+          Log.warn "Could not load #{job_path}"
+        end
+      end
+
       case backend
       when :openai, "openai"
         require_relative 'backends/openai'
@@ -60,20 +97,28 @@ module LLM
       when :openwebui, "openwebui"
         require_relative 'backends/openwebui'
         LLM::OpenWebUI.ask(messages, options, &block)
+      when :huggingface, "huggingface"
+        require_relative 'backends/huggingface'
+        LLM::Huggingface.ask(messages, options, &block)
       when :relay, "relay"
         require_relative 'backends/relay'
         LLM::Relay.ask(messages, options, &block)
       when :bedrock, "bedrock"
         require_relative 'backends/bedrock'
         LLM::Bedrock.ask(messages, options, &block)
+      when :glm, "glm"
+        require_relative 'backends/glm'
+        LLM::GLM.ask(messages, options, &block)
       else
-        raise "Unknown backend: #{backend}"
+        mod = BACKENDS[backend]
+        raise "Unknown backend: #{backend}" if mod.nil?
+        mod.ask(messages, options, &block)
       end
     end
 
     Chat.setup res if Array === res
 
-    Log.high Log.color :blue, "Response:\n" + Chat.print_brief(res, %w(meta assistant)) 
+    Log.high Log.color :blue, "Response:\n" + Chat.print_brief(res, %w(meta assistant)) if Array === res
 
     res
   end

data/lib/scout/llm/backends/anthropic.rb

@@ -21,7 +21,15 @@ module LLM
       url, key = IndiferentHash.process_options options,
         :url, :key
 
-      Object::Anthropic::Client.new(access_token: key)
+      # ScoutCoder: the anthropic gem (>= 1.x, e.g. 1.17.0) Client#initialize
+      # accepts api_key: / auth_token: / base_url:, NOT access_token: (that
+      # is ruby-openai's keyword). Passing access_token: raises
+      # ArgumentError: unknown keyword. Verify kwargs against the installed
+      # gem's lib/anthropic/client.rb when touching this.
+      Object::Anthropic::Client.new(
+        api_key: key,
+        base_url: url
+      )
     end
 
     def query(client, messages, tools = [], parameters = {})
@@ -68,7 +76,6 @@ module LLM
     end
 
     def process_response(messages, response, tools, options, &block)
-      Log.debug "Respose: #{Log.fingerprint response}"
       IndiferentHash.setup response
 
       response[:content].collect do |output|

data/lib/scout/llm/backends/bedrock.rb

@@ -13,6 +13,15 @@ module LLM
       Aws::BedrockRuntime::Client.new(options)
     end
 
+    # ScoutCoder: Bedrock/Anthropic style payloads wrap the individual calls in
+    # a content entry ({"type" => "tool_use", "tool_calls" => [...]}). The ask
+    # loop used to pass that wrapper itself to LLM.tool_response, which then
+    # found neither 'function' nor 'name'/'arguments' and called the tool block
+    # with (nil, nil). Unwrap and return the inner tool call entries.
+    def self.tool_calls(message)
+      [message.dig('content')].compact.flatten.select{|m| m['tool_calls']}.collect{|m| m['tool_calls']}.flatten
+    end
+
     def self.messages_to_prompt(messages)
       system = []
       user = []
@@ -44,7 +53,10 @@ module LLM
       type ||= Scout::Config.get(:type, model, default: :messages)
 
       role, previous_response_id, tools = IndiferentHash.process_options options, :role, :previous_response_id, :tools
-      messages = LLM.parse(question, role)
+      # ScoutCoder: LLM.parse does not exist (the parser lives in Chat and is
+      # exposed as LLM.messages, which delegates to Chat.parse); this call
+      # raised NoMethodError on every ask. Use the LLM.messages delegation.
+      messages = LLM.messages(question, role)
 
       case type.to_sym 
       when :messages
@@ -72,7 +84,7 @@ module LLM
       result = JSON.parse(response.body.string)
       Log.debug "Response: #{Log.fingerprint result}"
       message = result
-      tool_calls = message.dig('content').select{|m| m['tool_calls']}
+      tool_calls = self.tool_calls message
 
       while tool_calls && tool_calls.any?
         messages << message
@@ -94,7 +106,7 @@ module LLM
         Log.debug "Response: #{Log.fingerprint result}"
 
         message = result
-        tool_calls = message.dig('content').select{|m| m['tool_calls']}
+        tool_calls = self.tool_calls message
       end
 
       message.dig('content').collect{|m|