scout-ai 1.0.0 → 1.1.0

data/lib/scout-ai.rb

@@ -5,5 +5,6 @@ require 'scout/resource'
 Path.add_path :scout_ai_lib, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
 
 require 'scout/llm/ask'
+require 'scout/llm/chat'
 require 'scout/llm/embed'
 require 'scout/llm/agent'

data/scout-ai.gemspec

@@ -2,37 +2,45 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-ai 1.0.0 ruby lib
+# stub: scout-ai 1.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-ai".freeze
-  s.version = "1.0.0".freeze
+  s.version = "1.1.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-06-05"
+  s.date = "1980-01-02"
   s.description = "assorted functionalities to help scouts use AI".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout-ai".freeze]
   s.extra_rdoc_files = [
     "LICENSE",
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
     ".vimproject",
     "LICENSE",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "bin/scout-ai",
+    "doc/Agent.md",
+    "doc/Chat.md",
+    "doc/LLM.md",
+    "doc/Model.md",
+    "doc/RAG.md",
     "lib/scout-ai.rb",
     "lib/scout/llm/agent.rb",
     "lib/scout/llm/agent/chat.rb",
+    "lib/scout/llm/agent/delegate.rb",
+    "lib/scout/llm/agent/iterate.rb",
     "lib/scout/llm/ask.rb",
+    "lib/scout/llm/backends/anthropic.rb",
     "lib/scout/llm/backends/bedrock.rb",
     "lib/scout/llm/backends/huggingface.rb",
     "lib/scout/llm/backends/ollama.rb",
@@ -42,9 +50,14 @@ Gem::Specification.new do |s|
     "lib/scout/llm/backends/responses.rb",
     "lib/scout/llm/chat.rb",
     "lib/scout/llm/embed.rb",
+    "lib/scout/llm/mcp.rb",
     "lib/scout/llm/parse.rb",
     "lib/scout/llm/rag.rb",
     "lib/scout/llm/tools.rb",
+    "lib/scout/llm/tools/call.rb",
+    "lib/scout/llm/tools/knowledge_base.rb",
+    "lib/scout/llm/tools/mcp.rb",
+    "lib/scout/llm/tools/workflow.rb",
     "lib/scout/llm/utils.rb",
     "lib/scout/model/base.rb",
     "lib/scout/model/python/base.rb",
@@ -61,37 +74,30 @@ Gem::Specification.new do |s|
     "lib/scout/model/util/run.rb",
     "lib/scout/model/util/save.rb",
     "python/scout_ai/__init__.py",
-    "python/scout_ai/__pycache__/__init__.cpython-310.pyc",
-    "python/scout_ai/__pycache__/__init__.cpython-311.pyc",
-    "python/scout_ai/__pycache__/huggingface.cpython-310.pyc",
-    "python/scout_ai/__pycache__/huggingface.cpython-311.pyc",
-    "python/scout_ai/__pycache__/util.cpython-310.pyc",
-    "python/scout_ai/__pycache__/util.cpython-311.pyc",
-    "python/scout_ai/atcold/__init__.py",
-    "python/scout_ai/atcold/plot_lib.py",
-    "python/scout_ai/atcold/spiral.py",
     "python/scout_ai/huggingface/data.py",
     "python/scout_ai/huggingface/eval.py",
     "python/scout_ai/huggingface/model.py",
     "python/scout_ai/huggingface/rlhf.py",
     "python/scout_ai/huggingface/train/__init__.py",
-    "python/scout_ai/huggingface/train/__pycache__/__init__.cpython-310.pyc",
-    "python/scout_ai/huggingface/train/__pycache__/next_token.cpython-310.pyc",
     "python/scout_ai/huggingface/train/next_token.py",
-    "python/scout_ai/language_model.py",
     "python/scout_ai/util.py",
     "scout-ai.gemspec",
     "scout_commands/agent/ask",
     "scout_commands/agent/kb",
+    "scout_commands/documenter",
     "scout_commands/llm/ask",
     "scout_commands/llm/process",
+    "scout_commands/llm/server",
     "scout_commands/llm/template",
+    "share/server/chat.html",
+    "share/server/chat.js",
     "test/data/cat.jpg",
     "test/data/person/brothers",
     "test/data/person/identifiers",
     "test/data/person/marriages",
     "test/data/person/parents",
     "test/scout/llm/agent/test_chat.rb",
+    "test/scout/llm/backends/test_anthropic.rb",
     "test/scout/llm/backends/test_bedrock.rb",
     "test/scout/llm/backends/test_huggingface.rb",
     "test/scout/llm/backends/test_ollama.rb",
@@ -103,10 +109,15 @@ Gem::Specification.new do |s|
     "test/scout/llm/test_ask.rb",
     "test/scout/llm/test_chat.rb",
     "test/scout/llm/test_embed.rb",
+    "test/scout/llm/test_mcp.rb",
     "test/scout/llm/test_parse.rb",
     "test/scout/llm/test_rag.rb",
     "test/scout/llm/test_tools.rb",
     "test/scout/llm/test_utils.rb",
+    "test/scout/llm/tools/test_call.rb",
+    "test/scout/llm/tools/test_knowledge_base.rb",
+    "test/scout/llm/tools/test_mcp.rb",
+    "test/scout/llm/tools/test_workflow.rb",
     "test/scout/model/python/huggingface/causal/test_next_token.rb",
     "test/scout/model/python/huggingface/test_causal.rb",
     "test/scout/model/python/huggingface/test_classification.rb",
@@ -120,11 +131,13 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-ai".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.6".freeze
+  s.rubygems_version = "3.7.0.dev".freeze
   s.summary = "AI gear for scouts".freeze
 
   s.specification_version = 4
 
   s.add_runtime_dependency(%q<scout-rig>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<ruby-openai>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<ruby-mcp-client>.freeze, [">= 0".freeze])
 end
 

data/scout_commands/agent/ask

@@ -22,8 +22,9 @@ Use STDIN to add context to the question
 -c--chat* Follow a conversation
 -m--model* Model to use
 -e--endpoint* Endpoint to use
--f--file* Incorporate file at the start
+-f--file* Incorporate file
 -wt--workflow_tasks* Export these tasks to the agent
+-i--imports* Chat files to import, separated by comma
 EOF
 if options[:help]
   if defined? scout_usage
@@ -36,86 +37,42 @@ end
 
 Log.severity = options.delete(:log).to_i if options.include? :log
 
-file = options.delete(:file)
-
-agent, *question_parts = ARGV
-
-agent_path = Scout.var.Agent[agent].find_with_extension(:rb)
-
-agent = LLM::Agent.load agent_path
-
-#workflow_tasks = options.delete(:workflow_tasks)
-#
-#workflow = begin
-#             if agent_dir.workflow.set_extension('rb').exists?
-#               Workflow.require_workflow agent_dir.workflow.set_extension('rb').find
-#             else
-#               Misc.with_env "SCOUT_WORKFLOW_AUTOINSTALL", false do
-#                 Workflow.require_workflow agent
-#               end
-#             end
-#           rescue
-#           end
-#
-#if workflow_tasks and workflow
-#  workflow.clear_exports
-#  workflow.export_asynchronous *workflow_tasks.split(',')
-#end
-#
-#knowledge_base = KnowledgeBase.load(agent_dir.knowledge_base) if agent_dir.knowledge_base.exists?
-#knowledge_base ||= begin workflow.knowledge_base rescue nil end || KnowledgeBase.new(agent_dir.knowledge_base)
-#
-#agent = LLM::Agent.new **options.merge(workflow: workflow, knowledge_base: knowledge_base)
-
-#question = question_parts * " " 
-#
-#if template = options.delete(:template)
-#  if Open.exists?(template)
-#    template_question = Open.read(template)
-#  else
-#    template_question = Scout.questions[template].read
-#  end
-#  if template_question.include?('???')
-#    question = template_question.sub('???', question)
-#  else
-#    question = template_question
-#  end
-#end
-#
-#if question.include?('...')
-#  context = file ? Open.read(file) : STDIN.read 
-#  question = question.sub('...', context)
-#end
-#
-#if chat
-#  conversation = Open.exist?(chat)? Open.read(chat) : "" 
-#  question = question.empty? ? conversation : conversation + "\nuser:\n" +  question
-#  new = agent.ask(question, options)
-#  conversation = question + "\nassistant:\n" + new
-#  Open.write(chat, conversation)
-#else
-#  puts agent.ask(question, options)
-#end
-
-file, chat, inline, template, dry_run = IndiferentHash.process_options options, :file, :chat, :inline, :template, :dry_run
-
-agent, question = ARGV * " "
+agent_name, *question_parts = ARGV
+
+question = question_parts * " "
+
+file, chat, inline, template, dry_run, imports, endpoint, model = IndiferentHash.process_options options, :file, :chat, :inline, :template, :dry_run, :imports, :endpoint, :model
+
+file = Path.setup(file) if file
+
+imports = imports.split(/,\s*/) if imports
 
 agent_name ||= 'default'
-agent_file = Scout.chats[agent_name]
 
-agent_file = agent_file.find_with_extension('rb')
+agent_file = Scout.workflows[agent_name]
+
+agent_file = Scout.chats[agent_name] unless agent_file.exists?
+
+agent_file = agent_file.find_with_extension('rb') unless agent_file.exists?
+
 
 if agent_file.exists?
   if agent_file.directory?
-    agent = load agent_file.agent.find_with_extension('rb')
+    if agent_file.agent.find_with_extension('rb').exists?
+      agent = load agent_file.agent.find_with_extension('rb')
+    else
+      agent = LLM::Agent.load_from_path agent_file
+    end
   else
     agent = load agent_file
   end
 else
-  raise ParameterException agent_file
+  #raise ParameterException agent_file
 end
 
+agent.other_options[:endpoint] = endpoint if endpoint
+agent.other_options[:model] = model if model
+
 if template
   if Open.exists?(template)
     template_question = Open.read(template)
@@ -139,28 +96,47 @@ if question.include?('...')
   context = file ? Open.read(file) : STDIN.read
   question = question.sub('...', context)
 elsif file
-  question = "<file basename=#{File.basename file}>[[[\n" + Open.read(file) + "\n]]]</file>"
+  question = "<file basename=#{File.basename file}>\n" + Open.read(file) + "\n</file>\n\n" + question
 end
 
 if chat
-  conversation = Open.exist?(chat)? LLM.chat(chat) : [] 
+  #conversation = Open.exist?(chat)? LLM.chat(chat) : []
+  #convo_options = LLM.options conversation
+  #conversation = question.empty? ? conversation : conversation +  LLM.chat(question)
+
+  #if dry_run
+  #  ppp LLM.print conversation
+  #  exit 0
+  #end
+
+  #new = agent.ask(conversation, convo_options.merge(options.merge(return_messages: true)))
+  #conversation = Open.read(chat) + LLM.print(new)
+  #Open.write(chat, conversation)
+
+  conversation = Open.exist?(chat)? LLM.chat(chat) : []
   convo_options = LLM.options conversation
-  conversation = question.empty? ? conversation : conversation +  LLM.chat(question)
+  agent.start(Chat.setup(conversation)) if conversation.any?
+  agent.current_chat.concat LLM.chat(question) if question && ! question.empty?
+  imports.each{|import| agent.import import } if imports
 
   if dry_run
     ppp LLM.print conversation
     exit 0
   end
-  new = agent.ask(conversation, convo_options.merge(options.merge(return_messages: true)))
-  conversation = Open.read(chat) + LLM.print(new)
-  Open.write(chat, conversation)
+
+  new = agent.ask(agent.current_chat, convo_options.merge(options.merge(return_messages: true)))
+  conversation += LLM.chat(question) if question
+  conversation += new
+
+  Open.open(chat, mode: 'a'){|f| f.puts LLM.print(new) }
+  puts LLM.purge(new).last[:content]
 elsif inline
 
   file = Open.read inline
 
   new_file = ""
   while true
-    pre, question, post = 
+    pre, question, post =
       file.partition(/^\s*#\s*ask:(?:.*?)(?=^\s*[^\s#])/smu)
 
       break if post.empty?
@@ -169,7 +145,7 @@ elsif inline
     new_file << question
     clean_question = question.gsub('#', '').gsub(/\s+/,' ').sub(/.*ask:\s*/,'').strip
     chat = [
-      {role: :system, content: "Write a succint reply with no commentary and no formatting."}, 
+      {role: :system, content: "Write a succint reply with no commentary and no formatting."},
       {role: :user, content: "Find the following question as a comment in the file give a response to be placed inline: #{question}"},
       LLM.tag('file', file, inline)
     ]
@@ -184,5 +160,10 @@ elsif inline
   new_file << file
   Open.write(inline, new_file)
 else
-  puts agent.ask(question, options)
+  #conversation = Chat.setup(LLM.chat question)
+  #imports.each{|import| conversation.import import } if imports
+  #puts agent.ask(conversation, nil, options)
+  agent.current_chat.concat LLM.chat(question)
+  imports.each{|import| agent.import import } if imports
+  puts agent.chat
 end

data/scout_commands/documenter

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout-ai #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Scout documenter tool
+
+$ #{$0} [<options>] <topic>
+
+Generates technical, example-driven documentation for a given Scout topic by
+analyzing Ruby source and corresponding test files. For a specified topic, it
+locates main and subtopic files, invokes an LLM agent to synthesize markdown
+documentation using real test behavior and examples, then outputs comprehensive
+topic and subtopic documentation files. 
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+require 'scout-ai'
+
+topic = ARGV.first
+
+raise MissingParameterException if topic.nil?
+
+src_files = Scout.lib.scout.glob("#{topic}*/**/*") + Scout.lib.scout.glob("#{topic}.rb")
+src_files.collect! do |file|
+  file.sub(Dir.pwd, '.')
+end
+
+bin_files = Scout.scout_commands.glob("**/*").select{|file| file.include?(topic) && ! file.directory? }
+
+main = src_files.select{|f| f.split("/").length == 4}.first
+subtopics = src_files.select{|f| f.split("/").length == 5}.collect{|f| File.basename(f).sub(".rb",'') }.uniq
+
+def source_to_test(file)
+  file.sub(%r{\A./lib/}, './test/').sub(%r{([^/]+)\.rb\z}, 'test_\1.rb')
+end
+
+documenter = LLM::Agent.new
+
+documenter.start_chat.system <<-EOF
+You are a world-class Ruby documentation author. For each (source, test) file
+pair given, produce technically precise module- and file-level documentation,
+incorporating specific code usage and behavior from the test file as worked
+examples, code idioms, and edge-case handling. 
+
+Never insert your own example code: always use live content from the tests as examples. 
+
+Integrate documentation and test-derived examples smoothly.
+
+You will be given first the main topic documentation for the main file and
+test_file, then you will be asked to produce documentation for a subtopic.
+
+Finally you will be ask to aggregate all the documentation portions into 
+a final topic documentation file
+
+User markdown
+
+Avoid initial and final comments like: Certainly! I'll do this and that
+EOF
+
+documenter.start_chat.file main
+documenter.start_chat.file source_to_test(main)
+
+documenter.start_chat.user <<-EOF
+This is the basic topic file. Write the markdown documentation for it.
+EOF
+
+docs = {}
+subtopics.each do |subtopic|
+  src = src_files.select{|f| f.include? subtopic}.select{|f| f.end_with?(".rb") }
+  test = src.collect{|f| source_to_test(f) }.select{|f| Open.exists? f  }.select{|f| f.end_with?(".rb")}
+
+  documenter.start
+  (src + test + bin_files).each do |file|
+    documenter.file file
+  end
+
+  documenter.start_chat.user <<-EOF
+Write documentation for topic #{topic} subtopic #{subtopic}
+  EOF
+  docs[subtopic] = documenter.respond
+end
+
+documenter.start
+
+docs.each do |subtopic, documentation|
+  documenter.user <<-EOF
+Please construct a comprehensive documentation on topic #{topic}. For each
+subtopic reproduce all the most important from the original documentation
+files. The subtopic documentation files will not be available anymore, so
+don't leave anything imporant out
+
+<file subtopic=#{subtopic}>
+#{documentation}
+<file>
+  EOF
+end
+
+main_documentation = documenter.chat
+
+documenter.start_chat.user <<-EOF
+This is the revise documentation for the topic:
+
+---
+
+#{main_documentation}
+
+---
+EOF
+
+revised_subtopics = {}
+docs.each do |subtopic, documentation|
+  documenter.start
+
+  src = src_files.select{|f| f.include? subtopic}.select{|f| f.end_with?(".rb") }
+  test = src.collect{|f| source_to_test(f) }.select{|f| Open.exists? f  }.select{|f| f.end_with?(".rb")}
+
+  documenter.start
+  (src + test + bin_files).each do |file|
+    documenter.file file
+  end
+
+  documenter.user <<-EOF
+Please revise the subtopic documentation in light of the revised main_documentation
+
+<file subtopic=#{subtopic}>
+#{documentation}
+<file>
+  EOF
+  revised_subtopics[subtopic] = documenter.respond
+end
+
+
+Open.write Scout.doc.lib.scout[topic + '.md'].find(:current), main_documentation
+revised_subtopics.each do |subtopic,documentation|
+  Open.write Scout.doc.lib.scout[topic][subtopic + '.md'].find(:current), documentation
+end

data/scout_commands/llm/ask

@@ -78,6 +78,7 @@ if chat
     ppp LLM.print conversation
     exit 0
   end
+
   new = LLM.ask(conversation, convo_options.merge(options.merge(return_messages: true)))
   conversation = Open.read(chat) + LLM.print(new)
   Open.write(chat, conversation)
@@ -88,9 +89,9 @@ elsif inline
   new_file = ""
   while true
     pre, question, post = 
-      file.partition(/^\s*#\s*ask:(?:.*?)(?=^\s*[^\s#])/smu)
+      file.partition(/^\s*#\s*ask:(?:.*?)(?=^\s*[^\s#]|\z)/smu)
 
-      break if post.empty?
+      break if question.empty?
 
     new_file << pre
     new_file << question