scout-ai 0.2.0 → 1.0.1

data/scout_commands/agent/ask

@@ -19,8 +19,12 @@ Use STDIN to add context to the question
 -h--help Print this help
 -l--log* Log level
 -t--template* Use a template
+-c--chat* Follow a conversation
 -m--model* Model to use
--f--file* Incorporate file at the start
+-e--endpoint* Endpoint to use
+-f--file* Incorporate file
+-wt--workflow_tasks* Export these tasks to the agent
+-i--imports* Chat files to import, separated by comma
 EOF
 if options[:help]
   if defined? scout_usage
@@ -33,38 +37,109 @@ end
 
 Log.severity = options.delete(:log).to_i if options.include? :log
 
-file = options.delete(:file)
+agent_name, *question_parts = ARGV
 
-agent, *question_parts = ARGV
+question = question_parts * " "
 
+file, chat, inline, template, dry_run, imports = IndiferentHash.process_options options, :file, :chat, :inline, :template, :dry_run, :imports
 
-workflow = begin
-             Workflow.require_workflow agent
-           rescue
-           end
+file = Path.setup(file) if file
 
-knowledge_base = begin workflow.knowledge_base rescue nil end || KnowledgeBase.new(Scout.var.Agent[agent])
+imports = imports.split(/,\s*/) if imports
 
-agent = LLM::Agent.new workflow: workflow, knowledge_base: knowledge_base
 
-question = question_parts * " " 
+agent_name ||= 'default'
 
-if template = options.delete(:template)
+agent_file = Scout.workflows[agent_name]
+
+agent_file = Scout.chats[agent_name] unless agent_file.exists?
+
+agent_file = agent_file.find_with_extension('rb') unless agent_file.exists?
+
+
+if agent_file.exists?
+  if agent_file.directory?
+    if agent_file.agent.find_with_extension('rb').exists?
+      agent = load agent_file.agent.find_with_extension('rb')
+    else
+      agent = LLM::Agent.load_from_path agent_file
+    end
+  else
+    agent = load agent_file
+  end
+else
+  #raise ParameterException agent_file
+end
+
+if template
   if Open.exists?(template)
     template_question = Open.read(template)
-  else
+  elsif Scout.questions[template].exists?
     template_question = Scout.questions[template].read
+  elsif Scout.chats.system[template].exists?
+    template_question = Scout.chats.system[template].read
+  elsif Scout.chats[template].exists?
+    template_question = Scout.chats[template].read
   end
   if template_question.include?('???')
     question = template_question.sub('???', question)
+  elsif not question.empty?
+    question = template_question + "\nuser: #{question}"
   else
     question = template_question
   end
 end
 
 if question.include?('...')
-  context = file ? Open.read(file) : STDIN.read 
+  context = file ? Open.read(file) : STDIN.read
   question = question.sub('...', context)
+elsif file
+  question = "<file basename=#{File.basename file}>\n" + Open.read(file) + "\n</file>\n\n" + question
 end
 
-puts LLM.ask(question, options)
+if chat
+  conversation = Open.exist?(chat)? LLM.chat(chat) : [] 
+  convo_options = LLM.options conversation
+  conversation = question.empty? ? conversation : conversation +  LLM.chat(question)
+
+  if dry_run
+    ppp LLM.print conversation
+    exit 0
+  end
+  new = agent.ask(conversation, convo_options.merge(options.merge(return_messages: true)))
+  conversation = Open.read(chat) + LLM.print(new)
+  Open.write(chat, conversation)
+elsif inline
+
+  file = Open.read inline
+
+  new_file = ""
+  while true
+    pre, question, post = 
+      file.partition(/^\s*#\s*ask:(?:.*?)(?=^\s*[^\s#])/smu)
+
+      break if post.empty?
+
+    new_file << pre
+    new_file << question
+    clean_question = question.gsub('#', '').gsub(/\s+/,' ').sub(/.*ask:\s*/,'').strip
+    chat = [
+      {role: :system, content: "Write a succint reply with no commentary and no formatting."}, 
+      {role: :user, content: "Find the following question as a comment in the file give a response to be placed inline: #{question}"},
+      LLM.tag('file', file, inline)
+    ]
+    response = LLM.ask(LLM.chat(chat))
+    new_file << <<-EOF
+# Response start
+#{response}
+# Response end
+    EOF
+    file = post
+  end
+  new_file << file
+  Open.write(inline, new_file)
+else
+  conversation = Chat.setup(LLM.chat question)
+  imports.each{|import| conversation.import import } if imports
+  puts agent.ask(conversation, nil, options)
+end

data/scout_commands/agent/kb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+agent = ARGV.shift
+
+agent_dir = Scout.var.Agent[agent]
+
+if ARGV.any?
+  ARGV.push "--knowledge_base"
+  ARGV.push agent_dir.knowledge_base
+  ARGV.push "--log"
+  ARGV.push Log.severity.to_s
+end
+ARGV.unshift 'kb'
+
+load Scout.bin.scout.find

data/scout_commands/documenter

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout-ai #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Scout documenter tool
+
+$ #{$0} [<options>] <topic>
+
+Generates technical, example-driven documentation for a given Scout topic by
+analyzing Ruby source and corresponding test files. For a specified topic, it
+locates main and subtopic files, invokes an LLM agent to synthesize markdown
+documentation using real test behavior and examples, then outputs comprehensive
+topic and subtopic documentation files. 
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+require 'scout-ai'
+
+topic = ARGV.first
+
+raise MissingParameterException if topic.nil?
+
+src_files = Scout.lib.scout.glob("#{topic}*/**/*") + Scout.lib.scout.glob("#{topic}.rb")
+src_files.collect! do |file|
+  file.sub(Dir.pwd, '.')
+end
+
+bin_files = Scout.scout_commands.glob("**/*").select{|file| file.include?(topic) && ! file.directory? }
+
+main = src_files.select{|f| f.split("/").length == 4}.first
+subtopics = src_files.select{|f| f.split("/").length == 5}.collect{|f| File.basename(f).sub(".rb",'') }.uniq
+
+def source_to_test(file)
+  file.sub(%r{\A./lib/}, './test/').sub(%r{([^/]+)\.rb\z}, 'test_\1.rb')
+end
+
+documenter = LLM::Agent.new
+
+documenter.start_chat.system <<-EOF
+You are a world-class Ruby documentation author. For each (source, test) file
+pair given, produce technically precise module- and file-level documentation,
+incorporating specific code usage and behavior from the test file as worked
+examples, code idioms, and edge-case handling. 
+
+Never insert your own example code: always use live content from the tests as examples. 
+
+Integrate documentation and test-derived examples smoothly.
+
+You will be given first the main topic documentation for the main file and
+test_file, then you will be asked to produce documentation for a subtopic.
+
+Finally you will be ask to aggregate all the documentation portions into 
+a final topic documentation file
+
+User markdown
+
+Avoid initial and final comments like: Certainly! I'll do this and that
+EOF
+
+documenter.start_chat.file main
+documenter.start_chat.file source_to_test(main)
+
+documenter.start_chat.user <<-EOF
+This is the basic topic file. Write the markdown documentation for it.
+EOF
+
+docs = {}
+subtopics.each do |subtopic|
+  src = src_files.select{|f| f.include? subtopic}.select{|f| f.end_with?(".rb") }
+  test = src.collect{|f| source_to_test(f) }.select{|f| Open.exists? f  }.select{|f| f.end_with?(".rb")}
+
+  documenter.start
+  (src + test + bin_files).each do |file|
+    documenter.file file
+  end
+
+  documenter.start_chat.user <<-EOF
+Write documentation for topic #{topic} subtopic #{subtopic}
+  EOF
+  docs[subtopic] = documenter.respond
+end
+
+documenter.start
+
+docs.each do |subtopic, documentation|
+  documenter.user <<-EOF
+Please construct a comprehensive documentation on topic #{topic}. For each
+subtopic reproduce all the most important from the original documentation
+files. The subtopic documentation files will not be available anymore, so
+don't leave anything imporant out
+
+<file subtopic=#{subtopic}>
+#{documentation}
+<file>
+  EOF
+end
+
+main_documentation = documenter.chat
+
+documenter.start_chat.user <<-EOF
+This is the revise documentation for the topic:
+
+---
+
+#{main_documentation}
+
+---
+EOF
+
+revised_subtopics = {}
+docs.each do |subtopic, documentation|
+  documenter.start
+
+  src = src_files.select{|f| f.include? subtopic}.select{|f| f.end_with?(".rb") }
+  test = src.collect{|f| source_to_test(f) }.select{|f| Open.exists? f  }.select{|f| f.end_with?(".rb")}
+
+  documenter.start
+  (src + test + bin_files).each do |file|
+    documenter.file file
+  end
+
+  documenter.user <<-EOF
+Please revise the subtopic documentation in light of the revised main_documentation
+
+<file subtopic=#{subtopic}>
+#{documentation}
+<file>
+  EOF
+  revised_subtopics[subtopic] = documenter.respond
+end
+
+
+Open.write Scout.doc.lib.scout[topic + '.md'].find(:current), main_documentation
+revised_subtopics.each do |subtopic,documentation|
+  Open.write Scout.doc.lib.scout[topic][subtopic + '.md'].find(:current), documentation
+end

data/scout_commands/llm/ask

@@ -3,22 +3,30 @@
 require 'scout'
 require 'scout-ai'
 
-$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+$0 = "scout-ai #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
 options = SOPT.setup <<EOF
+Ask an LLM model
 
-Ask GPT
+$ #{$0} [<options>] [<question>]
 
-$ #{$0} [<options>] [question]
-
-Use STDIN to add context to the question
+Use STDIN to add context to the question. The context can be referenced using
+three dots '...'. The model will be prompted with the question, unless the
+inline option is used. If the chat option is used, the response will be added
+to the end of the file. If the file option is used the file contents will be
+prepended before the question.  With the template option, the file will be read
+as if it were the question, and the actual question will be placed under the
+characters '???', if they are present.
 
 -h--help Print this help
--l--log* Log level
 -t--template* Use a template
+-c--chat* Follow a conversation
+-i--inline* Ask inline questions about a file
+-f--file* Incorporate file at the start
 -m--model* Model to use
 -e--endpoint* Endpoint to use
--f--file* Incorporate file at the start
+-b--backend* Backend to use
+-d--dry_run Dry run, don't ask
 EOF
 if options[:help]
   if defined? scout_usage
@@ -31,26 +39,77 @@ end
 
 Log.severity = options.delete(:log).to_i if options.include? :log
 
-file = options.delete(:file)
+file, chat, inline, template, dry_run = IndiferentHash.process_options options, :file, :chat, :inline, :template, :dry_run
 
 question = ARGV * " "
 
-if template = options.delete(:template)
+if template
   if Open.exists?(template)
     template_question = Open.read(template)
-  else
+  elsif Scout.questions[template].exists?
     template_question = Scout.questions[template].read
+  elsif Scout.chats.system[template].exists?
+    template_question = Scout.chats.system[template].read
+  elsif Scout.chats[template].exists?
+    template_question = Scout.chats[template].read
   end
   if template_question.include?('???')
     question = template_question.sub('???', question)
+  elsif not question.empty?
+    question = template_question + "\nuser: #{question}"
   else
     question = template_question
   end
 end
 
 if question.include?('...')
-  context = file ? Open.read(file) : STDIN.read 
+  context = file ? Open.read(file) : STDIN.read
   question = question.sub('...', context)
+elsif file
+  question = "<file basename=#{File.basename file}>[[[\n" + Open.read(file) + "\n]]]</file>"
 end
 
-puts LLM.ask(question, options)
+if chat
+  conversation = Open.exist?(chat)? LLM.chat(chat) : [] 
+  convo_options = LLM.options conversation
+  conversation = question.empty? ? conversation : conversation +  LLM.chat(question)
+
+  if dry_run
+    ppp LLM.print conversation
+    exit 0
+  end
+  new = LLM.ask(conversation, convo_options.merge(options.merge(return_messages: true)))
+  conversation = Open.read(chat) + LLM.print(new)
+  Open.write(chat, conversation)
+elsif inline
+
+  file = Open.read inline
+
+  new_file = ""
+  while true
+    pre, question, post = 
+      file.partition(/^\s*#\s*ask:(?:.*?)(?=^\s*[^\s#]|\z)/smu)
+
+      break if question.empty?
+
+    new_file << pre
+    new_file << question
+    clean_question = question.gsub('#', '').gsub(/\s+/,' ').sub(/.*ask:\s*/,'').strip
+    chat = [
+      {role: :system, content: "Write a succint reply with no commentary and no formatting."}, 
+      {role: :user, content: "Find the following question as a comment in the file give a response to be placed inline: #{question}"},
+      LLM.tag('file', file, inline)
+    ]
+    response = LLM.ask(LLM.chat(chat))
+    new_file << <<-EOF
+# Response start
+#{response}
+# Response end
+    EOF
+    file = post
+  end
+  new_file << file
+  Open.write(inline, new_file)
+else
+  puts LLM.ask(question, options)
+end

data/scout_commands/llm/process

@@ -32,7 +32,9 @@ directory = ARGV.first || Scout.var.ask.find
 directory = Path.setup directory
 
 while true
-  directory.glob('*.json').each do |file|
+  files = directory.glob('*.json')
+
+  files.each do |file|
     target = directory.reply[id + '.json']
 
     if ! File.exist?(target)
@@ -46,5 +48,5 @@ while true
     Open.rm(file)
   end
 
-  sleep 1
+  sleep 1 if files.empty?
 end