cl-magic 0.4.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08b4655abcd92a0d5e3a779fdc7e0e09231c7cb15d073e06878c085227ef3f9d'
-  data.tar.gz: 63999d535d6ad18d22b8cff1e259b9b054bbf7faf75939e54c9819885ca24feb
+  metadata.gz: e7f003696dcc6355ea8ee76ac94c01d8d8b324032a1c5214352de53ef0d90fd8
+  data.tar.gz: bf5e85e1e57a35a840eae95707af831e8079ce0fc85b1da87e229c89fb20c292
 SHA512:
-  metadata.gz: 4157d207e824e2d0a309790187873d3b6b02c41bd892fc3436e359cc1f4639044576129d1a8d36ac9a0faee8afa6dc2366db506f93401bf4165e93766435f3d7
-  data.tar.gz: 0bce3b21105e81dd1bc1d5656e941ceec30814bb01e19c16a7a5ee57eba98403093d7359eb9e113bd60b0850a87764b1241a08febe1881dcde74c303fb9d35be
+  metadata.gz: 80a97ad2355650603ac4627fc9fecc11cf5b7f558f1a835c3824d579de3bb82ac627a49d0f46cac119518a12b0215f107467e5753dff202d003d7bd578dd6f5a
+  data.tar.gz: 65023f6827983ddcd9f4fddca778f1cd8df8566764a943f6b7e0d6bce7b81743358262900b63964a7e5f511499293573c46bb5ce8f4e73f049f739ecb50be71b

data/Gemfile.lock

@@ -1,22 +1,27 @@
 PATH
   remote: .
   specs:
-    cl-magic (0.4.0)
+    cl-magic (1.2.0)
       activesupport
+      baran
+      concurrent-ruby
       optparse-subcommand
       pastel
       tty-command
       tty-logger
+      tty-progressbar
       tty-prompt
+      tty-spinner
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.6)
+    activesupport (7.0.7)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
+    baran (0.1.7)
     byebug (11.1.3)
     concurrent-ruby (1.2.2)
     i18n (1.14.1)
@@ -26,12 +31,18 @@ GEM
     pastel (0.8.0)
       tty-color (~> 0.5)
     rake (13.0.6)
+    strings-ansi (0.2.0)
     tty-color (0.6.0)
     tty-command (0.10.1)
       pastel (~> 0.8)
     tty-cursor (0.7.1)
     tty-logger (0.6.0)
       pastel (~> 0.8)
+    tty-progressbar (0.18.2)
+      strings-ansi (~> 0.2)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.8)
+      unicode-display_width (>= 1.6, < 3.0)
     tty-prompt (0.23.1)
       pastel (~> 0.8)
       tty-reader (~> 0.8)
@@ -40,8 +51,11 @@ GEM
       tty-screen (~> 0.8)
       wisper (~> 2.0)
     tty-screen (0.8.1)
+    tty-spinner (0.9.3)
+      tty-cursor (~> 0.7)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    unicode-display_width (2.4.2)
     wisper (2.0.1)
 
 PLATFORMS

data/cl-magic.gemspec

@@ -33,8 +33,13 @@ Gem::Specification.new do |spec|
   spec.add_dependency "tty-logger"
   spec.add_dependency "tty-command"
   spec.add_dependency "tty-prompt"
+  spec.add_dependency "tty-spinner"
+  spec.add_dependency "tty-progressbar"
+  spec.add_dependency "concurrent-ruby"
   spec.add_dependency "pastel"
   spec.add_dependency "activesupport"
+  spec.add_dependency "baran"
+
   spec.add_development_dependency "byebug"
 
 end

data/lib/cl/magic/cl

@@ -3,7 +3,6 @@ require 'optparse'
 require 'tty-command'
 require 'cl/magic/common/logging.rb'
 
-
 @logger = get_logger()
 
 #
@@ -76,26 +75,34 @@ def print_commands(commands, prefix='   ')
   end
 end
 
-def execute_subcommand(subcommand, command_tree)
-  commands = command_tree.fetch(subcommand, [])
+def execute_subcommand(cmd, command_tree)
+  # global command?
+  global_commands = command_tree['']
+  global_cmd = global_commands.select {|c| c[:cmd]==cmd}.first
+
+  # subcommand?
+  sub_commands = command_tree.fetch(cmd, [])
+  next_arg = ARGV.first
+  sub_cmd = sub_commands.select {|c| c[:cmd]==next_arg}.first
 
-  case commands.length
-  when 0
-    commands = command_tree[''] # global command?
+  # found sub command?
+  if sub_cmd
+    ARGV.shift
+    cmd = sub_cmd
   else
-    commands = command_tree.fetch(subcommand, [])
-    sub_subcommand = ARGV.shift if commands.any? # next part of command
-    if sub_subcommand.nil?
+    # found global command?
+    if global_cmd
+      cmd = global_cmd
+    else
+      # else print help
       puts ""
-      puts "#{subcommand}"
-      print_commands(commands)
+      puts "#{cmd}"
+      print_commands(sub_commands)
       puts ""
       exit(0)
-    else
-      subcommand = sub_subcommand
     end
   end
-  cmd = commands.select {|c| c[:cmd]==subcommand}.first
+
   final = "bundle exec ./#{cmd[:filename]} #{ARGV.join(' ')}"
   exec(final)
 end

data/lib/cl/magic/cl-ai-chat

@@ -0,0 +1,117 @@
+#!/usr/bin/env ruby
+# Call open ai chat api
+require 'optparse'
+require 'optparse/subcommand'
+require 'tty-command'
+require 'tty-prompt'
+
+require 'cl/magic/common/common_options.rb'
+require 'cl/magic/common/logging.rb'
+require 'cl/magic/common/ai_prompt.rb'
+
+@logger = get_logger()
+@cl_cmd_name = File.basename(__FILE__).split('-').join(' ')
+
+def do_prompt(options, ai, data)
+  @logger.info("#{data.length} chars")
+  data = ai.prompt(data, options[:prompt], options[:split_as_markdown], options[:split_separator])
+  return data if data.count == 1 or not options[:recurse]
+
+  # recurse
+  options[:split_as_markdown] = false # disable on second pass
+  return do_prompt(options, ai, data.join("\n"))
+end
+
+#
+# Features
+#
+
+def do_work(options, data)
+  ai = AIPrompt.new(@logger, @working_dir, options[:max_chunk_size], options[:temperature])
+  ai.clear_cache()
+
+  @logger.info "max_chunk_size: #{options[:max_chunk_size]} | temperature: #{options[:temperature]}"
+  @logger.info "prompt: #{options[:prompt]}"
+
+  data = do_prompt(options, ai, data)
+  data.each do |json|
+    @logger.puts ""
+    puts json
+  end
+end
+
+#
+# Options
+#
+
+options = {
+  max_chunk_size: 10000,
+  temperature: 1.0
+}
+global_banner = <<DOC
+
+ Chat API - https://platform.openai.com/docs/api-reference/chat
+
+ Usage: #{@cl_cmd_name} [options] -- prompt
+
+DOC
+
+global = OptionParser.new do |g|
+  g.banner = global_banner
+  add_help_and_verbose(g)
+
+  g.on("-f", "--filepath FILEPATH", "relative path to file containing prompt data") do |v|
+    options[:data_filepath] = v
+  end
+
+  g.on("-m", "--max-chunk SIZE", "(default: 10,000) lower this if you hit token limit with LLM") do |v|
+    options[:max_chunk_size] = v.to_i
+  end
+
+  g.on("-t", "--temperature NUMBER", "(default: 1.0)    lower for more deterministic results; whatever that means?") do |v|
+    options[:temperature] = v.to_f
+  end
+
+  g.on("--split-separator VALUE", "(default: '\\n\\n') string to split text by") do |v|
+    options[:split_separator] = v
+  end
+
+  g.on("--split-md", "optional, split by markdown") do |v|
+    options[:split_as_markdown] = v
+  end
+
+  g.on("--recurse", "optional, reduce down to 1 chunk") do |v|
+    options[:recurse] = v
+  end
+end
+
+#
+# Run
+#
+
+@working_dir = ENV['CL_WORKING_DIR'] # passed through cl-magic to here
+global.parse!(ARGV)
+options[:prompt] = ARGV.join(' ')
+
+# keys
+["OPENAPI_KEY", "OPENAPI_URL"].each do |env_key|
+  unless ENV[env_key]
+    @logger.error "missing environment variable: #{env_key}"
+    exit 1
+  end
+end
+
+if options[:data_filepath].nil?
+  puts global.parse! %w[--help]
+  exit 1
+end
+
+ask_and_store_option(options, :prompt, "prompt?")
+
+# chat
+filepath = File.join(@working_dir, options[:data_filepath])
+data = File.readlines(filepath).join("\n")
+do_work(options, data)
+
+# history
+write_history("""#{@cl_cmd_name} --filepath #{options[:data_filepath]} -- #{options[:prompt]}""")

data/lib/cl/magic/cl-ai-query

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+# Prompt for similar content
+require 'optparse'
+require 'optparse/subcommand'
+require 'tty-command'
+require 'tty-prompt'
+
+require 'cl/magic/common/common_options.rb'
+require 'cl/magic/common/logging.rb'
+require 'cl/magic/common/jira.rb'
+require 'cl/magic/common/ai_prompt.rb'
+require 'cl/magic/common/milvus.rb'
+require 'cl/magic/common/elastic.rb'
+
+@logger = get_logger()
+
+#
+# Features
+#
+
+def do_work(options)
+
+  # embedding from prompt
+  @logger.wait "convert prompt to embedding"
+  ai = AIPrompt.new(@logger, @working_dir)
+  embedding = ai.gen_embeddings(options[:prompt])
+
+  # search
+  @logger.wait "search vector db"
+  milvus = Milvus.new(options[:milvus_host], options[:milvus_port])
+  collection_name = "jira_markdown"
+  response = milvus.search(collection_name, embedding)
+
+  # doc keys
+  data = JSON.parse(response)["data"]
+  data = data.select{|h| h["distance"] > 0.6} # reduce
+  doc_keys = data.map {|h| h["doc_key"]}
+
+  # elastic content
+  @logger.wait "fetch documents from elastic"
+  elastic = Elastic.new(options[:elastic_url])
+  response = elastic.query_by_id(doc_keys)
+  docs = JSON.parse(response)
+  puts docs["hits"]["hits"].map { |h| h["_source"]["markdown"] }.join("\n\n")
+end
+
+#
+# Options
+#
+
+options = {}
+
+global_banner = <<DOC
+
+ Prompt for similar content
+
+ Usage: cl ai query [options]
+
+DOC
+
+global = OptionParser.new do |g|
+  g.banner = global_banner
+  add_help_and_verbose(g)
+
+  g.on("--milvus-host HOST", "host for milvus vector db") do |v|
+    options[:milvus_host] = v
+  end
+
+  g.on("--milvus-port PORT", "port for milvus vector db") do |v|
+    options[:milvus_port] = v
+  end
+
+  g.on("--elastic URL", "url to elastic search server") do |v|
+    options[:elastic_url] = v
+  end
+
+end
+
+#
+# Run
+#
+
+@working_dir = ENV['CL_WORKING_DIR'] # passed through cl-magic to here
+global.parse!(ARGV)
+options[:prompt] = ARGV.join(' ')
+
+# keys
+["OPENAPI_KEY", "OPENAPI_URL"].each do |env_key|
+  unless ENV[env_key]
+    @logger.error "missing environment variable: #{env_key}"
+    exit 1
+  end
+end
+
+# required options
+[
+  :milvus_host,
+  :milvus_port,
+  :elastic_url,
+].each do |key|
+  if options[key].nil?
+    puts global.parse! %w[--help]
+    exit 1
+  end
+end
+
+ask_and_store_option(options, :prompt, "prompt?")
+
+write_history("""cl ai query \\
+  --elastic #{options[:elastic_url]} \\
+  --milvus-host #{options[:milvus_host]} \\
+  --milvus-port #{options[:milvus_port]} \\
+  -- #{options[:prompt]}
+""")
+
+do_work(options)

data/lib/cl/magic/cl-ai-store-jira

@@ -0,0 +1,158 @@
+#!/usr/bin/env ruby
+# Store jira json for AI
+require 'optparse'
+require 'optparse/subcommand'
+require 'tty-command'
+require 'tty-prompt'
+
+require 'cl/magic/common/common_options.rb'
+require 'cl/magic/common/logging.rb'
+require 'cl/magic/common/jira.rb'
+require 'cl/magic/common/ai_prompt.rb'
+require 'cl/magic/common/milvus.rb'
+require 'cl/magic/common/elastic.rb'
+
+@logger = get_logger()
+
+def store_elastic(elastic, elastic_index, issues)
+  issues.each do |i|
+    issue_md, comments = Jira.jira_to_markdown(i)
+
+    # issue
+    url = "#{elastic_index}/_doc/issue_#{i["id"]}"
+    @logger.puts elastic.post(url, "POST", {
+      "markdown": issue_md
+    })
+
+    comments.each do |comment|
+
+      # comment
+      id, comment_md = comment
+      url = "#{elastic_index}/_doc/comment_#{id}"
+      @logger.puts elastic.post(url, "POST", {
+        "markdown": comment_md
+      })
+
+    end
+  end
+end
+
+def store_vector(ai, milvus, issues)
+  collection_name = "jira_markdown"
+  @logger.puts milvus.create_collection(collection_name)
+
+  issues.each do |i|
+    issue_md, comments = Jira.jira_to_markdown(i)
+
+    # issue
+    doc_key = "issue_#{i["id"]}"
+    embedding = ai.gen_embeddings(issue_md)
+    @logger.puts milvus.post_to_collection(collection_name, doc_key, embedding)
+
+    comments.each do |comment|
+      comment_id, comment_md = comment
+
+      # comment
+      doc_key = "comment_#{comment_id}"
+      embedding = ai.gen_embeddings(issue_md)
+      @logger.puts milvus.post_to_collection(collection_name, doc_key, embedding)
+    end
+  end
+end
+
+#
+# Features
+#
+
+def do_work(options)
+
+  filepath = File.join(@working_dir, options[:data_filepath])
+  issues = JSON.parse(File.read(filepath))
+
+  # elastic
+  elastic = Elastic.new(options[:elastic_url])
+  index = options[:elastic_index]
+  @logger.puts elastic.create_index(index, {
+    "mappings": {
+      "properties": {
+        "markdown": {
+          "type": "text"
+        }
+      }
+    }
+  })
+  store_elastic(elastic, index, issues)
+
+  # vector
+  ai = AIPrompt.new(@logger, @working_dir)
+  milvus = Milvus.new(options[:milvus_host], options[:milvus_port])
+  store_vector(ai, milvus, issues)
+end
+
+#
+# Options
+#
+
+options = {
+  elastic_index: "jira_markdown"
+}
+
+global_banner = <<DOC
+
+ Store a jira json for the AI
+
+ Usage: cl ai store-jira [options]
+
+DOC
+
+global = OptionParser.new do |g|
+  g.banner = global_banner
+  add_help_and_verbose(g)
+
+  g.on("--milvus-host HOST", "host for milvus vector db") do |v|
+    options[:milvus_host] = v
+  end
+
+  g.on("--milvus-port PORT", "port for milvus vector db") do |v|
+    options[:milvus_port] = v
+  end
+
+  g.on("--elastic URL", "url to elastic search server") do |v|
+    options[:elastic_url] = v
+  end
+
+  g.on("-f", "--filepath FILEPATH", "relative path to jira json") do |v|
+    options[:data_filepath] = v
+  end
+
+end
+
+#
+# Run
+#
+
+@working_dir = ENV['CL_WORKING_DIR'] # passed through cl-magic to here
+global.parse!(ARGV)
+
+# required options
+[
+  :data_filepath,
+  :milvus_host,
+  :milvus_port,
+  :elastic_url,
+  :data_filepath
+].each do |key|
+  if options[key].nil?
+    puts global.parse! %w[--help]
+    exit 1
+  end
+end
+
+write_history("""cl ai store-jira \\
+  --elastic #{options[:elastic_url]} \\
+  --filepath #{options[:data_filepath]} \\
+  --milvus-host #{options[:milvus_host]} \\
+  --milvus-port #{options[:milvus_port]}
+""")
+
+do_work(options)