cl-magic 0.4.0 → 1.2.0

data/lib/cl/magic/common/jira.rb

@@ -1,11 +1,70 @@
+require 'tty-progressbar'
+require 'tty-spinner'
+require 'concurrent'
 
 class Jira
 
+  MAX_THREADS = 20 # set to 1 to debug without concurrency
+
   def initialize(base_uri, username, token, break_at_one_page=false)
     @base_uri = base_uri.chomp("/")
     @username = username
     @token = token
     @break_at_one_page = break_at_one_page
+
+    @thread_pool = Concurrent::ThreadPoolExecutor.new(
+      min_threads: 0,
+      max_threads: MAX_THREADS,
+      max_queue: 0,
+      fallback_policy: :caller_runs
+    )
+  end
+
+  #
+  # Formatter
+  #
+
+  def self.jira_to_markdown(issue)
+
+    md = []
+    md << ""
+    md << "# #{issue['key']}"
+    md << "project:     #{issue['fields']['project']['key']}"
+    md << "created:     #{issue['fields']['created']}"
+    md << "updated:     #{issue['fields']['updated']}"
+    md << "status:      #{issue['fields']['status']['statusCategory']['name']}" unless issue['fields']["status"].nil?
+    md << "priority:    #{issue['fields']['priority']['name']}"
+    md << "labels:      #{issue['fields']['labels'].join(',')}"
+    md << "issue_type:  #{issue['fields']['issuetype']['name']}" unless issue['fields']["issuetype"].nil?
+    md << "assignee:    #{issue['fields']['assignee']['displayName']}" unless issue['fields']["assignee"].nil?
+    md << ""
+    md << "## Summary"
+    md << "#{issue['fields']['summary']}"
+    md << ""
+    md << ""
+    issue_md = md.join("\n")
+
+    comments = []
+    issue["comments"].each_with_index do |comment, i|
+      c_md = []
+      c_md << "### Comment - #{comment["author"]["displayName"]} "
+      c_md << ""
+      c_md << "created: #{comment["created"]}"
+
+      # nest markdown deeper
+      comment["body"].split("\n").each do |line|
+        c_md << if line.start_with?("#")
+          "####{line}"
+        else
+          line
+        end
+      end
+
+      c_md << ""
+      comments << [comment["id"], c_md.join("\n")]
+    end
+
+    return issue_md, comments
   end
 
   #
@@ -20,7 +79,7 @@ class Jira
     return epic_ids, epics
   end
 
-  def get_issues(project, epic_ids)
+  def get_issues_by_epic_ids(project, epic_ids)
     jql_query = "project = \"#{project}\" AND parentEpic IN (#{epic_ids.join(',')})"
     return run_jql_query(jql_query)
   end
@@ -33,6 +92,14 @@ class Jira
     end
   end
 
+  def get_issue_comments(issue_key)
+    uri = URI.parse("#{@base_uri}/rest/api/2/issue/#{issue_key}/comment")
+    jira_get(uri) do |response|
+      result = JSON.parse(response.body)
+      return result["comments"]
+    end
+  end
+
   #
   # Helpers: GET & POST
   #
@@ -50,7 +117,11 @@ class Jira
     if response.code == '200'
       yield response
     else
-      raise "Jira query failed with HTTP status code #{response.code}"
+      raise """
+      Jira query failed with HTTP status code #{response.code}
+
+      #{response.body}
+      """
     end
   end
 
@@ -69,7 +140,13 @@ class Jira
     if response.code == '200'
       yield response
     else
-      raise "Jira query failed with HTTP status code #{response.code}"
+      raise """
+      Jira query failed with HTTP status code #{response.code}
+
+      BODY: #{body.to_json}
+
+      RESPONSE: #{response.body}
+      """
     end
   end
 
@@ -78,6 +155,9 @@ class Jira
   #
 
   def run_jql_query(jql)
+    spinner = TTY::Spinner.new("[:spinner] fetching ...", format: :pulse_2)
+    spinner.auto_spin # Automatic animation with default interval
+
     start_at = 0
     max_results = 50
     total_results = nil
@@ -111,53 +191,100 @@ class Jira
           start_at += max_results # else next page
         end
       end
-
-      print '.' # loop
     end
+    spinner.stop("#{all_results.count} issues")
     all_results.map {|h| h}
   end
-end
 
-#
-# Collect status changelogs
-#
-# Given a array of jira issue hashes
-# * fetch the change log
-# * filter down to status changes
-# * add it to the issue hash as ["status_changelogs"]
-#
-
-def collect_status_changelogs(jira, issues, options)
-  final_issue_hashes = []
-
-  issues.each do |issue|
-    issue_key = issue["key"]
-    issue["status_changelogs"] = []
-
-    # fetch change log
-    print '.'
-    changelogs = jira.get_issue_status_changelog(issue_key)
-
-    changelogs.each do |change_log|
-
-      # all items that are status changes
-      status_logs = change_log["items"].select {|i| i["field"]=="status"}
-      status_logs = status_logs.collect do |status_log|
-        {
-          "key": issue_key,
-          "created": change_log["created"],
-          "toString": status_log["toString"],
-          "fromString": status_log["fromString"]
-        }
+  def collect_comments(jira, issues)
+    final_issue_hashes = []
+    bar = TTY::ProgressBar.new("fetching [:bar]", total: issues.count)
+
+    issues.each do |issue|
+      do_concurently do
+        issue_key = issue["key"]
+        issue["comments"] = []
+
+        # fetch change log
+        comments = get_issue_comments(issue_key)
+        issue["comments"] = comments
+        final_issue_hashes << issue # save
+        bar.advance
+      end
+    end
+
+    # wait
+    wait_concurrently
+    return final_issue_hashes
+  end
+
+  #
+  # Collect status changelogs
+  #
+  # Given a array of jira issue hashes
+  # * fetch the change log
+  # * filter down to status changes
+  # * add it to the issue hash as ["status_changelogs"]
+  #
+
+  def collect_status_changelogs(jira, issues)
+    final_issue_hashes = []
+    bar = TTY::ProgressBar.new("fetching [:bar]", total: issues.count)
+
+    issues.each do |issue|
+      do_concurently do
+        issue_key = issue["key"]
+        issue["status_changelogs"] = []
+
+        # fetch change log
+        changelogs = get_issue_status_changelog(issue_key)
+
+        changelogs.each do |change_log|
+
+          # all items that are status changes
+          status_logs = change_log["items"].select {|i| i["field"]=="status"}
+          status_logs = status_logs.collect do |status_log|
+            {
+              "key": issue_key,
+              "created": change_log["created"],
+              "toString": status_log["toString"],
+              "fromString": status_log["fromString"]
+            }
+          end
+
+          # append them to issue
+          status_logs.each do |status_log|
+            issue["status_changelogs"] << status_log
+          end
+        end
+
+        final_issue_hashes << issue # save
+        bar.advance
       end
+    end
 
-      # append them to issue
-      status_logs.each do |status_log|
-        issue["status_changelogs"] << status_log
+    # wait
+    wait_concurrently
+    return final_issue_hashes
+  end
+
+  private
+
+  def do_concurently
+    if MAX_THREADS > 1
+      @thread_pool.post do
+        yield
       end
+    else
+      yield
     end
+  end
 
-    final_issue_hashes << issue # save
+  def wait_concurrently
+    if MAX_THREADS > 1
+      @thread_pool.shutdown
+      @thread_pool.wait_for_termination
+    end
   end
-  return final_issue_hashes
+
 end

data/lib/cl/magic/common/milvus.rb

@@ -0,0 +1,78 @@
+
+class Milvus
+  def initialize(host, port)
+    @host = host
+    @port = port
+  end
+
+  def search(collection_name, embedding)
+    final_url = "http://#{@host}:#{@port}/v1/vector/search"
+    data = {
+      collectionName: collection_name,
+      vector: embedding,
+      outputFields: ["id", "name", "doc_key", "distance"],
+    }
+
+    # post
+    sanitized_data = data.to_json
+    cmd = """
+      curl -s \
+      '#{final_url}' \
+      -X 'POST' \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '#{sanitized_data}'
+    """
+    return `#{cmd}`
+  end
+
+  def create_collection(collection_name)
+    final_url = "http://#{@host}:#{@port}/v1/vector/collections/create"
+    data = {
+      dbName: "default",
+      collectionName: collection_name,
+      dimension: 1536,
+      metricType: "L2",
+      primaryField: "id",
+      vectorField: "vector"
+    }
+
+    # post
+    sanitized_data = data.to_json
+    cmd = """
+      curl -s \
+      '#{final_url}' \
+      -X 'POST' \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '#{sanitized_data}'
+    """
+    return `#{cmd}`
+  end
+
+  def post_to_collection(collection_name, doc_key, embedding)
+    final_url = "http://#{@host}:#{@port}/v1/vector/insert"
+    data = {
+      collectionName: collection_name,
+      data: {
+        doc_key: doc_key,
+        vector: embedding
+      }
+    }
+
+    # post
+    sanitized_data = data.to_json
+    cmd = """
+      curl -s \
+      '#{final_url}' \
+      -X POST \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '#{sanitized_data}'
+    """
+    response = `#{cmd}`
+    data = JSON.parse(response)
+    raise "Error: #{data.to_json}\n\nData #{sanitized_data}" if data.has_key?("message")
+    return data.to_json
+  end
+end

data/lib/cl/magic/dk/help_printer.rb

@@ -0,0 +1,29 @@
+
+class HelpPrinter
+
+  def initialize(logger)
+    @logger = logger
+  end
+
+  def print_dk_help_line(key, help)
+    if $stdout.isatty
+      if help.nil?
+        @logger.puts("#{key.ljust(15, ' ')} ???no help???")
+      else
+        key = key.ljust(15, ' ')
+        help_parts = help.split(";")
+
+        # first line
+        @logger.puts(key, help_parts.shift)
+
+        # following lines
+        padding = "".ljust(15, ' ')
+        help_parts.each do |p|
+          @logger.puts(padding, p)
+        end
+        @logger.puts("") if help.end_with?(";")
+      end
+    end
+  end
+
+end

data/lib/cl/magic/dk/parts_merger.rb

@@ -0,0 +1,67 @@
+require 'tty-command'
+require 'yaml'
+
+class PartsMerger
+
+  def initialize(working_dir, yaml_arg_munger, help_printer, logger)
+    @working_dir = working_dir
+    @yaml_arg_munger = yaml_arg_munger
+    @help_printer = help_printer
+    @logger = logger
+  end
+
+  def merge_parts(compose_hash, dk_parts_hash, args)
+    selected_part_keys = []
+
+    # merge: saved parts
+    saved_part_keys = get_saved_parts(dk_parts_hash)
+    saved_part_keys.each do |potential_part_key|
+      dk_part = dk_parts_hash.fetch(potential_part_key, nil) # yml detail
+      if dk_part
+        compose_hash = print_and_merge_part(potential_part_key, dk_part, compose_hash)
+        selected_part_keys << potential_part_key
+      end
+    end
+
+    # merge: arg parts
+    while true
+      potential_part_key = args.first
+      dk_part = dk_parts_hash.fetch(potential_part_key, nil) # yml detail
+      if dk_part
+        unless selected_part_keys.include?(potential_part_key)
+          compose_hash = print_and_merge_part(potential_part_key, dk_part, compose_hash)
+          selected_part_keys << potential_part_key
+        end
+        args.shift # remove part arg
+      else
+        break
+      end
+    end
+    @logger.puts "" if $stdout.isatty     # tailing line break
+
+    return compose_hash, selected_part_keys, args
+  end
+
+  def get_saved_parts(dk_parts_hash)
+
+    # get saved parts
+    cmd = "cd #{@working_dir} && cl dk parts list"
+    out, err = TTY::Command.new(:printer => :null).run("#{cmd}")
+    return out.split("\n")
+
+  end
+
+  private
+
+  def print_and_merge_part(part_key, dk_part, compose_hash)
+
+    # print
+    if $stdout.isatty
+      help_str = dk_part.fetch('help')
+      @help_printer.print_dk_help_line("#{part_key}", "#{help_str ? '- ' + help_str : ''}") if dk_part.keys.any?
+    end
+    # merge
+    return @yaml_arg_munger.dk_merge_and_remove(compose_hash, dk_part)
+  end
+
+end

data/lib/cl/magic/dk/world_settings.rb

@@ -0,0 +1,52 @@
+require 'tty-command'
+require 'yaml'
+
+class WorldSettings
+
+  def initialize(working_dir)
+    @working_dir = working_dir
+  end
+
+  def get_world_settings_hash()
+    filepath = get_world_settings_filepath()
+    return File.exist?(filepath) ? YAML.load_file(filepath) : {}
+  end
+
+  def save_world_settings(world_settings_hash)
+    filepath = get_world_settings_filepath()
+    tempfile = File.new(filepath, 'w')
+    tempfile.write(world_settings_hash.to_yaml)
+    tempfile.close
+  end
+
+  def get_world_project_path()
+    repo_basename = get_repo_basename()
+    world_path = get_world_path_from_settings()
+    return File.join(world_path, repo_basename) if world_path and repo_basename
+    return nil
+  end
+
+  def get_world_path_from_settings()
+    world_settings = get_world_settings_hash()
+    if world_settings.key?(:world_path) and world_settings.key?(:context)
+      return File.join(world_settings[:world_path], world_settings[:context])
+    end
+    return ""
+  end
+
+  private
+
+  def get_repo_basename()
+    command = "cd #{@working_dir} && basename $(git remote get-url origin 2> /dev/null) .git"
+    repo_basename = TTY::Command.new(:printer => :null).run(command).out.gsub('.git', '').strip.chomp
+    if repo_basename==".git" or repo_basename==""
+      return File.basename(@working_dir)
+    end
+    return repo_basename
+  end
+
+  def get_world_settings_filepath()
+    return File.join(".cl-dk-world.yml")
+  end
+
+end

data/lib/cl/magic/dk/yaml_arg_munger.rb

@@ -0,0 +1,107 @@
+require 'yaml'
+
+class YamlArgMunger
+
+  def initialize(working_dir, world_settings)
+    @working_dir = working_dir
+    @world_path = world_settings.get_world_path_from_settings()
+    @dk_proj_path = world_settings.get_world_project_path()
+  end
+
+  def get_base_compose_parts_and_make_hashes()
+    compose_hash = get_base_compose_hash()
+    dk_parts_hash = {}
+    dk_make_hash = {}
+    if compose_hash
+      compose_hash = merge_world_files(compose_hash, show_help=ARGV.include?("--help"))
+      dk_parts_hash = compose_hash['x-dk-parts'] ? compose_hash.delete('x-dk-parts') : {}
+      dk_make_hash = compose_hash['x-dk-make'] ? compose_hash.delete('x-dk-make') : {}
+    end
+    return compose_hash, dk_parts_hash, dk_make_hash
+  end
+
+  def dk_merge_and_remove(compose_hash, yml_hash)
+    # remove help & merge
+    clean_yml = yml_hash.clone
+    clean_yml.delete('help')
+    return compose_hash.dk_merge(clean_yml).dk_reject! { |k, v| v=='<dk-remove>' }
+  end
+
+  def save_yaml_and_adjust_args(compose_hash, args)
+
+    # generate final compose file
+    tempfile = File.new(File.join(@working_dir, ".cl-dk.yml"), 'w')
+    tempfile.write(compose_hash.to_yaml)  # write it to the tempfile
+
+    # remove existing '-f' flag, if needed
+    file_flag_index = args.index('-f')
+    if file_flag_index==0
+      args.delete_at(file_flag_index)
+      args.delete_at(file_flag_index)
+    end
+    args.unshift('-f', tempfile.path) # add new '-f' flag
+
+    tempfile.close
+    return args
+  end
+
+  private
+
+  def get_base_compose_hash()
+    cmd = "cd #{@working_dir} && docker compose config 2> /dev/null"
+    return YAML.load(`#{cmd}`)
+  end
+
+  def merge_world_files(compose_hash, show_help=false)
+    if @dk_proj_path
+      print_dk_help_line("dk-project-path", "#{@dk_proj_path}") if show_help and $stdout.isatty
+
+      Dir.glob("#{@dk_proj_path}/*.yml").sort.each do |filepath|
+        print_dk_help_line("dk-world", "#{filepath}") if show_help and $stdout.isatty
+
+        # read file and replace
+        contents = File.read(filepath)
+        contents.gsub!('<dk-world-path>', @world_path)
+        contents.gsub!('<dk-project-path>', @dk_proj_path)
+        contents.gsub!('<dk-working-path>', @working_dir)
+
+        # yml merge
+        yml_hash = YAML.load(contents)
+        compose_hash = dk_merge_and_remove(compose_hash, yml_hash)
+      end
+    end
+    return compose_hash
+  end
+
+  class ::Hash
+    def dk_merge(second)
+      merger = proc { |_, v1, v2|
+        if Hash === v1 && Hash === v2
+          v1.merge(v2, &merger)
+        else
+          if Array === v1 && Array === v2
+            if v2.first=="<dk-replace>"
+              v2[1..-1] # everything but the first item
+            else
+              v1 | v2 # union arrays
+            end
+          else
+            if [:undefined, nil, :nil].include?(v2)
+              v1
+            else
+              v2
+            end
+          end
+        end
+      }
+      merge(second.to_h, &merger)
+    end
+    def dk_reject!(&blk)
+      self.each do |k, v|
+        v.dk_reject!(&blk)  if v.is_a?(Hash)
+        self.delete(k)  if blk.call(k, v)
+      end
+    end
+  end
+
+end

data/lib/cl/magic/version.rb

@@ -2,6 +2,6 @@
 
 module Cl
   module Magic
-    VERSION = "0.4.0"
+    VERSION = "1.2.0"
   end
 end