greenhat 0.1.4

data/lib/greenhat/shell/faststats.rb

@@ -0,0 +1,195 @@
+module GreenHat
+  # CLI Helper
+  module Shell
+    # Logs
+    module Faststats
+      # rubocop:disable Metrics/MethodLength)
+      def self.help
+        puts "\u2500".colorize(:cyan) * 25
+        puts "Gimme #{'Performance Stats'.colorize(:yellow)}"
+        puts "\u2500".colorize(:cyan) * 25
+
+        puts 'General'.colorize(:blue)
+        puts "  Any double dash arguments (e.g. #{'--color-output'.colorize(:green)}) are passed directly to fast-stats"
+        puts "  See #{'`fast-stats --help`'.colorize(:light_black)} for all available options"
+        puts
+
+        puts 'Common Options'.colorize(:blue)
+        puts '  --raw'.colorize(:green)
+        puts '    Do not use less/paging'
+        puts '  --display'.colorize(:green)
+        puts '     Show percentage of totals and/or actual durations (value, perc, both, default: both)'
+        puts '  --sort'.colorize(:green)
+        puts '     count,fail,max,median,min,p95,p99,rps,score'
+        puts '  --limit'.colorize(:green)
+        puts '     The number of rows to print'
+        puts '  --verbose'.colorize(:green)
+        puts '     Prints the component details of the maximum, P99, P95, and median events by total duration for each'
+        puts
+
+        puts 'Commands'.colorize(:blue)
+        puts 'ls'.colorize(:green)
+        puts '  List available files'
+        puts '  Options'
+        puts '    -a, --all, show all files including source'
+        puts
+
+        puts '<file names+>'.colorize(:green)
+        puts '  Print any file names'
+        puts '  Ex: `gitaly/current`'
+        puts '  Ex: `gitlab-rails/api_json.log gitlab-rails/production_json.log --raw`'
+        puts
+
+        puts 'top'.colorize(:green)
+        puts '  Print a summary of top items in a category'
+        puts
+
+        puts 'errors'.colorize(:green)
+        puts '  Show summary of errors captured in log'
+      end
+      # rubocop:enable Metrics/MethodLength)
+
+      # List Files Helpers
+      def self.list(args = [])
+        all = false
+        all = true if args.include?('-a') || args.include?('--all')
+
+        files = ShellHelper::Faststats.things
+
+        # Sort
+        files.sort_by!(&:name)
+
+        # Short & Uniq
+        files.uniq!(&:name) unless all
+
+        # Print
+        files.each do |log|
+          if all
+            puts "- #{log.friendly_name}"
+          else
+            puts "- #{log.name.colorize(:yellow)}"
+          end
+        end
+      end
+
+      def self.ls(args = [])
+        list(args)
+      end
+
+      # Vanilla Fast Stats
+      def self.default(file_list, _other = nil)
+        # Extract Args
+        file_list, opts, args = ShellHelper.param_parse(file_list)
+
+        cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
+        cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
+
+        # Prepare List
+        file_list = ShellHelper.prepare_list(file_list, ShellHelper::Faststats.things)
+
+        # Convert to Things
+        files = ShellHelper.find_things(file_list)
+
+        results = ShellHelper.file_process(files) do |file|
+          [
+            file.friendly_name,
+            `fast-stats #{cmd} #{file.file}`.split("\n"),
+            "\n"
+          ]
+        end
+
+        ShellHelper.show(results.flatten, args)
+      end
+
+      # ========================================================================
+      # ===== [ Fast Stats - Top ] ====================
+      # -------------------------------------------------
+      # Options:
+      # --display, value,perc,both
+      #   Show percentage of totals, actual durations, or both. Defaults to both.
+      # --limit=X
+      # --sort=count,fail,max,median,min,p95,p99,rps,score
+      # ========================================================================
+      def self.top(log_list)
+        # Extract Args
+        log_list, opts, args = ShellHelper.param_parse(log_list)
+
+        # Default to color output
+        args['color-output'] ||= true
+
+        cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
+        cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
+
+        # Prepare Log List
+        log_list = ShellHelper.prepare_list(log_list, ShellHelper::Faststats.things)
+
+        # Convert to Things
+        files = ShellHelper.find_things(log_list)
+
+        results = ShellHelper.file_process(files) do |file|
+          [
+            file.friendly_name,
+            `fast-stats top #{cmd} #{file.file}`.split("\n"),
+            "\n"
+          ]
+        end
+
+        ShellHelper.show(results.flatten, args)
+      end
+      # ========================================================================
+
+      # ========================================================================
+      # ===== [ Fast Stats - Errors ] ====================
+      # ========================================================================
+      def self.errors(log_list)
+        # Extract Args
+        log_list, opts, args = ShellHelper.param_parse(log_list)
+
+        # Default to color output
+        args['color-output'] ||= true
+
+        cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
+        cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
+
+        # Prepare Log List
+        log_list = ShellHelper.prepare_list(log_list, ShellHelper::Faststats.things)
+
+        # Convert to Things
+        files = ShellHelper.find_things(log_list)
+
+        results = ShellHelper.file_process(files) do |file|
+          [
+            file.friendly_name,
+            `fast-stats errors #{cmd} #{file.file}`.split("\n"),
+            "\n"
+          ]
+        end
+
+        ShellHelper.show(results.flatten, args)
+      end
+      # ========================================================================
+    end
+  end
+end
+
+module GreenHat
+  module ShellHelper
+    # Hide from Commands
+    module Faststats
+      def self.files
+        %w[
+          production_json
+          api_json
+          gitaly/current
+          sidekiq/current
+        ]
+      end
+
+      def self.things
+        Thing.all.select do |thing|
+          files.any? { |x| thing.name.include? x }
+        end
+      end
+    end
+  end
+end

data/lib/greenhat/shell/gitlab.rb

@@ -0,0 +1,45 @@
+module GreenHat
+  # CLI Helper
+  module Shell
+    # Logs
+    module Gitlab
+      def self.version
+        Thing.where(type: 'gitlab/version-manifest.json').each do |file|
+          next unless file.data
+
+          puts file.friendly_name
+          puts Semantic::Version.new(file.data.build_version).to_s.colorize(:yellow)
+          puts
+        end
+      end
+
+      def self.status
+        Thing.where(type: 'gitlab_status').each do |file|
+          next unless file.data
+
+          puts file.friendly_name
+
+          file.data.each_value do |services|
+            output = services.each_with_index.flat_map do |service, i|
+              color = service.status == 'run' ? :green : :red
+
+              pad = i.zero? ? 18 : 0
+
+              [
+                "#{service.status}:",
+                service.name.ljust(pad).colorize(color),
+                "#{service.pid_uptime};".ljust(pad)
+
+              ]
+            end
+
+            puts output.join(' ')
+          end
+
+          puts
+        end
+      end
+    end
+    # -----
+  end
+end

data/lib/greenhat/shell/help.rb

@@ -0,0 +1,15 @@
+# module GreenHat
+#   module Shell
+#     # CLI Helper
+#     module Help
+#       def self.help
+#         puts 'Available Commands: '
+#         puts '=> help'
+
+#         ShellCommand.descendants_s.each do |item|
+#           puts "-> #{item.colorize(:yellow)}"
+#         end
+#       end
+#     end
+#   end
+# end

data/lib/greenhat/shell/helper.rb

@@ -0,0 +1,514 @@
+module GreenHat
+  # Common Helpers
+  # rubocop:disable Metrics/ModuleLength
+  module ShellHelper
+    # Generic Parameter Parsing
+    def self.param_parse(params)
+      # Turn Params into Hash
+      opts = params.flat_map { |param| param_opt_scan(param) }
+
+      # Collect and Remove Args
+      args = params.each_with_object({}) { |param, obj| param_arg_scan(param, obj) }
+
+      # Move Special Arguments
+      opts.reject! do |opt|
+        if param_special_opts.include? opt.field
+          args[opt.field] = opt.value.split(',').map(&:to_sym)
+          true
+        end
+      end
+
+      # Remove Opts and Args
+      params.reject! do |param|
+        opt_field_remove?(opts, param) || arg_field_remove?(args, param)
+      end
+
+      [params, opts, args]
+    end
+
+    def self.opt_field_remove?(opts, param)
+      opts.any? do |opt|
+        param.include? "--#{opt.field}#{opt.bang ? '!' : nil}=#{opt.value}"
+      end
+    end
+
+    def self.arg_field_remove?(args, param)
+      args.keys.any? { |field| param.include? "--#{field}" }
+    end
+
+    # Def Param Scan (Split -- values into keys)
+    def self.param_opt_scan(param)
+      param.scan(/--([^=]+)=(.*)/).map do |field, value|
+        bang = false
+        if field.include? '!'
+          field.delete!('!')
+          bang = true
+        end
+
+        { field: field.to_sym, value: value, bang: bang }
+      end
+    end
+
+    # Params that should be set as Args
+    # Comma Delimited
+    # --slice=thing,thing2
+    # --except=time,params
+    # --round=2
+
+    def self.param_special_opts
+      %i[slice except stats uniq pluck round archive start end sort]
+    end
+
+    # Parameter Extraction
+    # Special Opts: --slice, --except
+    def self.param_arg_scan(param, obj)
+      # TODO: Why is capture group doing two arrays
+      param.scan(/--([^=]+)$/).flatten.each do |field|
+        obj[field.to_sym] = param_arg_defaults(field.to_sym)
+      end
+
+      obj
+    end
+
+    # Arg Defaults
+    def self.param_arg_defaults(field)
+      case field
+      when *param_special_opts then []
+      when :round then 2
+      else
+        true
+      end
+    end
+
+    # Use File Process for Output
+    def self.file_output(files)
+      results = file_process(files) do |file|
+        [
+          file.friendly_name,
+          file.output(false),
+          "\n"
+        ]
+      end
+
+      ShellHelper.show(results.flatten)
+    end
+
+    def self.file_process(files, &block)
+      files.map do |file|
+        next if file.output(false).empty?
+
+        block.call(file)
+      end.flatten
+    end
+
+    # Pagination Helper
+    def self.page(data)
+      TTY::Pager.page do |pager|
+        data.flatten.each do |output|
+          pager.write("\n#{output}") # write line to the pager
+        end
+      end
+    end
+
+    # Show Data / Auto Paginate Helper
+    def self.show(data, args = {})
+      # If Block of String
+      if data.instance_of?(String)
+        TTY::Pager.page data
+        return true
+      end
+
+      # If raw just print out
+      if args.raw
+        puts data.join("\n")
+        return true
+      end
+
+      # Check if content needs to paged, or if auto_height is off
+      if !args.page && (data.sum(&:size) < TTY::Screen.height)
+        puts data.map { |x| entry_show(x, args) }.compact.join("\n")
+        return true
+      end
+
+      # Default Pager
+      TTY::Pager.page do |pager|
+        data.each do |entry|
+          output = entry_show(entry, args)
+
+          next if output.blank?
+
+          pager.write("\n#{output}") # write line to the pager
+        end
+      end
+    end
+
+    # Entry Shower
+    def self.entry_show(entry, args)
+      case entry
+      when Hash then render_table(entry, args)
+      when String, Float, Integer then entry
+      when Array
+        entry.map { |x| x.ai(multiline: false) }.join("\n")
+      else
+        LogBot.warn('Shell Show', "Unknown #{entry.class}")
+        nil
+      end
+    end
+
+    # Print the Table in a Nice way
+    def self.render_table(entry, args)
+      table = TTY::Table.new(header: entry.keys, rows: [entry], orientation: :vertical)
+
+      table.render(:unicode, padding: [0, 1, 0, 1]) do |renderer|
+        renderer.border.style = :cyan
+
+        renderer.filter = lambda do |val, _row_index, col_index|
+          if col_index == 1
+            if val.numeric?
+              # TODO:  Better Casting?
+              val = val.to_f.round(args.round.first.to_s.to_i) if args.round
+              val.to_s.colorize(:blue)
+            else
+              val.to_s
+            end
+          else
+            val.to_s
+          end
+        end
+      end
+
+      # Fall Back to Amazing Inspect
+    rescue StandardError => e
+      LogBot.warn('Table', message: e.message, backtrace: e.backtrace.first) if ENV['DEBUG']
+
+      [
+        entry.ai,
+        ('_' * (TTY::Screen.width / 3)).colorize(:cyan),
+        "\n"
+      ].join("\n")
+    end
+
+    # Main Entry Point for Filtering
+    def self.filter_start(log_list, filter_type, args, opts)
+      # Convert to Things
+      logs = ShellHelper.find_things(log_list, args).select(&:processed?)
+
+      # Ignore Archive/Host Dividers
+      if args[:combine]
+        results = logs.reject(&:blank?).map(&:data).flatten.compact
+        ShellHelper.filter(results, filter_type, args, opts)
+      else
+        # Iterate and Preserve Archive/Host Index
+        logs.each_with_object({}) do |log, obj|
+          # Ignore Empty Results / No Thing
+          next if log&.blank?
+
+          obj[log.friendly_name] = ShellHelper.filter(log.data, filter_type, args, opts)
+
+          obj
+        end
+      end
+    end
+
+    # Filter Logic
+    # TODO: Simplify
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def self.filter(data, type = :all?, args = {}, opts = {})
+      results = data.clone.flatten.compact
+      results.select! do |row|
+        opts.send(type) do |opt|
+          filter_row_key(row, opt)
+        end
+      end
+
+      # Time Filtering
+      results = filter_time(results, args) if args[:start] || args[:end]
+
+      # Strip Results if Slice is defined
+      results = filter_slice(results, args[:slice]) if args[:slice]
+
+      # Strip Results if Except is defined
+      results = filter_except(results, args[:except]) if args[:except]
+
+      # Remove Blank from either slice or except
+      results.reject!(&:empty?)
+
+      # Sort
+      results.sort_by! { |x| x.slice(*args[:sort]).values } if args[:sort]
+
+      # JSON Formatting
+      results = results.map { |x| Oj.dump(x) } if args[:json]
+
+      # Show Unique Only
+      results = filter_uniq(results, args[:uniq]) if args.key?(:uniq)
+
+      # Reverse
+      results.reverse! if args[:reverse]
+
+      # Count occurrences / Skip Results
+      return filter_stats(results, args[:stats]) if args[:stats]
+
+      # Pluck
+      results = filter_pluck(results, args[:pluck]) if args.key?(:pluck)
+
+      results
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+    # Filter Start and End Times
+    # rubocop:disable Metrics/MethodLength
+    # TODO: This is a bit icky, simplify/dry
+    def self.filter_time(results, args)
+      if args.start
+        begin
+          time_start = Time.parse(args.start.first.to_s)
+
+          results.select! do |x|
+            if x.time
+              time_start < x.time
+            else
+              true
+            end
+          end
+        rescue StandardError
+          puts 'Unable to Process Start Time Filter'.colorize(:red)
+        end
+      end
+
+      if args.end
+        begin
+          time_start = Time.parse(args.end.first.to_s)
+
+          results.select! do |x|
+            if x.time
+              time_start > x.time
+            else
+              true
+            end
+          end
+        rescue StandardError
+          puts 'Unable to Process End Time Filter'.colorize(:red)
+        end
+      end
+
+      results
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def self.filter_except(results, except)
+      # Avoid Empty Results
+      if slice.empty?
+        filter_empty_arg('slice')
+        return results
+      end
+
+      results.map { |row| row.except(*except) }
+    end
+
+    def self.filter_slice(results, slice)
+      # Avoid Empty Results
+      if slice.empty?
+        filter_empty_arg('slice')
+        return results
+      end
+
+      results.map { |row| row.slice(*slice) }
+    end
+
+    def self.filter_pluck(results, pluck)
+      # Avoid Empty Results
+      if pluck.empty?
+        filter_empty_arg('pluck')
+        return results
+      end
+
+      results.map { |x| x.slice(*pluck).values }.flatten
+    end
+
+    def self.filter_uniq(results, unique)
+      # Avoid Empty Results
+      if unique.empty?
+        filter_empty_arg('uniq')
+        return results
+      end
+
+      unique.map do |field|
+        results.uniq { |x| x[field] }
+      end.flatten
+    end
+
+    def self.filter_stats(results, stats)
+      # Avoid Empty Results
+      if stats.empty?
+        filter_empty_arg('stats')
+        return results
+      end
+
+      stats.map do |stat|
+        occurrences = filter_count_occurrences(results, stat)
+        occurrences.sort_by(&:last).to_h
+      end
+    end
+
+    # Helper to Count occurrences
+    def self.filter_count_occurrences(results, stat)
+      results.each_with_object(Hash.new(0)) do |entry, counts|
+        next unless entry.key? stat
+
+        counts[entry[stat]] += 1
+
+        counts
+      end
+    end
+
+    def self.filter_empty_arg(arg)
+      puts [
+        'Ignoring'.colorize(:light_yellow),
+        "--#{arg}".colorize(:cyan),
+        'it requires an argument'.colorize(:red)
+      ].join(' ')
+    end
+
+    # Break out filter row logic into separate method
+    def self.filter_row_key(row, param)
+      # Ignore Other Logic if Field isn't even included
+      return false unless row.key? param.field
+
+      # Not Included Param
+      included = row[param.field].to_s.include? param.value.to_s
+
+      if param.bang
+        !included
+      else
+        included
+      end
+    end
+
+    # Total Count Helper
+    def self.total_count(results)
+      results.each do |k, v|
+        puts k
+        puts "Total: #{v.count.to_s.colorize(:blue)}"
+        puts
+      end
+    end
+
+    # Total Log List Manipulator
+    def self.prepare_list(log_list, base_list = nil)
+      base_list ||= GreenHat::ShellHelper::Log.list
+
+      # Assume all
+      log_list.push '*' if log_list.empty?
+
+      # Map for All
+      log_list = base_list.map(&:name) if log_list == ['*']
+
+      log_list
+    end
+
+    # Shortcut find things
+    def self.find_things(log_list, args = {})
+      things = log_list.uniq.flat_map do |log|
+        Thing.where name: log
+      end
+
+      # Host / Archive
+      things.select! { |x| x.archive? args.archive } if args.archive
+
+      things
+    end
+
+    # Main Entry Point for Searching
+    def self.search_start(log_list, filter_type, args, opts)
+      # Convert to Things
+      logs = ShellHelper.find_things(log_list, args)
+
+      logs.each_with_object({}) do |log, obj|
+        # Ignore Empty Results / No Thing
+        next if log&.data.blank?
+
+        obj[log.friendly_name] = ShellHelper.search(log.data, filter_type, args, opts)
+
+        obj
+      end
+    end
+
+    # Generic Search Helper / String/Regex
+    def self.search(data, type = :all?, args = {}, opts = {})
+      results = data.clone.flatten.compact
+
+      results.select! do |row|
+        opts.send(type) do |opt|
+          search_row(row, opt)
+        end
+      end
+
+      # Strip Results if Slice is defined
+      results.map! { |row| row.slice(*args[:slice]) } if args[:slice]
+
+      # Strip Results if Except is defined
+      results.map! { |row| row.except(*args[:except]) } if args[:except]
+
+      # Remove Blank from either slice or except
+      results.reject!(&:empty?)
+
+      results
+    end
+
+    # Break out filter row logic into separate method
+    def self.search_row(row, param)
+      # Not Included Param
+      included = row.to_s.include? param.value
+
+      if param.bang
+        !included
+      else
+        included
+      end
+    end
+
+    # Color Reader Helper
+    def self.pastel
+      @pastel ||= Pastel.new
+    end
+
+    # Number Helper
+    # https://gitlab.com/zedtux/human_size_to_number/-/blob/master/lib/human_size_to_number/helper.rb
+    def self.human_size_to_number(string)
+      size, unit = string.scan(/(\d*\.?\d+)\s?(Bytes?|KB|MB|GB|TB)/i).first
+      number = size.to_f
+
+      number = case unit.downcase
+               when 'byte', 'bytes'
+                 number
+               when 'kb'
+                 number * 1024
+               when 'mb'
+                 number * 1024 * 1024
+               when 'gb'
+                 number * 1024 * 1024 * 1024
+               when 'tb'
+                 number * 1024 * 1024 * 1024 * 1024
+               end
+      number.round
+    end
+
+    # TODO: Needed?
+    def self.filter_and(data, params = {})
+      result = data.clone.flatten.compact
+      params.each do |k, v|
+        result.select! do |row|
+          if row.key? k.to_sym
+            row[k.to_sym].include? v
+          else
+            false
+          end
+        end
+        next
+      end
+
+      result
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end