greenhat 0.1.5 → 0.3.2

data/lib/greenhat/shell/faststats.rb

@@ -5,52 +5,82 @@ module GreenHat
     module Faststats
       # rubocop:disable Metrics/MethodLength)
       def self.help
-        puts "\u2500".colorize(:cyan) * 25
-        puts "Gimme #{'Performance Stats'.colorize(:yellow)}"
-        puts "\u2500".colorize(:cyan) * 25
+        puts "\u2500".pastel(:cyan) * 25
+        puts "Gimme #{'Performance Stats'.pastel(:yellow)}"
+        puts "\u2500".pastel(:cyan) * 25
 
-        puts 'General'.colorize(:blue)
-        puts "  Any double dash arguments (e.g. #{'--color-output'.colorize(:green)}) are passed directly to fast-stats"
-        puts "  See #{'`fast-stats --help`'.colorize(:light_black)} for all available options"
+        puts 'General'.pastel(:blue)
+        puts "  Any double dash arguments (e.g. #{'--color-output'.pastel(:green)}) are passed directly to fast-stats"
+        puts "  See #{'`fast-stats --help`'.pastel(:bright_black)} for all available options"
         puts
 
-        puts 'Common Options'.colorize(:blue)
-        puts '  --raw'.colorize(:green)
+        puts 'Common Options'.pastel(:blue)
+        puts '  --raw'.pastel(:green)
         puts '    Do not use less/paging'
-        puts '  --display'.colorize(:green)
-        puts '     Show percentage of totals and/or actual durations (value, perc, both, default: both)'
-        puts '  --sort'.colorize(:green)
+        puts '  --search'.pastel(:green)
+        puts '     Case-insensitive search of controller/method/worker field'
+        puts '  --sort'.pastel(:green)
         puts '     count,fail,max,median,min,p95,p99,rps,score'
-        puts '  --limit'.colorize(:green)
+        puts '  --limit'.pastel(:green)
         puts '     The number of rows to print'
-        puts '  --verbose'.colorize(:green)
+        puts '  --verbose'.pastel(:green)
         puts '     Prints the component details of the maximum, P99, P95, and median events by total duration for each'
         puts
 
-        puts 'Commands'.colorize(:blue)
-        puts 'ls'.colorize(:green)
+        puts 'Commands'.pastel(:blue)
+        puts 'ls'.pastel(:green)
         puts '  List available files'
         puts '  Options'
         puts '    -a, --all, show all files including source'
         puts
 
-        puts '<file names+>'.colorize(:green)
+        puts '<file names+>'.pastel(:green)
         puts '  Print any file names'
         puts '  Ex: `gitaly/current`'
         puts '  Ex: `gitlab-rails/api_json.log gitlab-rails/production_json.log --raw`'
         puts
 
-        puts 'top'.colorize(:green)
+        puts 'top'.pastel(:green)
         puts '  Print a summary of top items in a category'
         puts
 
-        puts 'errors'.colorize(:green)
+        puts 'errors'.pastel(:green)
         puts '  Show summary of errors captured in log'
+        puts
+
+        puts 'Examples'.pastel(:blue)
+        puts '  Default'.pastel(:bright_white)
+        puts '    gitaly/current'
+        puts '    --raw gitlab-rails/production_json.log'
+        puts '    gitlab-rails/api_json.log --sort=count'
+        puts '    --search=pipeline sidekiq/current'
+        puts
+
+        puts '  Top'.pastel(:bright_white)
+        puts '    top gitaly/current'
+        puts '    top --limit=5 sidekiq/current'
+        puts
+
+        puts '  Errors'.pastel(:bright_white)
+        puts '    errors gitaly/current'
+        puts '    errors gitaly/current --no-border'
       end
       # rubocop:enable Metrics/MethodLength)
 
+      def self.faststats_installation
+        puts "#{'Unable to find'.pastel(:red)} #{'fast-stats'.pastel(:blue)}"
+        puts '  Release Downloads here'
+        puts '    - https://gitlab.com/gitlab-com/support/toolbox/fast-stats/-/releases'.pastel(:yellow)
+        puts ''
+      end
+
       # List Files Helpers
       def self.list(args = [])
+        unless TTY::Which.exist? 'fast-stats'
+          faststats_installation
+          return false
+        end
+
         all = false
         all = true if args.include?('-a') || args.include?('--all')
 
@@ -67,7 +97,7 @@ module GreenHat
           if all
             puts "- #{log.friendly_name}"
           else
-            puts "- #{log.name.colorize(:yellow)}"
+            puts "- #{log.name.pastel(:yellow)}"
           end
         end
       end
@@ -77,18 +107,15 @@ module GreenHat
       end
 
       # Vanilla Fast Stats
-      def self.default(file_list, _other = nil)
-        # Extract Args
-        file_list, opts, args = ShellHelper.param_parse(file_list)
+      def self.default(raw, _other = nil)
+        unless TTY::Which.exist? 'fast-stats'
+          faststats_installation
+          return false
+        end
 
-        cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
-        cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
+        files, flags, cmd = ShellHelper::Faststats.parse(raw)
 
-        # Prepare List
-        file_list = ShellHelper.prepare_list(file_list, ShellHelper::Faststats.things)
-
-        # Convert to Things
-        files = ShellHelper.find_things(file_list)
+        LogBot.debug('FastStats CMD', cmd) if ENV['DEBUG']
 
         results = ShellHelper.file_process(files) do |file|
           [
@@ -98,7 +125,7 @@ module GreenHat
           ]
         end
 
-        ShellHelper.show(results.flatten, args)
+        ShellHelper.show(results.flatten, flags)
       end
 
       # ========================================================================
@@ -110,21 +137,15 @@ module GreenHat
       # --limit=X
       # --sort=count,fail,max,median,min,p95,p99,rps,score
       # ========================================================================
-      def self.top(log_list)
-        # Extract Args
-        log_list, opts, args = ShellHelper.param_parse(log_list)
-
-        # Default to color output
-        args['color-output'] ||= true
-
-        cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
-        cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
+      def self.top(raw)
+        unless TTY::Which.exist? 'fast-stats'
+          faststats_installation
+          return false
+        end
 
-        # Prepare Log List
-        log_list = ShellHelper.prepare_list(log_list, ShellHelper::Faststats.things)
+        files, flags, cmd = ShellHelper::Faststats.parse(raw)
 
-        # Convert to Things
-        files = ShellHelper.find_things(log_list)
+        LogBot.debug('FastStats CMD', cmd) if ENV['DEBUG']
 
         results = ShellHelper.file_process(files) do |file|
           [
@@ -134,28 +155,24 @@ module GreenHat
           ]
         end
 
-        ShellHelper.show(results.flatten, args)
+        ShellHelper.show(results.flatten, flags)
       end
       # ========================================================================
 
       # ========================================================================
       # ===== [ Fast Stats - Errors ] ====================
       # ========================================================================
-      def self.errors(log_list)
-        # Extract Args
-        log_list, opts, args = ShellHelper.param_parse(log_list)
-
-        # Default to color output
-        args['color-output'] ||= true
+      def self.errors(raw)
+        unless TTY::Which.exist? 'fast-stats'
+          faststats_installation
+          return false
+        end
 
-        cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
-        cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
+        # Add Color Output
+        raw.push '--color-output'
+        files, flags, cmd = ShellHelper::Faststats.parse(raw)
 
-        # Prepare Log List
-        log_list = ShellHelper.prepare_list(log_list, ShellHelper::Faststats.things)
-
-        # Convert to Things
-        files = ShellHelper.find_things(log_list)
+        LogBot.debug('FastStats CMD', cmd) if ENV['DEBUG']
 
         results = ShellHelper.file_process(files) do |file|
           [
@@ -165,7 +182,7 @@ module GreenHat
           ]
         end
 
-        ShellHelper.show(results.flatten, args)
+        ShellHelper.show(results.flatten, flags)
       end
       # ========================================================================
     end
@@ -176,6 +193,35 @@ module GreenHat
   module ShellHelper
     # Hide from Commands
     module Faststats
+      # Default Settings from arg parse that won't work
+      def self.invalid_settings
+        %i[page round truncate logic fuzzy_file_match]
+      end
+
+      def self.parse(raw)
+        file_list, flags, args = Args.parse(raw, invalid_settings)
+        cmd = args.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
+        cmd += flags.map do |flag, value|
+          # Don't Include Raw
+          next if flag == :raw
+
+          case value
+          when true then "--#{flag}"
+          when Array then "--#{flag}=#{value.join}"
+          else
+            "--#{flag}=#{value}"
+          end
+        end.join(' ')
+
+        # Prepare Log List
+        file_list = ShellHelper.prepare_list(file_list, ShellHelper::Faststats.things, flags)
+
+        # Convert to Things
+        files = ShellHelper.find_things(file_list)
+
+        [files, flags, cmd]
+      end
+
       def self.files
         %w[
           production_json

data/lib/greenhat/shell/field_helper.rb

@@ -0,0 +1,75 @@
+module GreenHat
+  # Common Helpers
+  module FieldHelper
+    def self.fields_find(files, word, flags = {})
+      fields = ShellHelper.find_things(files, flags).map(&:fields).flatten.uniq
+
+      if word.blank?
+        puts 'Possible Fields:'.pastel(:bright_blue)
+        puts ShellHelper.field_table(fields)
+
+        return [] # Empty Result
+      end
+
+      list_select(fields, word)
+    end
+
+    def self.list_select(list, word)
+      list.select! { |x| x[/^#{Regexp.escape(word)}/] }
+    end
+
+    def self.filter_flags(word)
+      if word.blank?
+        puts 'Filter Options:'.pastel(:bright_blue)
+        puts ShellHelper.field_table(filter_opts, 6)
+        puts
+
+        return []
+      end
+
+      list_select(filter_opts, word)
+    end
+
+    def self.filter_opts
+      %w[
+        archive case combine end exact except exists json limit or page pluck
+        raw reverse round slice sort start stats table_style text time_zone
+        total truncate uniq
+      ]
+    end
+
+    def self.filter_auto_completes
+      %w[
+        except exists pluck slice sort stats uniq
+      ]
+    end
+
+    def self.field_auto_complete?(word)
+      return false if word.blank?
+
+      filter_auto_completes.include? word.split('=', 2).first
+    end
+
+    def self.field_auto_complete(word, files, flags = {})
+      # Prevent weird dupes
+      return nil if word[-1] == ','
+
+      # Command Manipulation
+      cmd, fields = word.split('=', 2)
+      complete = fields.split(',')[0..-2]
+      auto = fields.split(',').last
+
+      # Field Finder
+      matches = fields_find(files, auto, flags)
+
+      if matches.count == 1
+        "--#{cmd}=#{(complete + matches).join(',')}"
+      elsif matches.count > 1
+        puts "#{'Field Options:'.pastel(:bright_blue)} #{matches.join(' ').pastel(:bright_green)}"
+
+        list = [Cli.common_substr(matches.map(&:to_s))]
+        "--#{cmd}=#{(complete + list).join(',')}"
+      end
+    end
+  end
+end

data/lib/greenhat/shell/filter_help.rb

@@ -0,0 +1,162 @@
+module GreenHat
+  # CLI Helper
+  module ShellHelper
+    # Unify Filter / Filter Help
+    module Filter
+      # rubocop:disable Metrics/MethodLength
+      def self.help
+        puts "\u2500".pastel(:cyan) * 20
+        puts 'Filter'.pastel(:yellow)
+        puts "\u2500".pastel(:cyan) * 20
+
+        puts 'Options'.pastel(:blue)
+        puts '--raw'.pastel(:green)
+        puts '  Disable formatting and page/less'
+        puts
+
+        puts '--page'.pastel(:green)
+        puts '  Specifically enable or disable paging'
+        puts '  E.g. --page (default to true), --page=true, --page=false'
+        puts
+
+        puts '--round'.pastel(:green)
+        puts '  Attempt to round all integers. Default: 2.'
+        puts '  E.g. --round, --round=3, --round=0'
+        puts
+
+        puts '--limit'.pastel(:green)
+        puts '  Limit total output lines. Disabled by default. Default without value is based on screen height'
+        puts '  E.g. --limit, --limit=5'
+        puts
+
+        puts '--json'.pastel(:green)
+        puts '  Print output back into JSON'
+        puts
+
+        puts '--or'.pastel(:green)
+        puts '  Filters will use OR instead of AND (all match vs any match)'
+        puts
+
+        puts '--total'.pastel(:green)
+        puts '  Print only total count of matching entries'
+        puts
+
+        puts '--fields'.pastel(:green)
+        puts '  Print only Available fields for selected files'
+        puts
+
+        puts '--slice'.pastel(:green)
+        puts '  Extract specific fields from entries (slice multiple with comma)'
+        puts '  Ex: --slice=path or --slice=path,params'
+        puts
+
+        puts '--except'.pastel(:green)
+        puts '  Exclude specific fields (except multiple with comma)'
+        puts '  Ex: --except=params --except=params,path'
+        puts
+
+        puts '--exists'.pastel(:green)
+        puts '  Ensure field exists regardless of contents'
+        puts '  Ex: --exists=params --exists=params,path'
+        puts
+
+        puts '--stats'.pastel(:green)
+        puts '  Order/Count occurrances by field. Combine with `truncate` for key names'
+        puts '  Ex: --stats=params --except=params,path'
+        puts
+
+        puts '--uniq'.pastel(:green)
+        puts '  Show unique values only'
+        puts '  Ex: --uniq=params --uniq=params,path'
+        puts
+
+        puts '--pluck'.pastel(:green)
+        puts '  Extract values from entries'
+        puts '  Ex: --pluck=params --pluck=params,path'
+        puts
+
+        puts '--archive'.pastel(:green)
+        puts '  Limit to specific archvie name (partial matching /inclusive). Matching SOS tar.gz name'
+        puts '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
+        puts
+
+        puts '--sort'.pastel(:green)
+        puts '  Sort by multiple fields'
+        puts '  Ex: --sort=duration_s,db_duration_s'
+        puts
+
+        puts '--reverse'.pastel(:green)
+        puts '  Reverse all results'
+        puts '  Ex: --reverse'
+        puts
+
+        puts '--combine'.pastel(:green)
+        puts '  Omit archive identifier dividers. Useful with sort or time filters'
+        puts '  Ex: --combine'
+        puts
+
+        puts '--case'.pastel(:green)
+        puts '  Exact case match. Defaults to case insensitive'
+        puts '  Ex: --case; --name=Jon, --name=jane --case'
+        puts
+
+        puts '--exact'.pastel(:green)
+        puts '  Exact parameter/value match. Defaults to partial match'
+        puts '  Ex: --field=CommonPartial --exact'
+        puts
+
+        puts '--start'.pastel(:green)
+        puts '  Show events after specified time. Filtered by the `time` field'
+        puts '  Use with `--end` for between selections'
+        puts '  Ex: log filter --start="2021-06-22 14:44 UTC" --end="2021-06-22 14:45 UTC"'
+        puts
+
+        puts '--end'.pastel(:green)
+        puts '  Show events before specified time. Filtered by the `time` field'
+        puts '  Use with `--start` for between selections'
+        puts '  Ex: log filter --end="2021-06-22"'
+        puts
+
+        puts '--time_zone'.pastel(:green)
+        puts '  Manipulate the `time` field into a specific timezone'
+        puts '  Ex: log filter --time_zone=EDT'
+        puts
+
+        puts '--text'.pastel(:green)
+        puts '  Full entry text searching (slow)'
+        puts '    --text="anything here"'
+        puts
+
+        puts '--table_style'.pastel(:green)
+        puts '  Renderer used for formatted output. basic, ascii, or unicode(default)'
+        puts '  Ex: log filter --table_style=base'
+        puts
+
+        puts '--truncate'.pastel(:green)
+        puts '  Truncate field length. On by default (4 rows). Performance issues!'
+        puts '  Disable with --truncate=0'.pastel(:bright_red)
+        puts '  Ex: --truncate=200, --truncate=2048"'
+        puts
+
+        puts 'Field Searching'.pastel(:blue)
+        puts '  --[key]=[value]'
+        puts '  Search in key for value'
+        puts '  Example: --path=mirror/pull'
+        puts
+
+        puts 'Search specific logs'.pastel(:blue)
+        puts '  Any non dash parameters will be the log list to search from'
+        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.pastel(:yellow)}` for log names"
+        puts
+
+        puts 'Example Queries'.pastel(:blue)
+        puts "  Also see #{'examples'.pastel(:bright_blue)} for even more examples"
+        puts '  log filter --class=BuildFinishedWorker sidekiq/current --slice=time,message'
+        puts '  log filter gitlab-rails/api_json.log --slice=ua --uniq=ua --ua=gitlab-runner'
+
+        puts
+      end
+      # rubocop:enable Metrics/MethodLength
+    end
+  end
+end

data/lib/greenhat/shell/gitlab.rb

@@ -3,16 +3,75 @@ module GreenHat
   module Shell
     # Logs
     module Gitlab
+      def self.ls
+        help
+      end
+
+      def self.help
+        puts "\u2500".pastel(:cyan) * 20
+        puts "#{'GitLab'.pastel(:yellow)} - Tanuki power"
+        puts "\u2500".pastel(:cyan) * 20
+
+        puts 'Command Summary'.pastel(:blue)
+
+        puts '  services'.pastel(:green)
+        puts '    Show services status and version'
+        puts
+
+        puts '  status'.pastel(:green)
+        puts '    Show gitlab-ctl status'
+        puts
+
+        puts '  architecture'.pastel(:green)
+        puts '    Show node responsibility. E.g. Webservice, Gitaly, and etc'
+        puts
+
+        puts '  version'.pastel(:green)
+        puts '    Show GitLab software version'
+        puts
+      end
+
+      def self.architecture
+        results = Archive.all.map { |x| GitLab.identify_node(x) }
+
+        GitLab.node_types.each do |entry|
+          list = results.select { |x| (x.services & entry.pattern).any? }.map(&:host).join(', ')
+          next if list.blank?
+
+          puts entry.name.pastel(:bright_green)
+          puts list
+          puts
+        end
+      end
+
       def self.version
         Thing.where(type: 'gitlab/version-manifest.json').each do |file|
           next unless file.data
 
           puts file.friendly_name
-          puts Semantic::Version.new(file.data.build_version).to_s.colorize(:yellow)
+          puts Semantic::Version.new(file.data.build_version).to_s.pastel(:yellow)
           puts
         end
       end
 
+      # Print service info / shared report method
+      def self.services(raw = {})
+        _files, flags, _args = Args.parse(raw, [:truncate])
+
+        results = {}
+
+        Archive.all.each do |archive|
+          result = GitLab.services(archive)
+          next unless result
+
+          # Generated Output
+          results[archive.friendly_name.pastel(:blue)] = result
+        end
+
+        # Print
+        ShellHelper.show(results, flags)
+      end
+
       def self.status
         Thing.where(type: 'gitlab_status').each do |file|
           next unless file.data
@@ -27,7 +86,7 @@ module GreenHat
 
               [
                 "#{service.status}:",
-                service.name.ljust(pad).colorize(color),
+                service.name.ljust(pad).pastel(color),
                 "#{service.pid_uptime};".ljust(pad)
 
               ]