greenhat 0.2.0 → 0.3.3

data/lib/greenhat/shell/log.rb

@@ -3,172 +3,186 @@ module GreenHat
   module Shell
     # Logs
     module Log
+      def self.auto_complete(list, word)
+        # Argument Parsing
+        files, flags, _args = Args.parse(list)
+
+        # Don't try to autocomplete anything else
+        return nil unless word =~ /^-/
+
+        # Clean Up
+        word.delete!('-')
+        matches = FieldHelper.filter_flags(word)
+
+        if matches.count == 1
+          "--#{matches.first}"
+        elsif matches.count > 1
+          puts "#{'Filter Options:'.pastel(:bright_blue)} #{matches.join(' ').pastel(:bright_green)}"
+          "--#{Cli.common_substr(matches)}"
+        # -----------------------------------
+        # TODO: Fix Icky Double Nesting
+        elsif files.count.nonzero?
+          matches = FieldHelper.fields_find(files, word, flags)
+
+          return nil if matches.nil?
+
+          if matches.count == 1
+            "--#{matches.first}"
+          elsif matches.count > 1
+            puts "#{'Field Options:'.pastel(:bright_blue)} #{matches.join(' ').pastel(:bright_green)}"
+            "--#{Cli.common_substr(matches.map(&:to_s))}"
+          elsif FieldHelper.field_auto_complete?(word)
+            FieldHelper.field_auto_complete(word, files, flags)
+          end
+          # -----------------------------------
+        end
+      end
+
       def self.help
-        puts "\u2500".colorize(:cyan) * 20
-        puts "#{'Logs'.colorize(:yellow)} find stuff"
-        puts "\u2500".colorize(:cyan) * 20
+        puts "\u2500".pastel(:cyan) * 20
+        puts "#{'Logs'.pastel(:yellow)} find stuff"
+        puts "\u2500".pastel(:cyan) * 20
+
+        puts 'Command Summary'.pastel(:blue)
+        puts '  filter'.pastel(:green)
+        puts "    Primary way for log searching within greenhat. See #{'filter_help'.pastel(:blue)}"
+        puts '    Time, round, slice/except, and/or, stats, uniq, sort'
+        puts
 
-        puts 'Command Summary'.colorize(:blue)
-        puts '  show'.colorize(:green)
+        puts '  show'.pastel(:green)
         puts '    Just print selected logs'
-        puts '  filter'.colorize(:green)
-        puts '    Key/Field Filtering'
-        puts '    - See `filter_help`'
-        puts '  search'.colorize(:green)
-        puts '    General text by entry searching'
-        puts '    - See `search_help`'
-        puts '  ls'.colorize(:green)
-        puts '    List available files'
         puts
-      end
-
-      def self.filter_help
-        ShellHelper::Filter.help
-      end
-
-      # List Files Helpers
-      def self.list(args = [])
-        all = false
-        all = true if args.include?('-a') || args.include?('--all')
 
-        files = ShellHelper::Log.list
+        puts '  search'.pastel(:green)
+        puts "    General full text by file searching. See #{'search_help'.pastel(:blue)}"
+        puts
 
-        # Sort
-        files.sort_by!(&:name)
+        puts ShellHelper::List.help
 
-        # Short & Uniq
-        files.uniq!(&:name) unless all
+        puts "See #{'examples'.pastel(:bright_blue)} for query examples"
+      end
 
-        # Print
-        files.each do |log|
-          if all
-            puts "- #{log.friendly_name}"
-          else
-            puts "- #{log.name.colorize(:yellow)}"
-          end
-        end
+      def self.filter_help
+        ShellHelper::Filter.help
       end
 
       def self.ls(args = [])
-        list(args)
+        ShellHelper::List.list(args, ShellHelper::Log.list)
       end
 
-      def self.show(log_list)
-        # Prepare Log List
-        log_list = ShellHelper.prepare_list(log_list)
-
-        # Convert to Things
-        logs = ShellHelper.find_things(log_list)
+      def self.show(raw = {})
+        # Extract Args
+        files_list, flags, _args = Args.parse(raw)
 
-        logs.map!(&:data)
+        # Collect Files
+        files = ShellHelper.files(files_list, Thing.all, flags)
 
-        ShellHelper.show logs.flatten
+        ShellHelper.show files.map(&:data).flatten
       end
 
       # ========================================================================
-      # Filter
+      # Filter (See Filter Help)
       # ========================================================================
-      # Supported Params
-      # --or (Filter OR instead of AND)
-      # --total Total Count Entries Only
-      # --project=thingy --exclude_this!=asdf *
-      # --slice:  only grab specific fields --slice=path (slice multiple with comma)
-      # --slice=time,path (E.g. log filter  --path='mirror/pull' --slice=path,time )
-      # --except: Exclude specific fields (except multiple with comma)
-      # Example: log filter  --path='mirror/pull' --except=params
       def self.default(raw_list)
         filter(raw_list)
       end
 
-      def self.filter(raw_list)
+      def self.filter(raw)
         # Print Helper
-        if raw_list == ['help']
+        if raw == ['help']
           filter_help
           return true
         end
 
-        # Extract Args
-        log_list, opts, args = ShellHelper.param_parse(raw_list)
+        # Argument Parsing
+        files, flags, args = Args.parse(raw)
 
         # Prepare Log List
-        log_list = ShellHelper.prepare_list(log_list)
+        files = ShellHelper.prepare_list(files, ShellHelper::Log.list, flags)
 
-        # AND / OR Filtering
-        filter_type = args.or ? :any? : :all?
+        results = ShellHelper.filter_start(files, flags, args)
 
-        results = ShellHelper.filter_start(log_list, filter_type, args, opts)
-
-        # Skipo and Print Total if set
-        if args.total
+        # Skip and Print Total if set
+        if flags[:total]
           ShellHelper.total_count(results)
           return true
         end
 
+        # Skip and Print Total if set
+        if flags[:fields]
+          ShellHelper.fields_print(results)
+          return true
+        end
+
         # Check Search Results
         if results.instance_of?(Hash) && results.values.flatten.empty?
-          puts 'No results'.colorize(:red)
+          puts 'No results'.pastel(:red)
         else
           # This causes the key 'colorized' output to also be included
-          ShellHelper.show(results.to_a.compact.flatten, args)
+          ShellHelper.show(results.to_a.compact.flatten, flags)
         end
 
         # log filter --path='cloud/gitlab-automation' --path='/pull' --all
         # log filter --project=thingy --other_filter=asdf *
       rescue StandardError => e
-        LogBot.fatal('Filter', message: e.message, backtrace: e.backtrace.first)
+        LogBot.fatal('Filter', message: e.message)
+        ap e.backtrace
       end
       # ========================================================================
 
       # rubocop:disable Layout/LineLength
       # TODO: Add a lot more examples
       def self.examples
-        puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.colorize(:light_green)
+        puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.pastel(:bright_green)
         puts 'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'
         puts
-        puts 'Find 500s only show exceptions'.colorize(:light_green)
+
+        puts 'Find 500s only show exceptions'.pastel(:bright_green)
         puts 'log filter --status=500 --slice=exception.message gitlab-rails/production_json.log'
         puts
+
+        puts 'Show unique sidekiq queue namespaces. Exclude Specifics'.pastel(:bright_green)
+        puts 'filter sidekiq/current --slice=queue_namespace --uniq=queue_namespace --queue_namespace!=jira_connect --queue_namespace!=hashed_storage'
+        puts
+
+        puts 'Show user,ip from API logs where `meta.user` field is present '.pastel(:bright_green)
+        puts 'gitlab-rails/api_json.log --slice=meta.user,meta.remote_ip --exists=meta.user'
+        puts
+
+        puts 'Count/% occurences for both user and remote ip fields'.pastel(:bright_green)
+        puts 'gitlab-rails/api_json.log --stats=meta.user,meta.remote_ip --exists=meta.user'
+        puts
       end
 
       # rubocop:enable Layout/LineLength
       # ========================================================================
       # Search (Full Text / String Search)
       # ========================================================================
-      # Supported Params
-      # --text='asdf'
-      # --text!='asdf'
-      # --regex='' # TODO?
-      # --slice=time,path (E.g. log filter  --path='mirror/pull' --slice=path,time )
-      # --except: Exclude specific fields (except multiple with comma)
-
-      # --total Total Count Entries Only
-      def self.search(initial_param)
+      def self.search(raw)
         # Extract Args
-        log_list, opts, args = ShellHelper.param_parse(initial_param)
+        files_list, flags, args = Args.parse(raw)
 
         # Prepare Log List
-        log_list = ShellHelper.prepare_list(log_list)
-
-        # AND / OR Filtering
-        filter_type = args.or ? :any? : :all?
+        files = ShellHelper.prepare_list(files_list)
 
-        results = ShellHelper.search_start(log_list, filter_type, args, opts)
+        results = ShellHelper.search_start(files, flags, args)
 
-        # Skipo and Print Total if set
-        if args.total
+        # Skip and Print Total if set
+        if flags[:total]
           ShellHelper.total_count(results)
           return true
         end
 
         # Check Search Results
         if results.values.flatten.empty?
-          puts 'No results'.colorize(:red)
+          puts 'No results'.pastel(:red)
         else
           # This causes the key 'colorized' output to also be included
-          ShellHelper.show(results.to_a.compact.flatten, args)
+          ShellHelper.show(results.to_a.compact.flatten, flags)
         end
       rescue StandardError => e
-        LogBot.fatal('Filter', message: e.message, backtrace: e.backtrace.first)
+        LogBot.fatal('Search', message: e.message)
+        ap e.backtrace
       end
       # ========================================================================
 
@@ -180,48 +194,48 @@ module GreenHat
 
       # rubocop:disable Metrics/MethodLength
       def self.search_help
-        puts "\u2500".colorize(:cyan) * 20
-        puts 'Log Search'.colorize(:yellow)
-        puts "\u2500".colorize(:cyan) * 20
+        puts "\u2500".pastel(:cyan) * 20
+        puts 'Log Search'.pastel(:yellow)
+        puts "\u2500".pastel(:cyan) * 20
 
         puts 'Search will do a full line include or exclude text search'
 
-        puts 'Options'.colorize(:blue)
-        puts '--text'.colorize(:green)
+        puts 'Options'.pastel(:blue)
+        puts '--text'.pastel(:green)
         puts '  Primary parameter for searching. Include or ! to exclude'
         puts '  Ex: --text=BuildHooksWorker --text!=start sidekiq/current'
         puts
 
-        puts '--total'.colorize(:green)
+        puts '--total'.pastel(:green)
         puts '  Print only total count of matching entries'
         puts
 
-        puts '--slice'.colorize(:green)
+        puts '--slice'.pastel(:green)
         puts '  Extract specific fields from entries (slice multiple with comma)'
         puts '  Ex: --slice=path or --slice=path,params'
         puts
 
-        puts '--except'.colorize(:green)
+        puts '--except'.pastel(:green)
         puts '  Exclude specific fields (except multiple with comma)'
         puts '  Ex: --except=params --except=params,path'
         puts
 
-        puts '--archive'.colorize(:green)
-        puts '  Limit to specific archvie name (inclusive). Matching SOS tar.gz name'
+        puts '--archive'.pastel(:green)
+        puts '  Limit to specific archive name (inclusive). Matching SOS tar.gz name'
         puts '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
         puts
 
-        puts '--limit'.colorize(:green)
+        puts '--limit'.pastel(:green)
         puts '  Limit total number of results. Default to half total screen size'
         puts '  Ex: --limit; --limit=10'
         puts
 
-        puts 'Search specific logs'.colorize(:blue)
+        puts 'Search specific logs'.pastel(:blue)
         puts '  Any non dash parameters will be the log list to search from'
-        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.colorize(:yellow)}` for log names)"
+        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.pastel(:yellow)}` for log names)"
         puts
 
-        puts 'Example Queries'.colorize(:blue)
+        puts 'Example Queries'.pastel(:blue)
         puts 'log search --text=BuildHooksWorker --text!=start sidekiq/current --total'
         puts 'log search --text=BuildHooksWorker --text!=start --slice=enqueued_at sidekiq/current'
         puts

data/lib/greenhat/shell/page.rb

@@ -3,14 +3,18 @@ module GreenHat
     # Helper to organize page check / methods
     module Page
       # Check if paging should be skipped
-      def self.skip?(args, data)
-        return count_rows(data) unless args[:page]
+      # True / False / Not Set
+      def self.skip?(flags, data)
+        # Pass if Explicitly Set / Inverse for skip
+        return !flags[:page] if flags.key? :page
 
-        args[:page].first == :false
+        LogBot.debug('Page', count_rows(data, flags)) if ENV['DEBUG']
+
+        count_rows(data, flags)
       end
 
       # Array/Hash and String pagination check. Don't unncessarily loop through everything
-      def self.count_rows(data)
+      def self.count_rows(data, flags)
         height = TTY::Screen.height
         size = 0
 
@@ -19,7 +23,9 @@ module GreenHat
                   when Hash then entry.keys.count
                   when Array then entry.count
                   else
-                    1
+                    # Each Boxed Entry is 3 Lines
+                    flags.key?(:row_size) ? flags[:row_size] : 1
+
                   end
 
           break if size > height

data/lib/greenhat/shell/process.rb

@@ -11,16 +11,22 @@ module GreenHat
       end
 
       def self.help
-        puts "\u2500".colorize(:cyan) * 20
-        puts "#{'Process'.colorize(:yellow)} -  ps helper"
-        puts "\u2500".colorize(:cyan) * 20
+        puts "\u2500".pastel(:cyan) * 22
+        puts "#{'Process'.pastel(:yellow)} -  ps helper"
+        puts "\u2500".pastel(:cyan) * 22
 
-        puts 'Command Summary'.colorize(:blue)
-        puts '  ps'.colorize(:green)
+        ShellHelper.common_opts
+
+        puts 'Command Summary'.pastel(:blue)
+        puts '  ps'.pastel(:green)
         puts '    Raw `ps`'
-        puts '  filter'.colorize(:green)
-        puts '    Key/Field Filtering'
-        puts '    - See `filter_help`'
+        puts
+        puts '  filter'.pastel(:green)
+        puts "    Key/Field Filtering.  See #{'filter_help'.pastel(:blue)}"
+        puts '    Examples'
+        puts '      filter --sort=mem --reverse'.pastel(:green)
+        puts '      filter --user=gitlab'.pastel(:green)
+
         puts
       end
 
@@ -29,25 +35,31 @@ module GreenHat
       end
 
       def self.ps(raw = {})
-        _log_list, _1opts, args = ShellHelper.param_parse(raw)
-        ShellHelper.file_output GreenHat::Ps.ps(args)
+        # Extract Args
+        files_list, flags, _args = Args.parse(raw)
+
+        # Collect Files
+        files = ShellHelper.files(files_list, GreenHat::Ps.things, flags)
+
+        # Output
+        ShellHelper.file_output(files, flags)
       end
 
       def self.filter(raw = {})
-        _log_list, opts, args = ShellHelper.param_parse(raw)
+        # Argument Parsing
+        files, flags, args = Args.parse(raw)
 
-        # AND / OR Filtering
-        filter_type = args.or ? :any? : :all?
+        # Prepare Log List
+        file_list = ShellHelper.prepare_list(files, GreenHat::Ps.things)
 
-        # ShellHelper.file_output
-        results = ShellHelper.filter_start(GreenHat::Ps.ps(args).map(&:name), filter_type, args, opts)
+        results = ShellHelper.filter_start(file_list, flags, args)
 
         # Check Search Results
         if results.instance_of?(Hash) && results.values.flatten.empty?
-          puts 'No results'.colorize(:red)
+          puts 'No results'.pastel(:red)
         else
           # This causes the key 'colorized' output to also be included
-          ShellHelper.show(results.to_a.compact.flatten, args)
+          ShellHelper.show(results.to_a.compact.flatten, flags)
         end
       end
     end

data/lib/greenhat/shell/report.rb

@@ -2,11 +2,17 @@ module GreenHat
   # Root Level Shell / Report Helper
   module Shell
     def self.report(raw)
-      args = {
-        raw: raw.include?('--raw')
-      }
+      _files, flags, _args = Args.parse(raw)
 
-      ShellHelper.show(Archive.all.map(&:report).map(&:show).flatten, args)
+      archives = if flags.archive
+                   Archive.all.select do |archive|
+                     flags.archive.any? { |x| archive.name.include? x.to_s }
+                   end
+                 else
+                   Archive.all
+                 end
+
+      ShellHelper.show(archives.map(&:report).map(&:show).flatten, flags)
     end
   end
 end
@@ -45,11 +51,11 @@ module GreenHat
 
     def show
       output = [
-        archive.friendly_name.colorize(:blue)
+        archive.friendly_name.pastel(:blue)
       ]
 
       # OS
-      output << 'OS'.colorize(:light_yellow)
+      output << 'OS'.pastel(:bright_yellow)
       output << hostname if host
       output << distro if os_release
       output << selinux if selinux_status
@@ -62,7 +68,7 @@ module GreenHat
 
       # Memory
       if meminfo || free_m
-        output << 'Memory'.colorize(:light_yellow)
+        output << 'Memory'.pastel(:bright_yellow)
         output << memory_perc if meminfo
         output << memory_free if free_m
         output << ''
@@ -75,7 +81,7 @@ module GreenHat
       end
 
       # Gitlab
-      output << 'GitLab'.colorize(:light_yellow) if gitlab_manifest
+      output << 'GitLab'.pastel(:bright_yellow) if gitlab_manifest
       output << gitlab_version if gitlab_manifest
       output << gitlab_services if gitlab_status
       output << title('Errors') if production_log || api_log || application_log || sidekiq_log
@@ -95,8 +101,8 @@ module GreenHat
       color = count.zero? ? :green : :red
 
       [
-        title('  Production', :light_red, 18),
-        count.to_s.colorize(color)
+        title('  Production', :bright_red, 18),
+        count.to_s.pastel(color)
       ].join
     end
 
@@ -105,8 +111,8 @@ module GreenHat
       color = count.zero? ? :green : :red
 
       [
-        title('  API', :light_red, 18),
-        count.to_s.colorize(color)
+        title('  API', :bright_red, 18),
+        count.to_s.pastel(color)
       ].join
     end
 
@@ -115,8 +121,8 @@ module GreenHat
       color = count.zero? ? :green : :red
 
       [
-        title('  Application', :light_red, 18),
-        count.to_s.colorize(color)
+        title('  Application', :bright_red, 18),
+        count.to_s.pastel(color)
       ].join
     end
 
@@ -125,32 +131,16 @@ module GreenHat
       color = count.zero? ? :green : :red
 
       [
-        title('  Sidekiq', :light_red, 18),
-        count.to_s.colorize(color)
+        title('  Sidekiq', :bright_red, 18),
+        count.to_s.pastel(color)
       ].join
     end
 
     def gitlab_services
-      list = gitlab_status.data.keys.sort.map do |service|
-        color = gitlab_status.data.dig(service, 0, :status) == 'run' ? :green : :red
-        service.colorize(color)
-      end
-
-      # Keep Alphabetical Sort
-      groups = list.each_slice((list.size / 3.to_f).round).to_a
-
-      table = TTY::Table.new do |t|
-        loop do
-          break if groups.all?(&:empty?)
-
-          t << groups.map(&:shift)
-        end
-      end
-
       [
         title('Services'),
         "\n   ",
-        table.render(:unicode, padding: [0, 1, 0, 1], indent: 3)
+        GitLab.services(archive, 3)
       ].join
     rescue StandardError => e
       LogBot.fatal('GitLab Services', message: e.message, backtrace: e.backtrace.first)
@@ -173,7 +163,7 @@ module GreenHat
     def distro
       [
         title('Distro'),
-        "[#{os_release.data.ID}] ".colorize(:light_black),
+        "[#{os_release.data.ID}] ".pastel(:bright_black),
         os_release.data.PRETTY_NAME
       ].join
     end
@@ -184,7 +174,7 @@ module GreenHat
 
       [
         title('SeLinux'),
-        status.colorize(status_color),
+        status.pastel(status_color),
         ' (',
         selinux_status.data['Current mode'],
         ')'
@@ -204,7 +194,7 @@ module GreenHat
       [
         title('Kernel'),
         value,
-        " (#{build})".colorize(:light_black)
+        " (#{build})".pastel(:bright_black)
       ].join
     end
 
@@ -231,9 +221,9 @@ module GreenHat
       [
         title('Sys Time'),
         timedatectl.data['Local time'],
-        ' (ntp: '.colorize(:light_black),
-        ntp_status.colorize(ntp_color),
-        ')'.colorize(:light_black)
+        ' (ntp: '.pastel(:bright_black),
+        ntp_status.pastel(ntp_color),
+        ')'.pastel(:bright_black)
       ].join
     end
 
@@ -258,14 +248,14 @@ module GreenHat
         [
           interval,
           ' (',
-          "#{value}%".colorize(color),
+          "#{value}%".pastel(color),
           ')'
         ].join
       end
 
       [
         title('LoadAvg'),
-        "[CPU #{cpu_count}] ".colorize(:light_white),
+        "[CPU #{cpu_count}] ".pastel(:bright_white),
         intervals_text.join(', ')
       ].join
     end
@@ -277,11 +267,11 @@ module GreenHat
 
       [
         title('Usage'),
-        '  ['.colorize(:light_black),
-        '='.colorize(:green) * (used / 2),
+        '  ['.pastel(:bright_black),
+        '='.pastel(:green) * (used / 2),
         ' ' * (50 - used / 2),
-        ']'.colorize(:light_black),
-        " #{100 - percent(free, total)}%".colorize(:green) # Inverse
+        ']'.pastel(:bright_black),
+        " #{100 - percent(free, total)}%".pastel(:green) # Inverse
       ].join
     end
 
@@ -317,7 +307,7 @@ module GreenHat
 
       # Preapre / Indent List
       [
-        'Disks(Top % Usage)'.colorize(:light_yellow),
+        'Disks'.pastel(:bright_yellow) + ' (Top % Usage)'.pastel(:bright_black),
         "\n",
         disk_list.each { |x| x.prepend(' ' * 4) }.join("\n")
       ].join
@@ -332,7 +322,7 @@ module GreenHat
 
     # Helper to Make Cyan Titles
     def title(name, color = :cyan, ljust = 12)
-      "  #{name}:".ljust(ljust).colorize(color)
+      "  #{name}:".ljust(ljust).pastel(color)
     end
   end
   # rubocop:enable Metrics/ClassLength