greenhat 0.2.0 → 0.3.0

data/lib/greenhat/shell/help.rb

@@ -1,15 +1,96 @@
-# module GreenHat
-#   module Shell
-#     # CLI Helper
-#     module Help
-#       def self.help
-#         puts 'Available Commands: '
-#         puts '=> help'
-
-#         ShellCommand.descendants_s.each do |item|
-#           puts "-> #{item.colorize(:yellow)}"
-#         end
-#       end
-#     end
-#   end
-# end
+module GreenHat
+  # Root Level Shell / Splitting Help into its own file
+  module Shell
+    # rubocop:disable Layout/LineLength
+    def self.help
+      puts 'Quickstart'.pastel(:blue)
+      puts "  Commands are organized by submodule: #{'log'.pastel(:blue)}, #{'cat'.pastel(:blue)}, #{'faststats'.pastel(:blue)}"
+      puts "  Use #{'help'.pastel(:bright_blue)} for available commands in each module"
+      puts
+
+      puts '  Example Commands'
+      puts '    log filter sidekiq/current'.pastel(:yellow)
+      puts '    disk free'.pastel(:yellow)
+      puts
+
+      puts 'Top Level Commands'.pastel(:blue)
+      puts '  report'.pastel(:green)
+      puts '    Show summary report of SOS Report. OS, CPU, Memory, Disk, and etc'
+      puts
+
+      puts '  ps,df,netstat,free,uptime,uname'.pastel(:green)
+      puts '    Show common files from archives / Emulate terminal commands'
+      puts
+
+      puts '  Noisy Output'.pastel(:green)
+      puts "    Use #{'quiet'.pastel(:blue)} or to #{'debug'.pastel(:blue)} to toggle greenhat logging"
+      puts
+
+      cli_shortcuts
+
+      puts "See #{'about'.pastel(:bright_blue)} for more details about GreenHat"
+    end
+    # rubocop:enable Layout/LineLength
+
+    def self.cli_shortcuts
+      puts "\u2500".pastel(:cyan) * 25
+      puts 'Nav / Keyboard Shortcuts'.pastel(:blue)
+      puts "\u2500".pastel(:cyan) * 25
+      puts <<~BLOCK
+        | Hotkey              | Description             |
+        | ------------------- | ----------------------- |
+        | Ctrl + U            | Clear Input             |
+        | Ctrl + A            | Go to beginning         |
+        | Ctrl + E            | Go to End               |
+        | Ctrl + Left/Right   | Move left/right by word |
+        | Ctrl + D, Ctrl + Z  | Exit                    |
+        | Ctrl + C, Shift Tab | Up one module           |
+      BLOCK
+      puts
+    end
+
+    def self.about
+      puts "\u2500".pastel(:cyan) * 20
+      puts "About GreenHat #{GreenHat::VERSION}".pastel(:yellow)
+      puts "\u2500".pastel(:cyan) * 20
+
+      puts 'TLDR; Put in SOS reports, run commands, and find stuffs'.pastel(:green)
+      puts
+
+      puts <<~BLOCK
+        General overview (OS, Memory, Disk, GitLab)
+        #{'report'.pastel(:bright_cyan)}
+
+        Log Searching
+        #{'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'.pastel(:bright_cyan)}
+
+        Read File(s) across SOS archives
+        #{'cat uptime'.pastel(:bright_cyan)} or #{'cat mount etc/fstab'.pastel(:bright_cyan)}
+
+      BLOCK
+
+      puts 'What it does / How it works'.pastel(:blue)
+      puts
+      puts <<~BLOCK
+        GreenHat is a support utility to enhance troubleshooting with GitLabSOS Reports and log files. Make it easy to find stuff
+
+        Supplied input files are staged, unpacked, identified, and normalized.
+        This enables other utilities to automatically find and present data. (Faststats, report, and etc)
+
+      BLOCK
+
+      puts 'Commands and Submodules'.pastel(:blue)
+      puts
+      puts <<~BLOCK
+        Greenhat is broken down into different "modules". Each module has its own commands. For example: log, cat, and faststats.
+        You can "cd" into or execute commands directly against with their names.
+
+        - Direct: #{'log filter sidekiq/current'.pastel(:cyan)}
+        - Or within: First #{'log'.pastel(:cyan)}, then #{'filter sidekiq/current'.pastel(:cyan)}
+
+        You can find the list of commands and submodules of each with #{'help'.pastel(:yellow)}
+
+      BLOCK
+    end
+  end
+end

data/lib/greenhat/shell/list.rb

@@ -0,0 +1,46 @@
+module GreenHat
+  module ShellHelper
+    # Helper to handle listing of files
+    module List
+      # List Files Helpers
+      def self.list(raw = [], files)
+        filter, flags, _args = Args.parse(raw)
+
+        # Sort
+        files.sort_by!(&:name)
+
+        # Simplified vs Full. Full file name/path / or just file kinds
+        all = flags.key?(:all) || flags.key?(:a)
+
+        # Short & Uniq
+        files.uniq!(&:name) unless all
+
+        # Filter / Pattern
+        files.select! { |f| filter.any? { |x| f.name.include? x } } unless filter.empty?
+
+        # Print
+        files.each do |log|
+          if all
+            puts "- #{log.friendly_name}"
+          else
+            puts "- #{log.name.pastel(:yellow)}"
+          end
+        end
+      end
+
+      # Unified Help
+      def self.help
+        puts '  ls'.pastel(:green)
+        puts '    List available files'
+        puts '    Options'.pastel(:cyan)
+        puts '      -a, --all, show full file name/path including source'
+        puts '      <string> filter available'
+        puts '    Examples'.pastel(:cyan)
+        puts '      ls -a rails'
+        puts '      ls sys'
+        puts
+      end
+      # ----
+    end
+  end
+end

data/lib/greenhat/shell/log.rb

@@ -4,171 +4,144 @@ module GreenHat
     # Logs
     module Log
       def self.help
-        puts "\u2500".colorize(:cyan) * 20
-        puts "#{'Logs'.colorize(:yellow)} find stuff"
-        puts "\u2500".colorize(:cyan) * 20
+        puts "\u2500".pastel(:cyan) * 20
+        puts "#{'Logs'.pastel(:yellow)} find stuff"
+        puts "\u2500".pastel(:cyan) * 20
+
+        puts 'Command Summary'.pastel(:blue)
+        puts '  filter'.pastel(:green)
+        puts "    Primary way for log searching within greenhat. See #{'filter_help'.pastel(:blue)}"
+        puts '    Time, round, slice/except, and/or, stats, uniq, sort'
+        puts
 
-        puts 'Command Summary'.colorize(:blue)
-        puts '  show'.colorize(:green)
+        puts '  show'.pastel(:green)
         puts '    Just print selected logs'
-        puts '  filter'.colorize(:green)
-        puts '    Key/Field Filtering'
-        puts '    - See `filter_help`'
-        puts '  search'.colorize(:green)
-        puts '    General text by entry searching'
-        puts '    - See `search_help`'
-        puts '  ls'.colorize(:green)
-        puts '    List available files'
         puts
-      end
 
-      def self.filter_help
-        ShellHelper::Filter.help
-      end
-
-      # List Files Helpers
-      def self.list(args = [])
-        all = false
-        all = true if args.include?('-a') || args.include?('--all')
-
-        files = ShellHelper::Log.list
+        puts '  search'.pastel(:green)
+        puts "    General full text by file searching. See #{'search_help'.pastel(:blue)}"
+        puts
 
-        # Sort
-        files.sort_by!(&:name)
+        puts ShellHelper::List.help
 
-        # Short & Uniq
-        files.uniq!(&:name) unless all
+        puts "See #{'examples'.pastel(:bright_blue)} for query examples"
+      end
 
-        # Print
-        files.each do |log|
-          if all
-            puts "- #{log.friendly_name}"
-          else
-            puts "- #{log.name.colorize(:yellow)}"
-          end
-        end
+      def self.filter_help
+        ShellHelper::Filter.help
       end
 
       def self.ls(args = [])
-        list(args)
+        ShellHelper::List.list(args, ShellHelper::Log.list)
       end
 
-      def self.show(log_list)
-        # Prepare Log List
-        log_list = ShellHelper.prepare_list(log_list)
-
-        # Convert to Things
-        logs = ShellHelper.find_things(log_list)
+      def self.show(raw = {})
+        # Extract Args
+        files_list, flags, _args = Args.parse(raw)
 
-        logs.map!(&:data)
+        # Collect Files
+        files = ShellHelper.files(files_list, Thing.all, flags)
 
-        ShellHelper.show logs.flatten
+        ShellHelper.show files.map(&:data).flatten
       end
 
       # ========================================================================
-      # Filter
+      # Filter (See Filter Help)
       # ========================================================================
-      # Supported Params
-      # --or (Filter OR instead of AND)
-      # --total Total Count Entries Only
-      # --project=thingy --exclude_this!=asdf *
-      # --slice:  only grab specific fields --slice=path (slice multiple with comma)
-      # --slice=time,path (E.g. log filter  --path='mirror/pull' --slice=path,time )
-      # --except: Exclude specific fields (except multiple with comma)
-      # Example: log filter  --path='mirror/pull' --except=params
       def self.default(raw_list)
         filter(raw_list)
       end
 
-      def self.filter(raw_list)
+      def self.filter(raw)
         # Print Helper
-        if raw_list == ['help']
+        if raw == ['help']
           filter_help
           return true
         end
 
-        # Extract Args
-        log_list, opts, args = ShellHelper.param_parse(raw_list)
+        # Argument Parsing
+        files, flags, args = Args.parse(raw)
 
         # Prepare Log List
-        log_list = ShellHelper.prepare_list(log_list)
-
-        # AND / OR Filtering
-        filter_type = args.or ? :any? : :all?
+        files = ShellHelper.prepare_list(files, ShellHelper::Log.list, flags)
 
-        results = ShellHelper.filter_start(log_list, filter_type, args, opts)
+        results = ShellHelper.filter_start(files, flags, args)
 
-        # Skipo and Print Total if set
-        if args.total
+        # Skip and Print Total if set
+        if flags[:total]
           ShellHelper.total_count(results)
           return true
         end
 
         # Check Search Results
         if results.instance_of?(Hash) && results.values.flatten.empty?
-          puts 'No results'.colorize(:red)
+          puts 'No results'.pastel(:red)
         else
           # This causes the key 'colorized' output to also be included
-          ShellHelper.show(results.to_a.compact.flatten, args)
+          ShellHelper.show(results.to_a.compact.flatten, flags)
         end
 
         # log filter --path='cloud/gitlab-automation' --path='/pull' --all
         # log filter --project=thingy --other_filter=asdf *
       rescue StandardError => e
-        LogBot.fatal('Filter', message: e.message, backtrace: e.backtrace.first)
+        LogBot.fatal('Filter', message: e.message)
+        ap e.backtrace
       end
       # ========================================================================
 
       # rubocop:disable Layout/LineLength
       # TODO: Add a lot more examples
       def self.examples
-        puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.colorize(:light_green)
+        puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.pastel(:bright_green)
         puts 'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'
         puts
-        puts 'Find 500s only show exceptions'.colorize(:light_green)
+
+        puts 'Find 500s only show exceptions'.pastel(:bright_green)
         puts 'log filter --status=500 --slice=exception.message gitlab-rails/production_json.log'
         puts
+
+        puts 'Show unique sidekiq queue namespaces. Exclude Specifics'.pastel(:bright_green)
+        puts 'filter sidekiq/current --slice=queue_namespace --uniq=queue_namespace --queue_namespace!=jira_connect --queue_namespace!=hashed_storage'
+        puts
+
+        puts 'Show user,ip from API logs where `meta.user` field is present '.pastel(:bright_green)
+        puts 'gitlab-rails/api_json.log --slice=meta.user,meta.remote_ip --exists=meta.user'
+        puts
+
+        puts 'Count/% occurences for both user and remote ip fields'.pastel(:bright_green)
+        puts 'gitlab-rails/api_json.log --stats=meta.user,meta.remote_ip --exists=meta.user'
+        puts
       end
 
       # rubocop:enable Layout/LineLength
       # ========================================================================
       # Search (Full Text / String Search)
       # ========================================================================
-      # Supported Params
-      # --text='asdf'
-      # --text!='asdf'
-      # --regex='' # TODO?
-      # --slice=time,path (E.g. log filter  --path='mirror/pull' --slice=path,time )
-      # --except: Exclude specific fields (except multiple with comma)
-
-      # --total Total Count Entries Only
-      def self.search(initial_param)
+      def self.search(raw)
         # Extract Args
-        log_list, opts, args = ShellHelper.param_parse(initial_param)
+        files_list, flags, args = Args.parse(raw)
 
         # Prepare Log List
-        log_list = ShellHelper.prepare_list(log_list)
-
-        # AND / OR Filtering
-        filter_type = args.or ? :any? : :all?
+        files = ShellHelper.prepare_list(files_list)
 
-        results = ShellHelper.search_start(log_list, filter_type, args, opts)
+        results = ShellHelper.search_start(files, flags, args)
 
-        # Skipo and Print Total if set
-        if args.total
+        # Skip and Print Total if set
+        if flags[:total]
           ShellHelper.total_count(results)
           return true
         end
 
         # Check Search Results
         if results.values.flatten.empty?
-          puts 'No results'.colorize(:red)
+          puts 'No results'.pastel(:red)
         else
           # This causes the key 'colorized' output to also be included
-          ShellHelper.show(results.to_a.compact.flatten, args)
+          ShellHelper.show(results.to_a.compact.flatten, flags)
         end
       rescue StandardError => e
-        LogBot.fatal('Filter', message: e.message, backtrace: e.backtrace.first)
+        LogBot.fatal('Search', message: e.message)
+        ap e.backtrace
       end
       # ========================================================================
 
@@ -180,48 +153,48 @@ module GreenHat
 
       # rubocop:disable Metrics/MethodLength
       def self.search_help
-        puts "\u2500".colorize(:cyan) * 20
-        puts 'Log Search'.colorize(:yellow)
-        puts "\u2500".colorize(:cyan) * 20
+        puts "\u2500".pastel(:cyan) * 20
+        puts 'Log Search'.pastel(:yellow)
+        puts "\u2500".pastel(:cyan) * 20
 
         puts 'Search will do a full line include or exclude text search'
 
-        puts 'Options'.colorize(:blue)
-        puts '--text'.colorize(:green)
+        puts 'Options'.pastel(:blue)
+        puts '--text'.pastel(:green)
         puts '  Primary parameter for searching. Include or ! to exclude'
         puts '  Ex: --text=BuildHooksWorker --text!=start sidekiq/current'
         puts
 
-        puts '--total'.colorize(:green)
+        puts '--total'.pastel(:green)
         puts '  Print only total count of matching entries'
         puts
 
-        puts '--slice'.colorize(:green)
+        puts '--slice'.pastel(:green)
         puts '  Extract specific fields from entries (slice multiple with comma)'
         puts '  Ex: --slice=path or --slice=path,params'
         puts
 
-        puts '--except'.colorize(:green)
+        puts '--except'.pastel(:green)
         puts '  Exclude specific fields (except multiple with comma)'
         puts '  Ex: --except=params --except=params,path'
         puts
 
-        puts '--archive'.colorize(:green)
-        puts '  Limit to specific archvie name (inclusive). Matching SOS tar.gz name'
+        puts '--archive'.pastel(:green)
+        puts '  Limit to specific archive name (inclusive). Matching SOS tar.gz name'
         puts '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
         puts
 
-        puts '--limit'.colorize(:green)
+        puts '--limit'.pastel(:green)
         puts '  Limit total number of results. Default to half total screen size'
         puts '  Ex: --limit; --limit=10'
         puts
 
-        puts 'Search specific logs'.colorize(:blue)
+        puts 'Search specific logs'.pastel(:blue)
         puts '  Any non dash parameters will be the log list to search from'
-        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.colorize(:yellow)}` for log names)"
+        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.pastel(:yellow)}` for log names)"
         puts
 
-        puts 'Example Queries'.colorize(:blue)
+        puts 'Example Queries'.pastel(:blue)
         puts 'log search --text=BuildHooksWorker --text!=start sidekiq/current --total'
         puts 'log search --text=BuildHooksWorker --text!=start --slice=enqueued_at sidekiq/current'
         puts