greenhat 0.2.0 → 0.3.3

data/lib/greenhat/shell/field_helper.rb

@@ -0,0 +1,75 @@
+module GreenHat
+  # Common Helpers
+  module FieldHelper
+    def self.fields_find(files, word, flags = {})
+      fields = ShellHelper.find_things(files, flags).map(&:fields).flatten.uniq
+
+      if word.blank?
+        puts 'Possible Fields:'.pastel(:bright_blue)
+        puts ShellHelper.field_table(fields)
+
+        return [] # Empty Result
+      end
+
+      list_select(fields, word)
+    end
+
+    def self.list_select(list, word)
+      list.select! { |x| x[/^#{Regexp.escape(word)}/] }
+    end
+
+    def self.filter_flags(word)
+      if word.blank?
+        puts 'Filter Options:'.pastel(:bright_blue)
+        puts ShellHelper.field_table(filter_opts, 6)
+        puts
+
+        return []
+      end
+
+      list_select(filter_opts, word)
+    end
+
+    def self.filter_opts
+      %w[
+        archive case combine end exact except exists json limit or page pluck
+        raw reverse round slice sort start stats table_style text time_zone
+        total truncate uniq
+      ]
+    end
+
+    def self.filter_auto_completes
+      %w[
+        except exists pluck slice sort stats uniq
+      ]
+    end
+
+    def self.field_auto_complete?(word)
+      return false if word.blank?
+
+      filter_auto_completes.include? word.split('=', 2).first
+    end
+
+    def self.field_auto_complete(word, files, flags = {})
+      # Prevent weird dupes
+      return nil if word[-1] == ','
+
+      # Command Manipulation
+      cmd, fields = word.split('=', 2)
+      complete = fields.split(',')[0..-2]
+      auto = fields.split(',').last
+
+      # Field Finder
+      matches = fields_find(files, auto, flags)
+
+      if matches.count == 1
+        "--#{cmd}=#{(complete + matches).join(',')}"
+      elsif matches.count > 1
+        puts "#{'Field Options:'.pastel(:bright_blue)} #{matches.join(' ').pastel(:bright_green)}"
+
+        list = [Cli.common_substr(matches.map(&:to_s))]
+        "--#{cmd}=#{(complete + list).join(',')}"
+      end
+    end
+  end
+end

data/lib/greenhat/shell/filter_help.rb

@@ -0,0 +1,162 @@
+module GreenHat
+  # CLI Helper
+  module ShellHelper
+    # Unify Filter / Filter Help
+    module Filter
+      # rubocop:disable Metrics/MethodLength
+      def self.help
+        puts "\u2500".pastel(:cyan) * 20
+        puts 'Filter'.pastel(:yellow)
+        puts "\u2500".pastel(:cyan) * 20
+
+        puts 'Options'.pastel(:blue)
+        puts '--raw'.pastel(:green)
+        puts '  Disable formatting and page/less'
+        puts
+
+        puts '--page'.pastel(:green)
+        puts '  Specifically enable or disable paging'
+        puts '  E.g. --page (default to true), --page=true, --page=false'
+        puts
+
+        puts '--round'.pastel(:green)
+        puts '  Attempt to round all integers. Default: 2.'
+        puts '  E.g. --round, --round=3, --round=0'
+        puts
+
+        puts '--limit'.pastel(:green)
+        puts '  Limit total output lines. Disabled by default. Default without value is based on screen height'
+        puts '  E.g. --limit, --limit=5'
+        puts
+
+        puts '--json'.pastel(:green)
+        puts '  Print output back into JSON'
+        puts
+
+        puts '--or'.pastel(:green)
+        puts '  Filters will use OR instead of AND (all match vs any match)'
+        puts
+
+        puts '--total'.pastel(:green)
+        puts '  Print only total count of matching entries'
+        puts
+
+        puts '--fields'.pastel(:green)
+        puts '  Print only Available fields for selected files'
+        puts
+
+        puts '--slice'.pastel(:green)
+        puts '  Extract specific fields from entries (slice multiple with comma)'
+        puts '  Ex: --slice=path or --slice=path,params'
+        puts
+
+        puts '--except'.pastel(:green)
+        puts '  Exclude specific fields (except multiple with comma)'
+        puts '  Ex: --except=params --except=params,path'
+        puts
+
+        puts '--exists'.pastel(:green)
+        puts '  Ensure field exists regardless of contents'
+        puts '  Ex: --exists=params --exists=params,path'
+        puts
+
+        puts '--stats'.pastel(:green)
+        puts '  Order/Count occurrances by field. Combine with `truncate` for key names'
+        puts '  Ex: --stats=params --except=params,path'
+        puts
+
+        puts '--uniq'.pastel(:green)
+        puts '  Show unique values only'
+        puts '  Ex: --uniq=params --uniq=params,path'
+        puts
+
+        puts '--pluck'.pastel(:green)
+        puts '  Extract values from entries'
+        puts '  Ex: --pluck=params --pluck=params,path'
+        puts
+
+        puts '--archive'.pastel(:green)
+        puts '  Limit to specific archvie name (partial matching /inclusive). Matching SOS tar.gz name'
+        puts '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
+        puts
+
+        puts '--sort'.pastel(:green)
+        puts '  Sort by multiple fields'
+        puts '  Ex: --sort=duration_s,db_duration_s'
+        puts
+
+        puts '--reverse'.pastel(:green)
+        puts '  Reverse all results'
+        puts '  Ex: --reverse'
+        puts
+
+        puts '--combine'.pastel(:green)
+        puts '  Omit archive identifier dividers. Useful with sort or time filters'
+        puts '  Ex: --combine'
+        puts
+
+        puts '--case'.pastel(:green)
+        puts '  Exact case match. Defaults to case insensitive'
+        puts '  Ex: --case; --name=Jon, --name=jane --case'
+        puts
+
+        puts '--exact'.pastel(:green)
+        puts '  Exact parameter/value match. Defaults to partial match'
+        puts '  Ex: --field=CommonPartial --exact'
+        puts
+
+        puts '--start'.pastel(:green)
+        puts '  Show events after specified time. Filtered by the `time` field'
+        puts '  Use with `--end` for between selections'
+        puts '  Ex: log filter --start="2021-06-22 14:44 UTC" --end="2021-06-22 14:45 UTC"'
+        puts
+
+        puts '--end'.pastel(:green)
+        puts '  Show events before specified time. Filtered by the `time` field'
+        puts '  Use with `--start` for between selections'
+        puts '  Ex: log filter --end="2021-06-22"'
+        puts
+
+        puts '--time_zone'.pastel(:green)
+        puts '  Manipulate the `time` field into a specific timezone'
+        puts '  Ex: log filter --time_zone=EDT'
+        puts
+
+        puts '--text'.pastel(:green)
+        puts '  Full entry text searching (slow)'
+        puts '    --text="anything here"'
+        puts
+
+        puts '--table_style'.pastel(:green)
+        puts '  Renderer used for formatted output. basic, ascii, or unicode(default)'
+        puts '  Ex: log filter --table_style=base'
+        puts
+
+        puts '--truncate'.pastel(:green)
+        puts '  Truncate field length. On by default (4 rows). Performance issues!'
+        puts '  Disable with --truncate=0'.pastel(:bright_red)
+        puts '  Ex: --truncate=200, --truncate=2048"'
+        puts
+
+        puts 'Field Searching'.pastel(:blue)
+        puts '  --[key]=[value]'
+        puts '  Search in key for value'
+        puts '  Example: --path=mirror/pull'
+        puts
+
+        puts 'Search specific logs'.pastel(:blue)
+        puts '  Any non dash parameters will be the log list to search from'
+        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.pastel(:yellow)}` for log names"
+        puts
+
+        puts 'Example Queries'.pastel(:blue)
+        puts "  Also see #{'examples'.pastel(:bright_blue)} for even more examples"
+        puts '  log filter --class=BuildFinishedWorker sidekiq/current --slice=time,message'
+        puts '  log filter gitlab-rails/api_json.log --slice=ua --uniq=ua --ua=gitlab-runner'
+
+        puts
+      end
+      # rubocop:enable Metrics/MethodLength
+    end
+  end
+end

data/lib/greenhat/shell/gitlab.rb

@@ -3,16 +3,75 @@ module GreenHat
   module Shell
     # Logs
     module Gitlab
+      def self.ls
+        help
+      end
+
+      def self.help
+        puts "\u2500".pastel(:cyan) * 20
+        puts "#{'GitLab'.pastel(:yellow)} - Tanuki power"
+        puts "\u2500".pastel(:cyan) * 20
+
+        puts 'Command Summary'.pastel(:blue)
+
+        puts '  services'.pastel(:green)
+        puts '    Show services status and version'
+        puts
+
+        puts '  status'.pastel(:green)
+        puts '    Show gitlab-ctl status'
+        puts
+
+        puts '  architecture'.pastel(:green)
+        puts '    Show node responsibility. E.g. Webservice, Gitaly, and etc'
+        puts
+
+        puts '  version'.pastel(:green)
+        puts '    Show GitLab software version'
+        puts
+      end
+
+      def self.architecture
+        results = Archive.all.map { |x| GitLab.identify_node(x) }
+
+        GitLab.node_types.each do |entry|
+          list = results.select { |x| (x.services & entry.pattern).any? }.map(&:host).join(', ')
+          next if list.blank?
+
+          puts entry.name.pastel(:bright_green)
+          puts list
+          puts
+        end
+      end
+
       def self.version
         Thing.where(type: 'gitlab/version-manifest.json').each do |file|
           next unless file.data
 
           puts file.friendly_name
-          puts Semantic::Version.new(file.data.build_version).to_s.colorize(:yellow)
+          puts Semantic::Version.new(file.data.build_version).to_s.pastel(:yellow)
           puts
         end
       end
 
+      # Print service info / shared report method
+      def self.services(raw = {})
+        _files, flags, _args = Args.parse(raw, [:truncate])
+
+        results = {}
+
+        Archive.all.each do |archive|
+          result = GitLab.services(archive)
+          next unless result
+
+          # Generated Output
+          results[archive.friendly_name.pastel(:blue)] = result
+        end
+
+        # Print
+        ShellHelper.show(results, flags)
+      end
+
       def self.status
         Thing.where(type: 'gitlab_status').each do |file|
           next unless file.data
@@ -27,7 +86,7 @@ module GreenHat
 
               [
                 "#{service.status}:",
-                service.name.ljust(pad).colorize(color),
+                service.name.ljust(pad).pastel(color),
                 "#{service.pid_uptime};".ljust(pad)
 
               ]

data/lib/greenhat/shell/help.rb

@@ -1,15 +1,98 @@
-# module GreenHat
-#   module Shell
-#     # CLI Helper
-#     module Help
-#       def self.help
-#         puts 'Available Commands: '
-#         puts '=> help'
-
-#         ShellCommand.descendants_s.each do |item|
-#           puts "-> #{item.colorize(:yellow)}"
-#         end
-#       end
-#     end
-#   end
-# end
+module GreenHat
+  # Root Level Shell / Splitting Help into its own file
+  module Shell
+    # rubocop:disable Layout/LineLength
+    def self.help
+      puts 'Quickstart'.pastel(:blue)
+      puts "  Commands are organized by submodule: #{'log'.pastel(:blue)}, #{'cat'.pastel(:blue)}, #{'faststats'.pastel(:blue)}"
+      puts "  Use #{'help'.pastel(:bright_blue)} for available commands in each module"
+      puts
+
+      puts '  Example Commands'
+      puts '    log filter sidekiq/current'.pastel(:yellow)
+      puts '    disk free'.pastel(:yellow)
+      puts
+
+      puts 'Top Level Commands'.pastel(:blue)
+      puts '  report'.pastel(:green)
+      puts '    Show summary report of SOS Report. OS, CPU, Memory, Disk, and etc'
+      puts '      --raw, no pagination'
+      puts '      --archive=<redis/archive>, filter by archive name'
+      puts
+
+      puts '  ps,df,netstat,free,uptime,uname'.pastel(:green)
+      puts '    Show common files from archives / Emulate terminal commands'
+      puts
+
+      puts '  Noisy Output'.pastel(:green)
+      puts "    Use #{'quiet'.pastel(:blue)} or to #{'debug'.pastel(:blue)} to toggle greenhat logging"
+      puts
+
+      cli_shortcuts
+
+      puts "See #{'about'.pastel(:bright_blue)} for more details about GreenHat"
+    end
+    # rubocop:enable Layout/LineLength
+
+    def self.cli_shortcuts
+      puts "\u2500".pastel(:cyan) * 25
+      puts 'Nav / Keyboard Shortcuts'.pastel(:blue)
+      puts "\u2500".pastel(:cyan) * 25
+      puts <<~BLOCK
+        | Hotkey              | Description             |
+        | ------------------- | ----------------------- |
+        | Ctrl + U            | Clear Input             |
+        | Ctrl + A            | Go to beginning         |
+        | Ctrl + E            | Go to End               |
+        | Ctrl + Left/Right   | Move left/right by word |
+        | Ctrl + D, Ctrl + Z  | Exit                    |
+        | Ctrl + C, Shift Tab | Up one module           |
+      BLOCK
+      puts
+    end
+
+    def self.about
+      puts "\u2500".pastel(:cyan) * 20
+      puts "About GreenHat #{GreenHat::VERSION}".pastel(:yellow)
+      puts "\u2500".pastel(:cyan) * 20
+
+      puts 'TLDR; Put in SOS reports, run commands, and find stuffs'.pastel(:green)
+      puts
+
+      puts <<~BLOCK
+        General overview (OS, Memory, Disk, GitLab)
+        #{'report'.pastel(:bright_cyan)}
+
+        Log Searching
+        #{'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'.pastel(:bright_cyan)}
+
+        Read File(s) across SOS archives
+        #{'cat uptime'.pastel(:bright_cyan)} or #{'cat mount etc/fstab'.pastel(:bright_cyan)}
+
+      BLOCK
+
+      puts 'What it does / How it works'.pastel(:blue)
+      puts
+      puts <<~BLOCK
+        GreenHat is a support utility to enhance troubleshooting with GitLabSOS Reports and log files. Make it easy to find stuff
+
+        Supplied input files are staged, unpacked, identified, and normalized.
+        This enables other utilities to automatically find and present data. (Faststats, report, and etc)
+
+      BLOCK
+
+      puts 'Commands and Submodules'.pastel(:blue)
+      puts
+      puts <<~BLOCK
+        Greenhat is broken down into different "modules". Each module has its own commands. For example: log, cat, and faststats.
+        You can "cd" into or execute commands directly against with their names.
+
+        - Direct: #{'log filter sidekiq/current'.pastel(:cyan)}
+        - Or within: First #{'log'.pastel(:cyan)}, then #{'filter sidekiq/current'.pastel(:cyan)}
+
+        You can find the list of commands and submodules of each with #{'help'.pastel(:yellow)}
+
+      BLOCK
+    end
+  end
+end

data/lib/greenhat/shell/list.rb

@@ -0,0 +1,46 @@
+module GreenHat
+  module ShellHelper
+    # Helper to handle listing of files
+    module List
+      # List Files Helpers
+      def self.list(raw = [], files)
+        filter, flags, _args = Args.parse(raw)
+
+        # Sort
+        files.sort_by!(&:name)
+
+        # Simplified vs Full. Full file name/path / or just file kinds
+        all = flags.key?(:all) || flags.key?(:a)
+
+        # Short & Uniq
+        files.uniq!(&:name) unless all
+
+        # Filter / Pattern
+        files.select! { |f| filter.any? { |x| f.name.include? x } } unless filter.empty?
+
+        # Print
+        files.each do |log|
+          if all
+            puts "- #{log.friendly_name}"
+          else
+            puts "- #{log.name.pastel(:yellow)}"
+          end
+        end
+      end
+
+      # Unified Help
+      def self.help
+        puts '  ls'.pastel(:green)
+        puts '    List available files'
+        puts '    Options'.pastel(:cyan)
+        puts '      -a, --all, show full file name/path including source'
+        puts '      <string> filter available'
+        puts '    Examples'.pastel(:cyan)
+        puts '      ls -a rails'
+        puts '      ls sys'
+        puts
+      end
+      # ----
+    end
+  end
+end