greenhat 0.1.4 → 0.3.1

data/lib/greenhat/shell.rb

@@ -2,15 +2,17 @@ module GreenHat
   # Root Level Shell
   module Shell
     def self.pry
+      # rubocop:disable Lint/Debugger
       binding.pry
+      # rubocop:enable Lint/Debugger
     end
 
     def self.df
       Disk.df
     end
 
-    def self.ps
-      Process.ps
+    def self.ps(raw = {})
+      Process.ps raw
     end
 
     def self.netstat
@@ -34,7 +36,7 @@ module GreenHat
       file_list.each do |file|
         next if File.size(file).zero?
 
-        puts "- Loading #{file.colorize(:green)}"
+        puts "- Loading #{file.pastel(:green)}"
 
         archive.things_create(file: file).setup
       end
@@ -42,14 +44,26 @@ module GreenHat
 
     def self.debug
       ENV['DEBUG'] = if ENV['DEBUG']
-                       puts "Debug #{'Off'.colorize(:red)}"
+                       puts "GreenHat Debug Logging #{'Off'.pastel(:red)}"
                        nil
                      else
-                       puts "Debug #{'On'.colorize(:green)}"
+                       puts "GreenHat Debug Logging #{'On'.pastel(:green)}"
                        'true'
                      end
     end
 
+    def self.quiet
+      Cli.quiet!
+
+      if Cli.quiet
+        puts "GreenHat Quiet Logging #{'Off'.pastel(:red)}"
+        nil
+      else
+        puts "GreenHat Quiet Logging #{'On'.pastel(:green)}"
+        'true'
+      end
+    end
+
     def self.uptime
       Shell::Cat.default ['uptime']
     end
@@ -63,26 +77,26 @@ module GreenHat
     end
 
     def self.history_clear
-      File.write(GreenHat::Cli.history_file, "\n")
+      Settings.cmd_history_clear
     end
 
     def self.history
-      File.read(GreenHat::Cli.history_file).split("\n").each_with_index do |line, i|
-        puts "#{i.to_s.ljust(3).colorize(:magenta)} #{line}"
+      Settings.cmd_history_clean.each_with_index do |line, i|
+        puts "#{i.to_s.ljust(3).pastel(:magenta)} #{line}"
       end
     end
 
     def self.ls
-      GreenHat::Cli.help
+      GreenHat::Cli.help(false)
     end
 
     def self.ll
-      GreenHat::Cli.help
+      GreenHat::Cli.help(false)
     end
 
-    def self.about
-      puts "#{'GreenHat'.colorize(:green)}: #{GreenHat::VERSION.colorize(:blue)}"
-      puts ' - https://gitlab.com/gitlab-com/support/toolbox/greenhat'.colorize(:cyan)
+    def self.version
+      puts "#{'GreenHat'.pastel(:green)}: #{GreenHat::VERSION.pastel(:blue)}"
+      puts ' - https://gitlab.com/gitlab-com/support/toolbox/greenhat'.pastel(:cyan)
     end
   end
 end

data/lib/greenhat/thing/file_types.rb

@@ -1,5 +1,19 @@
 module GreenHat
-  # Overall Type Parsing
+  # -------------------------------------
+  # File Types - Overall Type Parsing!
+  # -------------------------------------
+  # --- Adding a new File ----
+  # Duplicate one of the blocks and then update the name and pattern/regex to identify it
+  #
+  # The 'key' or 'name' is used to generically identify a type of file to be used in other collection tools.
+  #   - Looking for errors in `gitlab-workhorse/current` will also include any rotated files via `gitlab-workhorse/@.*`
+  #
+  # Formatters: See `thing/formmaters` for the different kinds of formatters
+  #   - `json` for plain json
+  #   - `raw` to ignore/just read (e.g. for cat)
+  #
+  # log true/false to be included in the log/filter fields
+
   # rubocop:disable Metrics/MethodLength, Metrics/ModuleLength
   module FileTypes
     def types
@@ -29,9 +43,16 @@ module GreenHat
             /dmesg/
           ]
         },
-        'log/syslog' => {
+        'repmgrd/current' => {
           format: :bracket_log,
           log: true,
+          pattern: [
+            %r{repmgrd/current}
+          ]
+        },
+        'log/syslog' => {
+          format: :syslog,
+          log: true,
           pattern: [
             %r{log/syslog}
           ]
@@ -81,6 +102,20 @@ module GreenHat
             %r{consul/current}
           ]
         },
+        'sentinel/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{sentinel/current}
+          ]
+        },
+        'consul/failover_pgbouncer.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{consul/failover_pgbouncer.log}
+          ]
+        },
         'pgbouncer/current' => {
           format: :time_space,
           log: true,
@@ -109,6 +144,13 @@ module GreenHat
             %r{gitaly/gitaly_ruby_json.log}
           ]
         },
+        'gitaly/gitaly_lfs_smudge.log' => {
+          format: :json,
+          log: true,
+          pattern: [
+            %r{gitaly/gitaly_lfs_smudge.log}
+          ]
+        },
         'gitlab_status' => {
           format: :gitlab_status,
           pattern: [
@@ -158,6 +200,13 @@ module GreenHat
             %r{logrotate/current}
           ]
         },
+        'gitlab-rails/grpc.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/grpc.log}
+          ]
+        },
         'gitlab-rails/api_json.log' => {
           format: :api_json,
           log: true,
@@ -280,15 +329,22 @@ module GreenHat
             /mount/
           ]
         },
+        'nginx/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{nginx/current}
+          ]
+        },
         'nginx/gitlab_pages_access.log' => {
-          format: :raw,
+          format: :nginx,
           log: true,
           pattern: [
             %r{nginx/gitlab_pages_access.log}
           ]
         },
         'nginx/gitlab_access.log' => {
-          format: :raw,
+          format: :nginx,
           log: true,
           pattern: [
             %r{nginx/gitlab_access.log}
@@ -428,7 +484,13 @@ module GreenHat
             %r{redis/config},
             %r{redis-exporter/config},
             %r{registry/config},
-            %r{sidekiq/config}
+            %r{sidekiq/config},
+            %r{pgbouncer/config},
+            %r{pgbouncer-exporter/config},
+            %r{repmgrd/config},
+            %r{sentinel/config},
+            %r{consul/config},
+            %r{mailroom/current}
           ]
         },
         'redis/current' => {
@@ -439,7 +501,6 @@ module GreenHat
             %r{redis/@.*}
           ]
         },
-
         'nginx/gitlab_error.log' => {
           format: :time_space,
           log: true,
@@ -447,6 +508,13 @@ module GreenHat
             %r{nginx/gitlab_error.log}
           ]
         },
+        'nginx/error.log' => {
+          format: :raw,
+          log: false,
+          pattern: [
+            %r{nginx/error.log}
+          ]
+        },
         'nginx/gitlab_registry_error.log' => {
           format: :time_space,
           log: true,
@@ -468,7 +536,7 @@ module GreenHat
           ]
         },
         'nginx/gitlab_registry_access.log' => {
-          format: :raw,
+          format: :nginx,
           log: true,
           pattern: [
             %r{nginx/gitlab_registry_access.log}
@@ -650,7 +718,7 @@ module GreenHat
           ]
         },
         'log/messages' => {
-          format: :raw,
+          format: :syslog,
           log: true,
           pattern: [
             %r{log/messages}

data/lib/greenhat/thing/formatters/json_shellwords.rb

@@ -18,9 +18,6 @@ module GreenHat
 
         result.sort.to_h
       rescue StandardError => e
-        binding.pry
-        # TODO: Background Logger?
-        e.message
         LogBot.warn('JSON Parse', e.message)
         next
       end

data/lib/greenhat/thing/formatters/nginx.rb

@@ -0,0 +1,44 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # ==========================================================================
+    def format_nginx
+      self.result = raw.map do |row|
+        ip, _sym, remote_user, rest = row.split(' ', 4)
+
+        time, rest = rest.split(']', 2)
+        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+
+        verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
+
+        method, path, http_version = verb.split
+
+        {
+          remote_addr: ip,
+          remote_user: remote_user,
+          method: method,
+          path: path,
+          status: status,
+          bytes: bytes,
+          http_version: http_version,
+          http_referer: http_referer,
+          http_user_agent: http_user_agent,
+          gzip_ratio: gzip_ratio,
+          time: time
+        }
+      end
+
+      :ok
+    end
+
+    # rubocop:disable Metrics/LineLength
+    # NGINX Conf: /var/opt/gitlab/nginx/conf/nginx.conf
+    # log_format gitlab_access '$remote_addr - $remote_user [$time_local] "$request_method $filtered_request_uri $server_protocol" $status $body_bytes_sent "$filtered_http_referer" "$http_user_agent" $gzip_ratio';
+    # rubocop:enable Metrics/LineLength
+
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/syslog.rb

@@ -0,0 +1,39 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Formatters for bracket logs (dmesg, sos)
+    def format_syslog
+      self.result = raw.map do |row|
+        next if row.empty? || row == "\n"
+
+        format_syslog_row(row)
+      end
+
+      result.compact!
+    end
+
+    # Split / Parse Time
+    # TODO: Better sys log parsing? Cannot find ready-made regex/ruby parser
+    def format_syslog_row(row)
+      month, day, time, device, service, message = row.split(' ', 6)
+      service.gsub!(':', '')
+      pid = service.match(/\[(.*?)\]/)&.captures&.first
+
+      {
+        time: format_time_parse("#{month} #{day} #{time}"),
+        device: device,
+        service: service,
+        message: message,
+        pid: pid
+      }
+
+    # Return everything incase of error
+    rescue StandardError => e
+      LogBot.warn('SysLog Parse', e.message)
+      {
+        message: row
+      }
+    end
+  end
+end

data/lib/greenhat/thing/helpers.rb

@@ -5,10 +5,10 @@ module GreenHat
     # Console Helper
     def inspect
       [
-        'Thing'.colorize(:light_black),
-        kind&.to_s&.colorize(:blue),
-        type&.colorize(:light_yellow),
-        name&.colorize(:cyan)
+        'Thing'.pastel(:bright_black),
+        kind&.to_s&.pastel(:blue),
+        type&.pastel(:bright_yellow),
+        name&.pastel(:cyan)
       ].compact.join(' ')
     end
 

data/lib/greenhat/thing/kind.rb

@@ -52,7 +52,9 @@ module GreenHat
 
     # rubocop:disable Style/SymbolProc
     def prompt_for_kind
+      # rubocop:disable Lint/Debugger
       binding.pry if ENV['DEBUG']
+      # rubocop:enable Lint/Debugger
 
       # Default to everything
       prompt_list = types.clone
@@ -61,13 +63,18 @@ module GreenHat
       json = check_oj_parse?(first_line)
 
       if json
+        if Settings.assume_json?
+          self.type = 'json'
+          return true
+        end
+
         prompt_list.select! do |_k, v|
           v.to_s.include? 'json'
         end
       end
 
-      puts "Unable to determine file type for #{name.colorize(:yellow)}"
-      puts "Use '#{'json'.colorize(:cyan)}' and '#{'raw'.colorize(:cyan)}' if you cannot find any matches"
+      puts "Unable to determine file type for #{name.pastel(:yellow)}"
+      puts "Use '#{'json'.pastel(:cyan)}' or '#{'raw'.pastel(:cyan)}' if there are no matches (see file_types.rb)"
 
       option = prompt.select('Wat is this?', prompt_list.keys.sort_by(&:length), filter: true)
 

data/lib/greenhat/thing/spinner.rb

@@ -7,7 +7,7 @@ module GreenHat
 
       @spin_time = Time.now
       @spinner = TTY::Spinner.new(
-        "#{time.colorize(:light_black)} - [:spinner] :title", hide_cursor: true, success_mark: '✔'.colorize(:green)
+        "#{time.pastel(:bright_black)} - [:spinner] :title", hide_cursor: true, success_mark: '✔'.pastel(:green)
       )
       @spinner.update(title: title)
 
@@ -25,7 +25,7 @@ module GreenHat
       title_update = if spin_end.blank?
                        title
                      else
-                       "#{title} (#{spin_end.colorize(:blue)})"
+                       "#{title} (#{spin_end.pastel(:blue)})"
                      end
 
       @spinner.update(title: title_update)
@@ -33,7 +33,7 @@ module GreenHat
     end
 
     def time
-      Time.now.strftime('%I:%M:%S').colorize(:light_black)
+      Time.now.strftime('%I:%M:%S').pastel(:bright_black)
     end
 
     # Replace TimeDifference with https://stackoverflow.com/a/4136485/1678507

data/lib/greenhat/thing.rb

@@ -28,7 +28,7 @@ class Thing < Teron
   field :raw_result # Flag for reading raw data
 
   def friendly_name
-    "#{archive.friendly_name.colorize(:blue)} #{name.colorize(:green)} #{id.colorize(:black)}"
+    "#{archive.friendly_name.pastel(:blue)} #{name.pastel(:green)}"
   end
 
   def setup
@@ -66,7 +66,7 @@ class Thing < Teron
   end
 
   def raw_read
-    spin_start("Read  #{name.colorize(:blue)} #{size.colorize(:light_black)}")
+    spin_start("Read  #{name.pastel(:blue)} #{size.pastel(:bright_black)}")
     self.raw_result = File.read(file).split("\n")
   rescue StandardError => e
     LogBot.fatal('Raw Read', message: e.message, backtrace: e.backtrace.first)
@@ -103,7 +103,7 @@ class Thing < Teron
     raw_read if raw_result.nil?
 
     if methods.include? formatter
-      spin_start("Parse #{name.colorize(:blue)} #{kind.to_s.colorize(:light_black)} ")
+      spin_start("Parse #{name.pastel(:blue)} #{kind.to_s.pastel(:bright_black)} ")
       begin
         send(formatter)
       rescue StandardError => e