greenhat 0.5.0 → 0.6.2

data/lib/greenhat/shell/reports.rb

@@ -0,0 +1,41 @@
+# Reporting
+require 'greenhat/reports/shared' # Methods for Shell and Builder
+require 'greenhat/reports/internal_methods' # Class / Execution Only
+require 'greenhat/reports/runner' # Class execution
+require 'greenhat/reports/builder' # CLI Reports
+require 'greenhat/reports/shell_helper'
+
+module GreenHat
+  # CLI Helper
+  module Shell
+    # Logs
+    module Reports
+      # Easy Show All
+      def self.ls(raw = [])
+        filter, _flags, _args = Args.parse(raw)
+
+        list = GreenHat::ShellHelper::Reports.list
+        list.select! { |x| x.include? filter.first } unless filter.blank?
+
+        list.each do |x|
+          puts "- #{File.basename(x, '.rb').pastel(:yellow)}"
+        end
+      end
+
+      def self.default(raw = [])
+        list, flags, args = Args.parse(raw)
+
+        # Collect Reporst to Run
+        run_list = GreenHat::ShellHelper::Reports.list.select do |entry|
+          list.include? File.basename(entry, '.rb')
+        end
+
+        run_list.uniq!
+
+        run_list.each do |file|
+          GreenHat::ShellHelper::Reports.run(file: file, args: args, flags: flags)
+        end
+      end
+    end
+  end
+end

data/lib/greenhat/shell/shell_helper.rb

@@ -1,6 +1,5 @@
 module GreenHat
   # Common Helpers
-  # rubocop:disable Metrics/ModuleLength
   module ShellHelper
     # Use File Process for Output
     def self.file_output(files, flags = {})
@@ -48,114 +47,156 @@ module GreenHat
 
       # Check if content needs to paged, or if auto_height is off
       if Page.skip?(flags, data)
-        puts data.map { |entry| entry_show(flags, entry) }.compact.join("\n")
+        create_ledger(data, flags).each { |entry| puts entry.render }
         return true
       end
 
-      # Default Pager
-      TTY::Pager.page do |pager|
-        data.each do |entry|
-          output = "\n#{entry_show(flags, entry)}"
-
-          # output += "\n" if flags[:table_style] == 'basic'
-
-          # Breaks any intentional spaces
-          # next if output.blank?
-
-          pager.write(output) # write line to the pager
+      # Create Jobs
+      # Generate List of Formatted Output
+      ledger = create_ledger(data, flags)
+
+      # Start Thread
+      # TODO: Confirm / Test or Remove
+      unless flags.no_paper_thread
+        thread = Thread.new do
+          pending = ledger.clone
+          loop do
+            pending = pending.reject(&:done)
+            break if pending.empty?
+
+            pending.first.async
+          end
         end
       end
-    end
-
-    # Entry Shower / Top Level
-    def self.entry_show(flags, entry, key = nil)
-      LogBot.debug('Entry Show', entry.class) if ENV['DEBUG']
-      case entry
-      when Hash then render_table(entry, flags)
-      when Float, Integer, Array
-        format_table_entry(flags, entry, key)
-        # Ignore Special Formatting for Strings / Usually already formatted
-      when String
-        entry
-      else
-        LogBot.warn('Shell Show', "Unknown #{entry.class}")
-        nil
-      end
-    end
 
-    # Format Table Entries
-    def self.format_table_entry(flags, entry, key = nil)
-      formatted_entry = case entry
-                        # Rounding
-                        when Float, Integer || entry.numeric?
-                          flags.key?(:round) ? entry.to_f.round(flags.round).ai : entry.ai
-
-                        # General Inspecting
-                        when Hash then entry.ai(ruby19_syntax: true)
-
-                        # Arrays often contain Hashes. Dangerous Recursive?
-                        when Array
-                          entry.map { |x| format_table_entry(flags, x) }.join("\n")
-                        when Time
-                          entry.to_s.pastel(:bright_white)
-
-                        # Default String Formatting
-                        else
-                          StringColor.do(key, entry)
-                        end
-
-      # Stats truncation handled separately
-      if flags[:truncate] && !flags[:stats]
-        entry_truncate(formatted_entry, flags[:truncate])
-      else
-        formatted_entry
-      end
-    rescue StandardError => e
-      if ENV['DEBUG']
-        LogBot.warn('Table Format Entry', message: e.message)
-        ap e.backtrace
-      end
-    end
-
-    # Print the Table in a Nice way
-    def self.render_table(entry, flags)
-      entry = entry.to_h { |k, v| [k, format_table_entry(flags, v, k)] }
-      # Pre-format Entry
-
-      table_style = flags[:table_style]&.to_sym || :unicode
-
-      table = TTY::Table.new(header: entry.keys, rows: [entry], orientation: :vertical)
-
-      LogBot.debug('Rendering Entries') if ENV['DEBUG']
-      output = table.render(table_style, padding: [0, 1, 0, 1], multiline: true) do |renderer|
-        renderer.border.style = :cyan
+      TTY::Pager.page do |pager|
+        ledger.each do |paper|
+          # New Line is only needed on pager.write
+          pager.write "#{paper.render}\n"
+        end
       end
 
-      # Line breaks for basic tables
-      output += "\n" if flags[:table_style] == 'basic'
+      # Terminate
+      thread.kill unless flags.no_paper_thread
 
-      output
+      # -----------------
+    end
 
-      # LogBot.debug('Finish Render Table') if ENV['DEBUG']
-      # Fall Back to Amazing Inspect
-    rescue StandardError => e
-      if ENV['DEBUG']
-        LogBot.warn('Table', message: e.message)
-        ap e.backtrace
+    # Collect Class Objects for output
+    def self.create_ledger(data, flags)
+      data.map do |x|
+        Paper.new(data: x, flags: flags)
       end
+    end
 
-      [
-        entry.ai,
-        ('_' * (TTY::Screen.width / 3)).pastel(:cyan),
-        "\n"
-      ].join("\n")
+    def self.puts(text = '')
+      $stdout.puts text
     end
 
-    def self.render_table_entry(val, col_index, flags)
-      return val.to_s unless col_index == 1
+    # Entry Shower / Top Level
+    # def self.entry_show(flags, entry, key = nil)
+    #   LogBot.debug('Entry Show', entry.class) if ENV['DEBUG']
+    #   case entry
+    #   when Hash then render_table(entry, flags)
+    #   when Float, Integer
+    #     format_table_entry(flags, entry, key)
+    #     # Ignore Special Formatting for Strings / Usually already formatted
+    #   when String
+    #     entry
+    #   when Array
+    #     render_array_table(entry, flags)
+    #   else
+    #     LogBot.warn('Shell Show', "Unknown #{entry.class}")
+    #     nil
+    #   end
+    # end
 
-      format_table_entry(flags, val)
-    end
+    # Format Table Entries
+    # def self.format_table_entry(flags, entry, key = nil)
+    #   formatted_entry = case entry
+    #                     # Rounding
+    #                     when Float, Integer || entry.numeric?
+    #                       flags.key?(:round) ? entry.to_f.round(flags.round).ai : entry.ai
+
+    #                     # General Inspecting
+    #                     when Hash then entry.ai(ruby19_syntax: true)
+
+    #                     # Arrays often contain Hashes. Dangerous Recursive?
+    #                     when Array
+    #                       entry.map { |x| format_table_entry(flags, x) }.join("\n")
+    #                     when Time
+    #                       entry.to_s.pastel(:bright_white)
+
+    #                     # Default String Formatting
+    #                     else
+    #                       StringColor.do(key, entry)
+    #                     end
+
+    #   # Stats truncation handled separately
+    #   if flags[:truncate] && !flags[:stats]
+    #     entry_truncate(formatted_entry, flags[:truncate])
+    #   else
+    #     formatted_entry
+    #   end
+    # rescue StandardError => e
+    #   if ENV['DEBUG']
+    #     LogBot.warn('Table Format Entry', message: e.message)
+    #     ap e.backtrace
+    #   end
+    # end
+
+    # def self.render_array_table(entry, flags)
+    #   table_style = flags[:table_style]&.to_sym || :unicode
+    #   header, rows = entry
+    #   table = TTY::Table.new(header: header, rows: rows)
+
+    #   LogBot.debug('Rendering Entries') if ENV['DEBUG']
+    #   output = table.render(table_style, padding: [0, 1, 0, 1], multiline: true) do |renderer|
+    #     renderer.border.style = :cyan
+    #   end
+
+    #   # Line breaks for basic tables
+    #   output += "\n" if flags[:table_style] == 'basic'
+
+    #   output
+    # end
+
+    # # Print the Table in a Nice way
+    # def self.render_table(entry, flags)
+    #   entry = entry.to_h { |k, v| [k, format_table_entry(flags, v, k)] }
+    #   # Pre-format Entry
+
+    #   table_style = flags[:table_style]&.to_sym || :unicode
+
+    #   table = TTY::Table.new(header: entry.keys, rows: [entry], orientation: :vertical)
+
+    #   LogBot.debug('Rendering Entries') if ENV['DEBUG']
+    #   output = table.render(table_style, padding: [0, 1, 0, 1], multiline: true) do |renderer|
+    #     renderer.border.style = :cyan
+    #   end
+
+    #   # Line breaks for basic tables
+    #   output += "\n" if flags[:table_style] == 'basic'
+
+    #   output
+    # rescue StandardError => e
+    #   if ENV['DEBUG']
+    #     LogBot.warn('Table', message: e.message)
+    #     ap e.backtrace
+    #   end
+
+    #   [
+    #     entry.ai,
+    #     ('_' * (TTY::Screen.width / 3)).pastel(:cyan),
+    #     "\n"
+    #   ].join("\n")
+    # end
+
+    # def self.render_table_entry(val, col_index, flags)
+    #   return val.to_s unless col_index == 1
+
+    #   format_table_entry(flags, val)
+    # end
 
     # Internal Query Helper
     # query = 'gitlab-rails/application_json.log --message!="Cannot obtain an exclusive lease" --severity=error'
@@ -183,28 +224,42 @@ module GreenHat
 
     # Total Count Helper
     def self.total_count(results, flags)
-      results.map do |k, v|
-        puts k
-        # puts "Total: #{v.count.to_s.pastel(:blue)}"
-        # puts "Time Covered: #{calculate_duration(v).pastel(:cyan)}"
-
-        output = {}
-        output[:total] = v.count
-        output[:duration] = Query.calculate_duration(v)
-
-        # Sort / Get first and Last
-        list = v.select(&:time).map(&:time)
-        unless list.blank?
-          output[:start] = list.first
-          output[:end] = list.last
-        end
+      # Support Combine Helper
+      # TODO I Believe this should be removed now
+      # total_count_hash(results, flags) if results.instance_of? Hash
 
-        puts render_table(output, flags)
+      total_count_array(results, flags) if results.instance_of? Array
+    end
 
-        puts
+    # Hash Iteration
+    def self.total_count_hash(results, flags)
+      results.flat_map do |k, values|
+        [k, total_count_array(values, flags)]
       end
     end
 
+    # List Calculation
+    def self.total_count_array(results, flags)
+      # Option to only return the count
+      return results.count if flags[:total] == 'simple'
+
+      output = {}
+      output[:total] = results.count
+      output[:duration] = Query.calculate_duration(results)
+
+      # Sort / Get first and Last
+      list = results.select(&:time).map(&:time)
+      unless list.blank?
+        output[:start] = list.first
+        output[:end] = list.last
+      end
+
+      # Hide empty results
+      output.reject! { |_k, v| v.blank? }
+
+      render_table(output, flags)
+    end
+
     # Table Printer Helper
     def self.fields_print(results)
       results.each do |k, v|
@@ -239,7 +294,7 @@ module GreenHat
       file_list = prepare_list(file_list, base_list)
 
       # Convert to Things
-      find_things(file_list, flags)
+      find_things(file_list, flags, base_list)
     end
 
     # Total Log List Manipulator
@@ -263,13 +318,15 @@ module GreenHat
     end
 
     # Shortcut find things
-    def self.find_things(files, flags = {})
-      things = files.uniq.flat_map do |file|
+    def self.find_things(file_list, flags = {}, base_list = nil)
+      base_list ||= Thing.all
+
+      things = file_list.uniq.flat_map do |file|
         # If Thing, Return Thing
         return file if file.instance_of?(Thing)
 
         if flags.fuzzy_file_match
-          Thing.list.select { |x| x.name.include?(file) || x.type.include?(file) }
+          base_list.select { |x| x.name.include?(file) || x.type.include?(file) }
         else
           Thing.where name: file
         end
@@ -315,6 +372,4 @@ module GreenHat
       puts
     end
   end
-
-  # rubocop:enable Metrics/ModuleLength
 end

data/lib/greenhat/shell.rb

@@ -18,6 +18,12 @@ module GreenHat
       GreenHat::Web.toggle
     end
 
+    # Append Log Location so follow up commands are executed there
+    def self.default(raw_list = [])
+      Cli.move_shell Shell::Log
+      Log.default(raw_list)
+    end
+
     def self.df
       Disk.df
     end
@@ -101,8 +107,13 @@ module GreenHat
       end
     end
 
-    def self.ls
-      GreenHat::Cli.help(false)
+    def self.ls(raw = [])
+      if raw.empty?
+        GreenHat::Cli.help(false)
+      else
+        Cli.move_shell Shell::Log
+        Log.ls(raw)
+      end
     end
 
     def self.ll

data/lib/greenhat/thing/file_types.rb

@@ -27,7 +27,8 @@ module GreenHat
         'df_h' => {
           format: :table,
           pattern: [
-            /df_h/
+            /df_h/,
+            /df_hT/
           ]
         },
         'df_inodes' => {
@@ -64,6 +65,13 @@ module GreenHat
             %r{log/syslog}
           ]
         },
+        'gitlab-runner.log' => {
+          format: :runner_log,
+          log: true,
+          pattern: [
+            /gitlab-runner.log/
+          ]
+        },
         'etc/fstab' => {
           format: :clean_raw,
           pattern: [
@@ -123,6 +131,20 @@ module GreenHat
             %r{consul/current}
           ]
         },
+        'consul/identify_primary_db_node.log' => {
+          format: :identify_db,
+          log: true,
+          pattern: [
+            %r{consul/identify_primary_db_node.log}
+          ]
+        },
+        'gitlab-kas/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{gitlab-kas/current}
+          ]
+        },
         'sentinel/current' => {
           format: :time_space,
           log: true,
@@ -151,6 +173,13 @@ module GreenHat
             %r{mailroom/mail_room_json.log}
           ]
         },
+        'gitlab_migrations' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            /gitlab_migrations/
+          ]
+        },
         'gitlab-rails/git_json.log' => {
           format: :json,
           log: true,
@@ -201,12 +230,19 @@ module GreenHat
           ]
         },
         'gitlab-exporter/current' => {
-          format: :time_space,
+          format: :exporters,
           log: true,
           pattern: [
             %r{gitlab-exporter/current}
           ]
         },
+        'gitlab-rails/web_exporter' => {
+          format: :exporters,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/web_exporter}
+          ]
+        },
         'gitlab-monitor/current' => {
           format: :time_space,
           log: true,

data/lib/greenhat/thing/formatters/clean_raw.rb

@@ -4,7 +4,7 @@ module GreenHat
   module Formatters
     # Remove Comments / Empty Lines
     def format_clean_raw
-      staging = raw.clone
+      staging = raw_full.clone
 
       # Empty
       staging.reject!(&:empty?)

data/lib/greenhat/thing/formatters/exporters.rb

@@ -0,0 +1,48 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # General Exporter Parser
+    # https://gitlab.com/gitlab-com/support/toolbox/greenhat/-/issues/30
+    # [YYYY-MM-DDTHH:MM:SS.sss-HHMM] …ip… - - [DD/Mon/YYYY:HH:MM:SS ZON]
+    # "GET /metrics HTTP/1.1" 200 12345 "-" "Prometheus/2.25.0"
+    # ==========================================================================
+    def format_exporters
+      self.result = raw.map do |row|
+        result = {}
+        time, output = row.split(' ', 2)
+        result[:time] = Time.parse(time)
+
+        # Weird Edge Cases for when IP is not included
+        msg = ''
+        if output.include? '- - '
+          ip, msg = output.split('- -', 2)
+          result[:ip] = ip unless ip.empty?
+        else
+          msg = output
+        end
+
+        # Styrip Extra Time / Get Call Details
+        if msg&.include? ']'
+          _time, request = msg.split('] ', 2)
+          result[:msg] = request # Pass full request
+
+          # Breakout
+          method, path, protocol, status, bytes = request.gsub('"', '').split(' ', 5)
+
+          result.merge!(method: method, path: path, protocol: protocol, status: status, bytes: bytes)
+
+        else
+          result[:msg] = msg
+        end
+
+        result
+      rescue StandardError => e
+        # TODO: Background logger
+        LogBot.warn('Exporters', "Unable to Parse, #{row}:#{e.message}")
+      end
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/identify_db.rb

@@ -0,0 +1,32 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # consul/identify_primary_db_node.log
+    # https://gitlab.com/gitlab-com/support/toolbox/greenhat/-/issues/33
+    # I, [YYYY-MM-DDThh:mm:ss.ssssss #12345]  INFO -- : Found master: sub.domain.tld
+    def format_identify_db
+      self.result = raw.map do |row|
+        next if row.empty? || row == "\n"
+
+        level_abrv, row = row.split(', ', 2)
+
+        time, output = row.split('] ', 2)
+        time = Time.parse time.split('[', 2).last.strip
+
+        level, _a, _b, msg = output.split(' ', 4)
+        {
+          time: time,
+          i: level_abrv,
+          level: level,
+          message: msg
+        }
+      rescue StandardError => e
+        # TODO: Background logger
+        LogBot.warn('identify_db', "Unable to Parse, #{row}:#{e.message}")
+      end
+
+      result.compact!
+    end
+  end
+end

data/lib/greenhat/thing/formatters/runner_log.rb

@@ -0,0 +1,70 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Attempt to make runner logs easier to parse
+    def format_runner_log
+      self.result = raw.map do |row|
+        next if row.empty? || row == "\n"
+        next if row.include? '-- Logs begin at' # Skip Header
+
+        process_runner_log_row(row)
+      end
+
+      result.compact!
+    end
+
+    # Split / Parse Time
+    # TODO: Better sys log parsing? Cannot find ready-made regex/ruby parser
+    def process_runner_log_row(row)
+      month, day, time, device, service, message = row.split(' ', 6)
+      service&.gsub!(':', '')
+      pid = service&.match(/\[(.*?)\]/)&.captures&.first
+
+      entry = {
+        time: format_time_parse("#{month} #{day} #{time}"),
+        device: device,
+        service: service,
+        message: message,
+        pid: pid,
+        row: row # Insert full row for different timestamp formats
+      }
+
+      # Remove Empty Keys
+      entry.compact!
+
+      # Shellwords split - Skip Blank
+      process_runner_row_message(message, entry) unless message.blank?
+
+      entry
+
+    # Return everything incase of error
+    rescue StandardError => e
+      LogBot.warn('Runner Log Parse', e.message)
+      {
+        message: row
+      }
+    end
+
+    def process_runner_row_message(message, entry)
+      # Insert Keywords / Format Time if Possible
+      Shellwords.split(message).select { |x| x.include? '=' }.each do |x|
+        key, value = x.split('=', 2)
+
+        # Don't overwrite stuff
+        case key
+        when 'time'
+          entry["#{key}-msg".to_sym] = format_time_parse value
+        when 'message', :message
+          entry["#{key}-msg".to_sym] = value
+        else
+          entry[key.to_sym] = value
+        end
+      end
+    rescue StandardError
+      # LogBot.debug('Runner Log Parse', e.message)
+      nil
+    end
+    # --
+  end
+end

data/lib/greenhat/thing/formatters/table.rb

@@ -2,10 +2,24 @@
 module GreenHat
   # Log
   module Formatters
+    # Looks like this may be different per Type
+    def header_count
+      case name
+      when 'df_hT' then 7
+      end
+    end
+
     # Table Formatting
     def format_table
+      # Default to split by space unless specifically needed
+      headers_raw = if header_count
+                      raw_full.first.split(' ', header_count)
+                    else
+                      raw_full.first.split
+                    end
+
       # Headers to Readable Symbol
-      headers = raw_full.first.split(' ', 6).map(&:downcase).map do |x|
+      headers = headers_raw.map(&:downcase).map do |x|
         x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
       end.map(&:to_sym)