greenhat 0.3.3 → 0.4.0

data/lib/greenhat/shell/platform.rb

@@ -0,0 +1,28 @@
+module GreenHat
+  # Common Helpers
+  module Platform
+    def self.platform
+      if @platform
+        @platform
+      else
+        require 'tty-platform'
+        @platform ||= TTY::Platform.new
+
+      end
+    end
+
+    def self.open(url = 'http://localhost:4567/chart/time')
+      cmd = if platform.linux? || platform.unix?
+              'xdg-open'
+            elsif platform.mac?
+              'open'
+            end
+
+      # platform.windows?  # => false
+      # platform.unix?     # => true
+      # platform.linux?    # => false
+      # platform.mac?      # => true
+      system("#{cmd} '#{url}'")
+    end
+  end
+end

data/lib/greenhat/shell/report.rb

@@ -12,7 +12,11 @@ module GreenHat
                    Archive.all
                  end
 
-      ShellHelper.show(archives.map(&:report).map(&:show).flatten, flags)
+      output = archives.map { |x| x.report(flags) }.map(&:show).flatten
+
+      flags[:page] = true if flags.full && !flags.raw
+
+      ShellHelper.show(output, flags)
     end
   end
 end
@@ -23,14 +27,16 @@ module GreenHat
   class Report
     include ActionView::Helpers::NumberHelper
 
-    attr_accessor :archive, :host, :os_release, :selinux_status, :cpu, :uname,
+    attr_accessor :archive, :flags, :host, :os_release, :selinux_status, :cpu, :uname,
                   :timedatectl, :uptime, :meminfo, :gitlab_manifest, :gitlab_status,
-                  :production_log, :api_log, :application_log, :sidekiq_log, :free_m, :disk_free
+                  :production_log, :api_log, :sidekiq_log,
+                  :exceptions_log, :gitaly_log, :free_m, :disk_free
 
     # Find Needed Files for Report
     # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
-    def initialize(archive)
+    def initialize(archive, flags)
       self.archive = archive
+      self.flags = flags
       self.host = archive.things.find { |x| x.name == 'hostname' }
       self.os_release = archive.things.find { |x| x.name == 'etc/os-release' }
       self.selinux_status = archive.things.find { |x| x.name == 'sestatus' }
@@ -44,7 +50,8 @@ module GreenHat
       self.gitlab_status = archive.things.find { |x| x.name == 'gitlab_status' }
       self.production_log = archive.things.find { |x| x.name == 'gitlab-rails/production_json.log' }
       self.api_log = archive.things.find { |x| x.name == 'gitlab-rails/api_json.log' }
-      self.application_log = archive.things.find { |x| x.name == 'gitlab-rails/application_json.log' }
+      self.exceptions_log = archive.things.find { |x| x.name == 'gitlab-rails/exceptions_json.log' }
+      self.gitaly_log = archive.things.find { |x| x.name == 'gitaly/current' }
       self.sidekiq_log = archive.things.find { |x| x.name == 'sidekiq/current' }
       self.disk_free = archive.things.find { |x| x.name == 'df_h' }
     end
@@ -84,11 +91,16 @@ module GreenHat
       output << 'GitLab'.pastel(:bright_yellow) if gitlab_manifest
       output << gitlab_version if gitlab_manifest
       output << gitlab_services if gitlab_status
-      output << title('Errors') if production_log || api_log || application_log || sidekiq_log
+      output << title('Errors') if production_log || api_log || sidekiq_log
       output << production_errors if production_log
-      output << api_errors if api_log
-      output << application_errors if application_log
+      output << application_errors if archive.thing?('gitlab-rails/application_json.log')
       output << sidekiq_errors if sidekiq_log
+      output << api_errors if api_log
+      output << exception_errors if exceptions_log
+      output << gitaly_errors if gitaly_log
+      output << workhorse_errors if archive.thing?('gitlab-workhorse/current')
+
+      full(output) if flags.full
 
       # Final Space / Return
       output << ''
@@ -96,6 +108,52 @@ module GreenHat
     end
     # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
 
+    def full(output)
+      output << ''
+      output << 'FastStats Top'
+      Shell::Faststats.top(['--raw'], true).each { |x| output << x } # Page Row Helper
+      output << ''
+
+      output << 'FastStats Errors'
+      Shell::Faststats.errors(['--raw'], true).each { |x| output << x } # Page Row Helper
+      output << ''
+    end
+
+    def exception_errors
+      count = exceptions_log.data.count
+      color = count.zero? ? :green : :red
+
+      [
+        title('  Exception', :bright_red, 18),
+        count.to_s.pastel(color)
+      ].join
+    end
+
+    def workhorse_errors
+      results = ShellHelper.filter_internal([
+        'gitlab-workhorse/current',
+        '--level=error',
+        "--archive=#{archive.name}"
+      ].join(' '))
+
+      color = results.count.zero? ? :green : :red
+
+      [
+        title('  Workhorse', :bright_red, 18),
+        results.count.to_s.pastel(color)
+      ].join
+    end
+
+    def gitaly_errors
+      count = gitaly_log.data.count { |x| x.level == 'error' }
+      color = count.zero? ? :green : :red
+
+      [
+        title('  Gitaly', :bright_red, 18),
+        count.to_s.pastel(color)
+      ].join
+    end
+
     def production_errors
       count = production_log.data.count { |x| x.status == 500 }
       color = count.zero? ? :green : :red
@@ -117,7 +175,14 @@ module GreenHat
     end
 
     def application_errors
-      count = application_log.data.count { |x| x&.severity == 'ERROR' }
+      results = ShellHelper.filter_internal([
+        'gitlab-rails/application_json.log',
+        '--message!="Cannot obtain an exclusive lease"',
+        '--severity=error',
+        "--archive=#{archive.name}"
+      ].join(' '))
+
+      count = results.count { |x| x&.severity == 'ERROR' }
       color = count.zero? ? :green : :red
 
       [
@@ -269,7 +334,7 @@ module GreenHat
         title('Usage'),
         '  ['.pastel(:bright_black),
         '='.pastel(:green) * (used / 2),
-        ' ' * (50 - used / 2),
+        ' ' * (50 - (used / 2)),
         ']'.pastel(:bright_black),
         " #{100 - percent(free, total)}%".pastel(:green) # Inverse
       ].join
@@ -282,21 +347,37 @@ module GreenHat
 
       formatted_mem = free_m.data.map { |x| GreenHat::Memory.memory_row x }
 
-      [
-        title('Total', :cyan, 14),
-        number_to_human_size(free.total.to_i * 1024**2),
-        "\n",
-        title('Used', :yellow, 14),
-        number_to_human_size(free.used.to_i * 1024**2),
-        "\n",
-        title('Free', :blue, 14),
-        number_to_human_size(free.free.to_i * 1024**2),
-        "\n",
-        title('Available', :green, 14),
-        number_to_human_size(free.available.to_i * 1024**2),
-        "\n\n",
-        formatted_mem.map { |x| x.prepend ' ' * 2 }.join("\n")
-      ].join
+      output = []
+      unless free.total.blank?
+        output << title('Total', :cyan, 14)
+        output << number_to_human_size(free.total.to_i * (1024**2))
+        output << "\n"
+      end
+
+      unless free.total.blank?
+        output << title('Used', :yellow, 14)
+        output << number_to_human_size(free.used.to_i * (1024**2))
+        output << "\n"
+      end
+
+      unless free.total.blank?
+        output << title('Free', :blue, 14)
+        output << number_to_human_size(free.free.to_i * (1024**2))
+        output << "\n"
+      end
+
+      unless free.total.blank?
+        output << title('Available', :green, 14)
+        output << number_to_human_size(free.available.to_i * (1024**2))
+        output << "\n"
+      end
+
+      output << "\n"
+      output << formatted_mem.map { |x| x.prepend ' ' * 2 }.join("\n")
+
+      output.join
+    rescue StandardError => e
+      LogBot.fatal('Memory', message: e.message, backtrace: e.backtrace.first)
     end
 
     def disks

data/lib/greenhat/shell/shell_helper.rb

@@ -55,12 +55,14 @@ module GreenHat
       # Default Pager
       TTY::Pager.page do |pager|
         data.each do |entry|
-          output = entry_show(flags, entry)
+          output = "\n#{entry_show(flags, entry)}"
+
+          # output += "\n" if flags[:table_style] == 'basic'
 
           # Breaks any intentional spaces
           # next if output.blank?
 
-          pager.write("\n#{output}") # write line to the pager
+          pager.write(output) # write line to the pager
         end
       end
     end
@@ -103,7 +105,8 @@ module GreenHat
                           StringColor.do(key, entry)
                         end
 
-      if flags[:truncate]
+      # Stats truncation handled separately
+      if flags[:truncate] && !flags[:stats]
         entry_truncate(formatted_entry, flags[:truncate])
       else
         formatted_entry
@@ -125,10 +128,15 @@ module GreenHat
       table = TTY::Table.new(header: entry.keys, rows: [entry], orientation: :vertical)
 
       LogBot.debug('Rendering Entries') if ENV['DEBUG']
-      table.render(table_style, padding: [0, 1, 0, 1], multiline: true) do |renderer|
+      output = table.render(table_style, padding: [0, 1, 0, 1], multiline: true) do |renderer|
         renderer.border.style = :cyan
       end
 
+      # Line breaks for basic tables
+      output += "\n" if flags[:table_style] == 'basic'
+
+      output
+
       # LogBot.debug('Finish Render Table') if ENV['DEBUG']
       # Fall Back to Amazing Inspect
     rescue StandardError => e
@@ -150,8 +158,21 @@ module GreenHat
       format_table_entry(flags, val)
     end
 
+    # Internal Query Helper
+    # query = 'gitlab-rails/application_json.log --message!="Cannot obtain an exclusive lease" --severity=error'
+    # ShellHelper.filter_internal(query)
+    def self.filter_internal(search = '')
+      files, flags, args = Args.parse(Shellwords.split(search))
+      flags[:combine] = true
+
+      # Default to everything
+      files = Thing.all.map(&:name) if files.empty?
+
+      ShellHelper.filter_start(files, flags, args)
+    end
+
     # Main Entry Point for Filtering
-    def self.filter_start(files, flags, args)
+    def self.filter_start(files, flags = {}, args = {})
       # Convert to Things
       logs = ShellHelper.find_things(files, flags).select(&:processed?)
 
@@ -167,11 +188,17 @@ module GreenHat
 
           # Include Total Count in Name
           results = ShellHelper.filter(log.data, flags, args)
+          duration = calculate_duration(results)
+
           title = [
             log.friendly_name,
             " #{results.count}".pastel(:bright_black)
+
           ]
 
+          # Append Duration
+          title.push(" #{duration.pastel(:cyan, :dim)}") unless duration.blank?
+
           # Save unless empty
           obj[title.join] = results unless results.count.zero?
 
@@ -180,6 +207,32 @@ module GreenHat
       end
     end
 
+    def self.calculate_duration(results)
+      # Skip for Pluck
+      only_with_time = results.select { |x| x.instance_of?(Hash) && x.key?(:time) }
+
+      # If slice is used ignore
+      return nil if only_with_time.empty?
+
+      sorted = only_with_time.map(&:time).sort
+      humanize_time(sorted.first, sorted.last)
+    end
+
+    # Replace TimeDifference with https://stackoverflow.com/a/4136485/1678507
+    def self.humanize_time(time_start, time_end, increments = 2)
+      miliseconds = (time_end - time_start) * 1000
+
+      list = [[1000, :ms], [60, :s], [60, :m], [24, :h]].map do |count, name|
+        next unless miliseconds.positive?
+
+        miliseconds, n = miliseconds.divmod(count)
+
+        "#{n.to_i}#{name}" unless n.to_i.zero?
+      end
+
+      list.compact.reverse[0..increments - 1].join(' ')
+    end
+
     # Filter Logic
     # TODO: Simplify
     # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
@@ -189,7 +242,7 @@ module GreenHat
       # Experimenting with deep clone
       results = Marshal.load(Marshal.dump(data))
       results.select! do |row|
-        args.send(flags.logic) do |arg|
+        args.send(flags.logic || :all?) do |arg|
           filter_row_key(row, arg, flags)
         end
       end
@@ -255,7 +308,6 @@ module GreenHat
     end
 
     # Filter Start and End Times
-    # rubocop:disable Metrics/MethodLength
     # TODO: This is a bit icky, simplify/dry
     def self.filter_time(results, flags)
       if flags.key?(:start)
@@ -292,7 +344,6 @@ module GreenHat
 
       results
     end
-    # rubocop:enable Metrics/MethodLength
 
     def self.filter_except(results, except)
       # Avoid Empty Results
@@ -370,6 +421,9 @@ module GreenHat
       stats.map do |field|
         occurrences = filter_count_occurrences(results, field, flags)
 
+        # Use Truncate For Long Keys
+        occurrences.transform_keys! { |key| key.to_s[0..flags[:truncate]] } if flags[:truncate]
+
         # Total Occurences
         total = occurrences.values.sum
 
@@ -387,13 +441,6 @@ module GreenHat
         # Append Header / Total with field name
         output.unshift([field.to_s.pastel(:bright_black), total])
 
-        # Use Truncate For Long Keys
-        if flags[:truncate]
-          output.map! do |key, value|
-            [key.to_s[0..flags[:truncate]], value]
-          end
-        end
-
         # Format
         output.to_h
       end
@@ -433,48 +480,83 @@ module GreenHat
     end
 
     # Break out filter row logic into separate method
-
     def self.filter_row_key(row, arg, flags)
       # Ignore Other Logic if Field isn't even included / Full Text Searching
       return false unless row.key?(arg[:field]) || arg[:field] == :text
 
       # Sensitivity Check / Check for Match / Full Text Searching
-      included = if arg[:field] == :text
-                   filter_row_entry(row.to_s, arg, flags)
-                 else
-                   filter_row_entry(row[arg.field].to_s, arg, flags)
-                 end
+      search_data = arg[:field] == :text ? row : row[arg.field]
+      match = filter_row_entry(search_data.to_s, arg, flags)
 
       # Pivot of off include vs exclude
       if arg.bang
-        !included
+        !match
       else
-        included
+        match
       end
     end
 
     # Field Partial / Case / Exact search
     def self.filter_row_entry(entry, arg, flags)
       # Exact Matching / Unless doing full text search
-      return entry.to_s == arg.value.to_s if flags.key?(:exact) && arg.field != :text
+      return entry == arg.value.to_s if flags.key?(:exact) && arg.field != :text
 
-      if flags.key?(:case)
-        entry.include? arg.value.to_s
-      else
-        entry.downcase.include? arg.value.to_s.downcase
+      # Cast to String/Integer Helper
+      entry, value = filter_entry_cast(entry, arg, flags)
+
+      entry.send(arg.logic, value)
+    end
+
+    # Handle casting to strings or integers
+    def self.filter_entry_cast(entry, arg, flags)
+      # Cast to String
+      value = arg.value.to_s
+
+      case arg.logic
+      when :include?
+
+        # Exact Case argument
+        unless flags.key?(:case)
+          entry.downcase!
+          value.downcase!
+        end
+      when :>=, :<=
+        entry = entry.to_i if entry.numeric?
+        value = value.to_i if value&.numeric?
       end
+
+      [entry, value]
+    end
+
+    def self.filter_entry_logic(entry, arg)
+      entry.send(arg.logic, arg.value)
     end
 
     # Total Count Helper
-    def self.total_count(results)
-      results.each do |k, v|
+    def self.total_count(results, flags)
+      results.map do |k, v|
         puts k
-        puts "Total: #{v.count.to_s.pastel(:blue)}"
+        # puts "Total: #{v.count.to_s.pastel(:blue)}"
+        # puts "Time Covered: #{calculate_duration(v).pastel(:cyan)}"
+
+        output = {}
+        output[:total] = v.count
+        output[:duration] = calculate_duration(v)
+
+        # Sort / Get first and Last
+        list = v.select(&:time).map(&:time)
+        unless list.blank?
+          output[:start] = list.first
+          output[:end] = list.last
+        end
+
+        puts render_table(output, flags)
+
         puts
       end
     end
 
-    # Total Count Helper
+    # Table Printer Helper
     def self.fields_print(results)
       results.each do |k, v|
         puts k
@@ -538,7 +620,7 @@ module GreenHat
         return file if file.instance_of?(Thing)
 
         if flags.fuzzy_file_match
-          Thing.all.select { |x| x.name.include? file }
+          Thing.all.select { |x| x.name.include?(file) || x.type.include?(file) }
         else
           Thing.where name: file
         end
@@ -550,62 +632,6 @@ module GreenHat
       things
     end
 
-    # Main Entry Point for Searching
-    # def self.search_start(log_list, filter_type, args, opts)
-    def self.search_start(files, flags, args)
-      # Convert to Things
-      logs = ShellHelper.find_things(files, flags)
-
-      logs.each_with_object({}) do |log, obj|
-        # Ignore Empty Results / No Thing
-        next if log&.data.blank?
-
-        obj[log.friendly_name] = ShellHelper.search(log.data, flags, args)
-
-        obj
-      end
-    end
-
-    # Generic Search Helper / String/Regex
-    def self.search(data, flags = {}, args = {})
-      results = data.clone.flatten.compact
-      results.select! do |row|
-        args.send(flags.logic) do |arg|
-          search_row(row, arg, flags)
-        end
-      end
-
-      # Strip Results if Slice is defined
-      results.map! { |row| row.slice(*flags[:slice]) } if flags[:slice]
-
-      # Strip Results if Except is defined
-      results.map! { |row| row.except(*flags[:except]) } if flags[:except]
-
-      # Remove Blank from either slice or except
-      results.reject!(&:empty?)
-
-      results
-    end
-
-    # Break out filter row logic into separate method
-    def self.search_row(row, arg, flags)
-      # Sensitivity Check / Check for Match
-      included = filter_row_entry(row.to_s, arg, flags)
-
-      # Pivot of off include vs exclude
-      if arg.bang
-        !included
-      else
-        included
-      end
-    end
-
-    # TODO: Remove?
-    # Color Reader Helper
-    # def self.pastel
-    #   @pastel ||= Pastel.new
-    # end
-
     # Number Helper
     # https://gitlab.com/zedtux/human_size_to_number/-/blob/master/lib/human_size_to_number/helper.rb
     def self.human_size_to_number(string)
@@ -627,23 +653,6 @@ module GreenHat
       number.round
     end
 
-    # TODO: Needed?
-    def self.filter_and(data, params = {})
-      result = data.clone.flatten.compact
-      params.each do |k, v|
-        result.select! do |row|
-          if row.key? k.to_sym
-            row[k.to_sym].include? v
-          else
-            false
-          end
-        end
-        next
-      end
-
-      result
-    end
-
     # General Helper for `show`
     def self.common_opts
       puts 'Common Options'.pastel(:blue)

data/lib/greenhat/shell.rb

@@ -7,6 +7,13 @@ module GreenHat
       # rubocop:enable Lint/Debugger
     end
 
+    def self.web
+      # Load Required Files
+      require 'greenhat/web'
+
+      GreenHat::Web.toggle
+    end
+
     def self.df
       Disk.df
     end
@@ -23,12 +30,12 @@ module GreenHat
       Memory.free
     end
 
-    def self.load_local
-      return false unless TTY::Prompt.new.yes?('Load local Omnibus GitLab Instance files?')
+    def self.load_local(prompt = true)
+      return false if prompt && !TTY::Prompt.new.yes?('Load local Omnibus GitLab Instance files?')
 
       archive_path = "#{$TMP}/#{Time.now.to_i}_local"
       Dir.mkdir(archive_path)
-      # archive = Archive.new(name: 'local')
+
       archive = Archive.new(name: archive_path, path: archive_path)
 
       file_list = Dir['/var/log/gitlab/*/current'] + Dir['/var/log/gitlab/*/*.log']