greenhat 0.3.1 → 0.3.5

data/lib/greenhat/shell/markdown.rb

@@ -0,0 +1,344 @@
+module GreenHat
+  # Root Level Shell / Report Helper
+  module Shell
+    def self.markdown_report(raw)
+      _files, flags, _args = Args.parse(raw)
+
+      archives = if flags.archive
+                   Archive.all.select do |archive|
+                     flags.archive.any? { |x| archive.name.include? x.to_s }
+                   end
+                 else
+                   Archive.all
+                 end
+
+      ShellHelper.show(archives.map(&:report_markdown).map(&:show).flatten, flags)
+    end
+  end
+end
+
+module GreenHat
+  # Report Generator Helper
+  # rubocop:disable Metrics/ClassLength
+  class ReportMarkdown
+    include ActionView::Helpers::NumberHelper
+
+    attr_accessor :archive, :host, :os_release, :selinux_status, :cpu, :uname,
+                  :timedatectl, :uptime, :meminfo, :gitlab_manifest, :gitlab_status,
+                  :production_log, :api_log, :application_log, :sidekiq_log,
+                  :exceptions_log, :gitaly_log, :free_m, :disk_free
+
+    # Find Needed Files for Report
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def initialize(archive)
+      self.archive = archive
+      self.host = archive.things.find { |x| x.name == 'hostname' }
+      self.os_release = archive.things.find { |x| x.name == 'etc/os-release' }
+      self.selinux_status = archive.things.find { |x| x.name == 'sestatus' }
+      self.cpu = archive.things.find { |x| x.name == 'lscpu' }
+      self.uname = archive.things.find { |x| x.name == 'uname' }
+      self.timedatectl = archive.things.find { |x| x.name == 'timedatectl' }
+      self.uptime = archive.things.find { |x| x.name == 'uptime' }
+      self.meminfo = archive.things.find { |x| x.name == 'meminfo' }
+      self.free_m = archive.things.find { |x| x.name == 'free_m' }
+      self.gitlab_manifest = archive.things.find { |x| x.name == 'gitlab/version-manifest.json' }
+      self.gitlab_status = archive.things.find { |x| x.name == 'gitlab_status' }
+      self.production_log = archive.things.find { |x| x.name == 'gitlab-rails/production_json.log' }
+      self.api_log = archive.things.find { |x| x.name == 'gitlab-rails/api_json.log' }
+      self.application_log = archive.things.find { |x| x.name == 'gitlab-rails/application_json.log' }
+      self.exceptions_log = archive.things.find { |x| x.name == 'gitlab-rails/exceptions_json.log' }
+      self.gitaly_log = archive.things.find { |x| x.name == 'gitaly/current' }
+      self.sidekiq_log = archive.things.find { |x| x.name == 'sidekiq/current' }
+      self.disk_free = archive.things.find { |x| x.name == 'df_h' }
+    end
+
+    def show
+      output = [
+        archive.friendly_name,
+        ''
+      ]
+
+      # OS
+      output << "**OS**\n"
+
+      output << collect_host
+      output << ''
+
+      # Memory
+      if meminfo || free_m
+        output << "**Memory**\n"
+        # output << memory_perc if meminfo
+        output << memory_free if free_m
+        output << ''
+      end
+
+      # Disk
+      if disk_free
+        output << disks
+        output << ''
+      end
+
+      # Gitlab
+      output << "**GitLab**\n" if gitlab_manifest || gitlab_status
+      output << gitlab_version if gitlab_manifest
+      output << gitlab_services if gitlab_status
+
+      output << ''
+
+      output << "**Errors**\n" if production_log || api_log || application_log || sidekiq_log
+      output << collect_errors
+
+      # Final Space / Return
+      output << ''
+      output
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+    def collect_host
+      output = []
+      output << hostname if host
+      output << distro if os_release
+      output << selinux if selinux_status
+      # output << arch if cpu
+      output << kernel if uname
+      output << sys_time if timedatectl
+      output << sys_uptime if uptime
+      output << load_average if uptime && cpu
+
+      groups = output.each_slice((output.size / 2.to_f).round).to_a
+
+      table = TTY::Table.new do |t|
+        loop do
+          break if groups.all?(&:empty?)
+
+          t << groups.map(&:shift)
+        end
+      end
+
+      "```\n#{table.render(:basic, padding: [0, 2, 0, 0])}\n```"
+    end
+
+    def collect_errors
+      output = []
+      output << production_errors if production_log
+      output << application_errors if application_log
+      output << sidekiq_errors if sidekiq_log
+      output << api_errors if api_log
+      output << exception_errors if exceptions_log
+      output << gitaly_errors if gitaly_log
+
+      # Keep Alphabetical Sort / Allow for only one
+      slice_size = (output.size / 3.to_f).round
+      slice_size = 1 unless slice_size.positive?
+
+      groups = output.each_slice(slice_size).to_a
+
+      table = TTY::Table.new do |t|
+        loop do
+          break if groups.all?(&:empty?)
+
+          t << groups.map(&:shift)
+        end
+      end
+
+      "```\n#{table.render(:basic, padding: [0, 2, 0, 0])}\n```"
+    end
+
+    def exception_errors
+      count = exceptions_log.data.count
+
+      "Exception: #{count}"
+    end
+
+    def gitaly_errors
+      count = gitaly_log.data.count { |x| x.level == 'error' }
+
+      "Gitaly: #{count}"
+    end
+
+    def production_errors
+      count = production_log.data.count { |x| x.status == 500 }
+
+      "Production: #{count}"
+    end
+
+    def api_errors
+      count = api_log.data.count { |x| x.status == 500 }
+
+      "API: #{count}"
+    end
+
+    def application_errors
+      results = ShellHelper.filter_internal([
+        'gitlab-rails/application_json.log',
+        '--message!="Cannot obtain an exclusive lease"',
+        '--severity=error'
+      ].join(' '))
+
+      "Application: #{results.count}"
+    end
+
+    def sidekiq_errors
+      count = sidekiq_log.data.count { |x| x&.severity == 'ERROR' }
+
+      "Sidekiq: #{count}"
+    end
+
+    def gitlab_services
+      [
+        "**Services**\n",
+        "\n",
+        GreenHat::GitLab.services_markdown(archive)
+      ].join
+    rescue StandardError => e
+      LogBot.fatal('GitLab Services', message: e.message, backtrace: e.backtrace.first)
+    end
+
+    def gitlab_version
+      "Version: #{gitlab_manifest.data.build_version}\n"
+    end
+
+    def hostname
+      "Hostname: #{host.data.first}"
+    end
+
+    def distro
+      [
+        "Distro: [#{os_release.data.ID}] ",
+        os_release.data.PRETTY_NAME
+      ].join
+    end
+
+    def selinux
+      status = selinux_status.data['SELinux status']
+
+      [
+        'SeLinux: ',
+        status,
+        ' (',
+        selinux_status.data['Current mode'],
+        ')'
+      ].join
+    end
+
+    def arch
+      [
+        'Arch: ',
+        cpu.data.Architecture
+      ].join
+    end
+
+    def kernel
+      # TODO: Better way to consistently get uname info?
+      value, build = uname.data.first.split[2].split('-')
+      [
+        'Kernel: ',
+        value,
+        " (#{build})"
+      ].join
+    end
+
+    # Helper for finding if NTP is enabled
+    def ntp_keys
+      [
+        'Network time on', 'NTP enabled', 'NTP service', 'System clock synchronized'
+      ]
+    end
+
+    def sys_time
+      # Ignore if Empty
+      return false if timedatectl.data.nil?
+
+      ntp_statuses = timedatectl.data.slice(*ntp_keys).values.compact
+
+      ntp_status = ntp_statuses.first
+
+      # Fall Back
+      ntp_status ||= 'unknown'
+
+      [
+        'Sys Time: ',
+        timedatectl.data['Local time'],
+        "(ntp: #{ntp_status})"
+      ].join
+    end
+
+    # Strip/Simplify Uptime
+    def sys_uptime
+      init = uptime.data.first.split(',  load average').first.strip
+
+      "Uptime: #{init.split('up ', 2).last}"
+    end
+
+    def load_average
+      cpu_count = cpu.data['CPU(s)'].to_i
+      intervals = uptime.data.first.split('load average: ', 2).last.split(', ').map(&:to_f)
+
+      # Generate Colorized Text for Output
+      intervals_text = intervals.map do |interval|
+        value = percent(interval, cpu_count)
+
+        "#{interval} (#{value}%)"
+      end
+
+      [
+        'LoadAvg: ',
+        "[CPU #{cpu_count}] ",
+        intervals_text.join(', ')
+      ].join
+    end
+
+    def memory_free
+      free = free_m.data.find { |x| x.kind == 'Mem' }
+
+      return unless free
+
+      pad = 6
+      list = [
+        "#{title('Total', pad)} #{number_to_human_size(free.total.to_i * 1024**2)}",
+        "#{title('Used', pad)} #{number_to_human_size(free.used.to_i * 1024**2)}",
+        "#{title('Free', pad)} #{number_to_human_size(free.free.to_i * 1024**2)}",
+        "#{title('Avail', pad)} #{number_to_human_size(free.available.to_i * 1024**2)}"
+      ]
+
+      # Keep Alphabetical Sort
+      groups = list.each_slice((list.size / 2.to_f).round).to_a
+
+      table = TTY::Table.new do |t|
+        loop do
+          break if groups.all?(&:empty?)
+
+          t << groups.map(&:shift)
+        end
+      end
+
+      "```\n#{table.render(:basic, padding: [0, 2, 0, 0])}\n```"
+    end
+
+    def disks
+      # GreenHat::Disk.df({archive: []})
+      file = GreenHat::Disk.df({ archive: [archive.name] })
+
+      disk_list = GreenHat::Disk.markdown_format(file.first, false, 3)
+
+      # Preapre / Indent List
+      [
+        '**Disks**',
+        "\n\n```\n#{disk_list.join("\n")}\n```"
+      ].join
+    end
+
+    # ----------------------------
+    # Helpers
+    # ----------------------------
+    def percent(value, total)
+      ((value / total.to_f) * 100).round
+    end
+
+    # Helper to Make Cyan Titles
+    def title(name, ljust = 16)
+      "#{name}:".ljust(ljust)
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/greenhat/shell/report.rb

@@ -25,7 +25,8 @@ module GreenHat
 
     attr_accessor :archive, :host, :os_release, :selinux_status, :cpu, :uname,
                   :timedatectl, :uptime, :meminfo, :gitlab_manifest, :gitlab_status,
-                  :production_log, :api_log, :application_log, :sidekiq_log, :free_m, :disk_free
+                  :production_log, :api_log, :application_log, :sidekiq_log,
+                  :exceptions_log, :gitaly_log, :free_m, :disk_free
 
     # Find Needed Files for Report
     # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
@@ -45,6 +46,8 @@ module GreenHat
       self.production_log = archive.things.find { |x| x.name == 'gitlab-rails/production_json.log' }
       self.api_log = archive.things.find { |x| x.name == 'gitlab-rails/api_json.log' }
       self.application_log = archive.things.find { |x| x.name == 'gitlab-rails/application_json.log' }
+      self.exceptions_log = archive.things.find { |x| x.name == 'gitlab-rails/exceptions_json.log' }
+      self.gitaly_log = archive.things.find { |x| x.name == 'gitaly/current' }
       self.sidekiq_log = archive.things.find { |x| x.name == 'sidekiq/current' }
       self.disk_free = archive.things.find { |x| x.name == 'df_h' }
     end
@@ -86,9 +89,11 @@ module GreenHat
       output << gitlab_services if gitlab_status
       output << title('Errors') if production_log || api_log || application_log || sidekiq_log
       output << production_errors if production_log
-      output << api_errors if api_log
       output << application_errors if application_log
       output << sidekiq_errors if sidekiq_log
+      output << api_errors if api_log
+      output << exception_errors if exceptions_log
+      output << gitaly_errors if gitaly_log
 
       # Final Space / Return
       output << ''
@@ -96,6 +101,26 @@ module GreenHat
     end
     # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
 
+    def exception_errors
+      count = exceptions_log.data.count
+      color = count.zero? ? :green : :red
+
+      [
+        title('  Exception', :bright_red, 18),
+        count.to_s.pastel(color)
+      ].join
+    end
+
+    def gitaly_errors
+      count = gitaly_log.data.count { |x| x.level == 'error' }
+      color = count.zero? ? :green : :red
+
+      [
+        title('  Gitaly', :bright_red, 18),
+        count.to_s.pastel(color)
+      ].join
+    end
+
     def production_errors
       count = production_log.data.count { |x| x.status == 500 }
       color = count.zero? ? :green : :red
@@ -117,7 +142,13 @@ module GreenHat
     end
 
     def application_errors
-      count = application_log.data.count { |x| x&.severity == 'ERROR' }
+      results = ShellHelper.filter_internal([
+        'gitlab-rails/application_json.log',
+        '--message!="Cannot obtain an exclusive lease"',
+        '--severity=error'
+      ].join(' '))
+
+      count = results.count { |x| x&.severity == 'ERROR' }
       color = count.zero? ? :green : :red
 
       [
@@ -275,6 +306,7 @@ module GreenHat
       ].join
     end
 
+    # rubocop:disable Metrics/MethodLength
     def memory_free
       free = free_m.data.find { |x| x.kind == 'Mem' }
 
@@ -282,22 +314,39 @@ module GreenHat
 
       formatted_mem = free_m.data.map { |x| GreenHat::Memory.memory_row x }
 
-      [
-        title('Total', :cyan, 14),
-        number_to_human_size(free.total.to_i * 1024**2),
-        "\n",
-        title('Used', :yellow, 14),
-        number_to_human_size(free.used.to_i * 1024**2),
-        "\n",
-        title('Free', :blue, 14),
-        number_to_human_size(free.free.to_i * 1024**2),
-        "\n",
-        title('Available', :green, 14),
-        number_to_human_size(free.available.to_i * 1024**2),
-        "\n\n",
-        formatted_mem.map { |x| x.prepend ' ' * 2 }.join("\n")
-      ].join
+      output = []
+      unless free.total.blank?
+        output << title('Total', :cyan, 14)
+        output << number_to_human_size(free.total.to_i * 1024**2)
+        output << "\n"
+      end
+
+      unless free.total.blank?
+        output << title('Used', :yellow, 14)
+        output << number_to_human_size(free.used.to_i * 1024**2)
+        output << "\n"
+      end
+
+      unless free.total.blank?
+        output << title('Free', :blue, 14)
+        output << number_to_human_size(free.free.to_i * 1024**2)
+        output << "\n"
+      end
+
+      unless free.total.blank?
+        output << title('Available', :green, 14)
+        output << number_to_human_size(free.available.to_i * 1024**2)
+        output << "\n"
+      end
+
+      output << "\n"
+      output << formatted_mem.map { |x| x.prepend ' ' * 2 }.join("\n")
+
+      output.join
+    rescue StandardError => e
+      LogBot.fatal('Memory', message: e.message, backtrace: e.backtrace.first)
     end
+    # rubocop:enable Metrics/MethodLength
 
     def disks
       # GreenHat::Disk.df({archive: []})

data/lib/greenhat/shell/shell_helper.rb

@@ -120,10 +120,12 @@ module GreenHat
       entry = entry.map { |k, v| [k, format_table_entry(flags, v, k)] }.to_h
       # Pre-format Entry
 
+      table_style = flags[:table_style]&.to_sym || :unicode
+
       table = TTY::Table.new(header: entry.keys, rows: [entry], orientation: :vertical)
 
       LogBot.debug('Rendering Entries') if ENV['DEBUG']
-      table.render(:unicode, padding: [0, 1, 0, 1], multiline: true) do |renderer|
+      table.render(table_style, padding: [0, 1, 0, 1], multiline: true) do |renderer|
         renderer.border.style = :cyan
       end
 
@@ -148,8 +150,18 @@ module GreenHat
       format_table_entry(flags, val)
     end
 
+    # Internal Query Helper
+    # query = 'gitlab-rails/application_json.log --message!="Cannot obtain an exclusive lease" --severity=error'
+    # ShellHelper.filter_internal(query)
+    def self.filter_internal(search = '')
+      files, flags, args = Args.parse(Shellwords.split(search))
+      flags[:combine] = true
+
+      ShellHelper.filter_start(files, flags, args)
+    end
+
     # Main Entry Point for Filtering
-    def self.filter_start(files, flags, args)
+    def self.filter_start(files, flags = {}, args = {})
       # Convert to Things
       logs = ShellHelper.find_things(files, flags).select(&:processed?)
 
@@ -182,9 +194,12 @@ module GreenHat
     # TODO: Simplify
     # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
     def self.filter(data, flags = {}, args = {})
-      results = data.clone.flatten.compact
+      # results = data.clone.flatten.compact
+
+      # Experimenting with deep clone
+      results = Marshal.load(Marshal.dump(data))
       results.select! do |row|
-        args.send(flags.logic) do |arg|
+        args.send(flags.logic || :all?) do |arg|
           filter_row_key(row, arg, flags)
         end
       end
@@ -192,6 +207,9 @@ module GreenHat
       # Ensure presecense of a specific field
       results = filter_exists(results, flags[:exists]) if flags.key?(:exists)
 
+      # Time Zone
+      results = filter_modify_timezone(results, flags[:time_zone]) if flags.key?(:time_zone)
+
       # Time Filtering
       results = filter_time(results, flags) if flags.key?(:start) || flags.key?(:end)
 
@@ -217,24 +235,34 @@ module GreenHat
       results.reverse! if flags[:reverse]
 
       # Count occurrences / Skip Results
-      return filter_stats(results, flags[:stats]) if flags.key?(:stats)
+      return filter_stats(results, flags) if flags.key?(:stats)
+
+      # Limit before Pluck / Flattening
+      results = filter_limit(results, flags[:limit]) if flags.key?(:limit)
 
       # Pluck
       results = filter_pluck(results, flags[:pluck]) if flags.key?(:pluck)
 
-      # Limit / Ensure Exists and Valid Number
-      if flags.key?(:limit) && flags[:limit]
-        # Old
-        # results[0..flags[:limit].map(&:to_s).join.to_i - 1]
+      results
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
-        # New
-        results.shift flags[:limit]
+    # Limit / Ensure Exists and Valid Number
+    def self.filter_limit(results, limit)
+      return results unless limit.integer? && limit.positive?
 
-      else
-        results
+      results.shift limit
+    end
+
+    def self.filter_modify_timezone(results, time_zone)
+      results.each do |x|
+        next unless x.key? :time
+
+        x[:time] = x[:time].in_time_zone time_zone
       end
+
+      results
     end
-    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
     # Filter Start and End Times
     # rubocop:disable Metrics/MethodLength
@@ -339,7 +367,9 @@ module GreenHat
       end.inject(:&)
     end
 
-    def self.filter_stats(results, stats)
+    def self.filter_stats(results, flags)
+      stats = flags[:stats]
+
       # Avoid Empty Results
       if stats.empty?
         filter_empty_arg('stats')
@@ -348,7 +378,7 @@ module GreenHat
 
       # Loop through Stats, Separate Hash/Tables
       stats.map do |field|
-        occurrences = filter_count_occurrences(results, field)
+        occurrences = filter_count_occurrences(results, field, flags)
 
         # Total Occurences
         total = occurrences.values.sum
@@ -367,6 +397,13 @@ module GreenHat
         # Append Header / Total with field name
         output.unshift([field.to_s.pastel(:bright_black), total])
 
+        # Use Truncate For Long Keys
+        if flags[:truncate]
+          output.map! do |key, value|
+            [key.to_s[0..flags[:truncate]], value]
+          end
+        end
+
         # Format
         output.to_h
       end
@@ -378,10 +415,17 @@ module GreenHat
     end
 
     # Helper to Count occurrences
-    def self.filter_count_occurrences(results, field)
+    def self.filter_count_occurrences(results, field, flags = {})
       results.each_with_object(Hash.new(0)) do |entry, counts|
         if entry.key? field
-          counts[entry[field]] += 1
+          # Rounding in pagination breaks stats
+          key = if flags.key?(:round) && entry[field].numeric?
+                  entry[field].to_f.round(flags.round)
+                else
+                  entry[field]
+                end
+
+          counts[key] += 1
         else
           counts['None'.pastel(:bright_black)] += 1
         end
@@ -440,6 +484,32 @@ module GreenHat
       end
     end
 
+    # Total Count Helper
+    def self.fields_print(results)
+      results.each do |k, v|
+        puts k
+        puts field_table(v.map(&:keys).flatten.uniq.sort)
+        puts
+      end
+    end
+
+    def self.field_table(list, columns = 4)
+      return nil if list.size.zero?
+
+      # Keep Alphabetical Sort
+      groups = list.each_slice((list.size / columns.to_f).round).to_a
+
+      table = TTY::Table.new do |t|
+        loop do
+          break if groups.all?(&:empty?)
+
+          t << groups.map(&:shift)
+        end
+      end
+
+      table.render(:unicode, padding: [0, 1, 0, 1])
+    end
+
     # Unified Files Interface
     def self.files(file_list, base_list = nil, flags = {})
       base_list ||= Thing.all