greenhat 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac7b0e70e5a13dc6f8f88a4f12fec812e38e7d838537abe83b7d4dc2e8b43663
-  data.tar.gz: 84246bc31bd38af74e9f4f8d95f2f7ffa1ca33978dc0d0862ec0b50eaff20196
+  metadata.gz: 4c055abc7397135be9ca07e7965fc88d67f1d7d16099a5cc5db1977986490361
+  data.tar.gz: 2e2f66a3bbc5bdc84e8b5bef9abfa28706f48841f25c7e642fe352c063745e22
 SHA512:
-  metadata.gz: ba104ea8d7b4e52babf8bdf27ea4f3dbdb9e92820589881bab639a01e8558c355389a0f10b3c17dbe5c7ac881f10364ad06ec0c973934f06d735f948bb57fbd3
-  data.tar.gz: a2013229826758ca61335f5cf8d58c3a4a4206e656dce084f90781c264e4930c5c3fd09b7475a43dd31819bc2562e85e5799de775cc5db9dfaa3f73c0228ac3a
+  metadata.gz: d9fcd1274e4887fa332a053185ea1049e5db2f3d449714542a31084234326d48623e95e0fb8678b6cb67b20b636ec04c82db1555ff43aac09f158d6df4fc7dc0
+  data.tar.gz: c6fe018479b7cc90f2e70b411761fe5e3b8da5d4237064e3bb450399f966cf00a0302adc433ebf39fad57565ca11efbd95fbd0601a15f2989bc1c6455029a4e8

data/lib/greenhat/accessors/disk.rb

@@ -79,5 +79,46 @@ module GreenHat
       output
     end
     # rubocop:enable Metrics/MethodLength
+
+    # Unified Output Handler
+    def self.markdown_format(file, name = false, limit = nil, filter = %w[tmpfs loop])
+      output = []
+
+      output << file.friendly_name if name
+
+      # Reject TMPFS
+      disks = file.data.sort_by { |x| x.use.to_i }.reverse
+
+      # Filter
+      disks.reject! { |x| filter.any? { |y| x.filesystem.include? y } }
+
+      disks = disks[0..limit - 1] if limit
+
+      pad_mount, pad_size, pad_used, pad_avail = GreenHat::Disk.padding(disks)
+
+      # Headers
+      output << [
+        'Mount'.ljust(pad_mount),
+        'Size'.ljust(pad_size),
+        'Used'.ljust(pad_used),
+        'Avail'.ljust(pad_avail),
+        '% Use'.ljust(pad_avail)
+      ].join
+
+      # Table Summary
+      disks.map do |disk|
+        # Whole Thing
+        output << [
+          disk.mounted_on.ljust(pad_mount),
+          disk[:size].to_s.ljust(pad_size),
+          disk.used.to_s.ljust(pad_used),
+          disk.avail.to_s.ljust(pad_avail),
+          disk.use.rjust(5).ljust(5)
+        ].join
+      end
+
+      output
+    end
+    # =
   end
 end

data/lib/greenhat/accessors/gitlab.rb

@@ -71,5 +71,31 @@ module GreenHat
 
       table.render(:unicode, padding: [0, 1, 0, 1], indent: indent)
     end
+
+    def self.services_markdown(archive)
+      # manifest = archive.things.find { |x| x.type == 'gitlab/version-manifest.json' }
+      gitlab_status = archive.things.find { |x| x.name == 'gitlab_status' }
+
+      list = gitlab_status.data.keys.sort.map do |service|
+        status = gitlab_status.data.dig(service, 0, :status) == 'run' ? '↑' : '!'
+
+        "#{service}#{status}"
+      end
+
+      # Keep Alphabetical Sort
+      groups = list.each_slice((list.size / 3.to_f).round).to_a
+
+      table = TTY::Table.new do |t|
+        loop do
+          break if groups.all?(&:empty?)
+
+          t << groups.map(&:shift)
+        end
+      end
+
+      "```\n#{table.render(:basic)}\n```"
+    end
+
+    # ==
   end
 end

data/lib/greenhat/archive.rb

@@ -105,4 +105,8 @@ class Archive < Teron
   def report
     GreenHat::Report.new(self)
   end
+
+  def report_markdown
+    GreenHat::ReportMarkdown.new(self)
+  end
 end

data/lib/greenhat/cli.rb

@@ -420,6 +420,7 @@ module GreenHat
 
       # CTL Tails need to be parsed for new 'things'
       Thing.where(kind: :gitlab_tail)&.map(&:process)
+      Thing.where(kind: :kube_webservice)&.map(&:process)
 
       Thing.all.each(&:process) if flags?(%i[load l], flags)
     end

data/lib/greenhat/shell/filter_help.rb

@@ -2,8 +2,8 @@ module GreenHat
   # CLI Helper
   module ShellHelper
     # Unify Filter / Filter Help
+    # rubocop:disable Metrics/MethodLength,Metrics/ModuleLength
     module Filter
-      # rubocop:disable Metrics/MethodLength
       def self.help
         puts "\u2500".pastel(:cyan) * 20
         puts 'Filter'.pastel(:yellow)
@@ -156,7 +156,208 @@ module GreenHat
 
         puts
       end
-      # rubocop:enable Metrics/MethodLength
+
+      def self.help_index
+        {
+          title: [
+            "\u2500".pastel(:cyan) * 20,
+            'Filter'.pastel(:yellow),
+            "\u2500".pastel(:cyan) * 20
+          ],
+          options: [
+            'Options'.pastel(:blue)
+          ],
+
+          raw: [
+            '--raw'.pastel(:green),
+            '  Disable formatting and page/less'
+          ],
+
+          page: [
+            '--page'.pastel(:green),
+            '  Specifically enable or disable paging',
+            '  E.g. --page (default to true), --page=true, --page=false'
+          ],
+
+          round: ['--round'.pastel(:green),
+                  '  Attempt to round all integers. Default: 2.',
+                  '  E.g. --round, --round=3, --round=0'],
+
+          limit: [
+
+            '--limit'.pastel(:green),
+            '  Limit total output lines. Disabled by default. Default without value is based on screen height',
+            '  E.g. --limit, --limit=5'
+          ],
+
+          json: [
+            '--json'.pastel(:green),
+            '  Print output back into JSON'
+          ],
+
+          or: [
+            '--or'.pastel(:green),
+            '  Filters will use OR instead of AND (all match vs any match)'
+          ],
+
+          total: [
+            '--total'.pastel(:green),
+            '  Print only total count of matching entries'
+          ],
+
+          fields: [
+            '--fields'.pastel(:green),
+            '  Print only Available fields for selected files'
+          ],
+
+          slice: [
+            '--slice'.pastel(:green),
+            '  Extract specific fields from entries (slice multiple with comma)',
+            '  Ex: --slice=path or --slice=path,params'
+
+          ],
+
+          except: [
+            '--except'.pastel(:green),
+            '  Exclude specific fields (except multiple with comma)',
+            '  Ex: --except=params --except=params,path'
+
+          ],
+
+          exists: [
+            '--exists'.pastel(:green),
+            '  Ensure field exists regardless of contents',
+            '  Ex: --exists=params --exists=params,path'
+
+          ],
+
+          stats: [
+            '--stats'.pastel(:green),
+            '  Order/Count occurrances by field. Combine with `truncate` for key names',
+            '  Ex: --stats=params --except=params,path'
+
+          ],
+
+          uniq: [
+            '--uniq'.pastel(:green),
+            '  Show unique values only',
+            '  Ex: --uniq=params --uniq=params,path'
+
+          ],
+
+          pluck: [
+            '--pluck'.pastel(:green),
+            '  Extract values from entries',
+            '  Ex: --pluck=params --pluck=params,path'
+
+          ],
+
+          archive: [
+            '--archive'.pastel(:green),
+            '  Limit to specific archvie name (partial matching /inclusive). Matching SOS tar.gz name',
+            '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
+
+          ],
+
+          sort: [
+            '--sort'.pastel(:green),
+            '  Sort by multiple fields',
+            '  Ex: --sort=duration_s,db_duration_s'
+
+          ],
+
+          reverse: [
+            '--reverse'.pastel(:green),
+            '  Reverse all results',
+            '  Ex: --reverse'
+
+          ],
+
+          combine: [
+            '--combine'.pastel(:green),
+            '  Omit archive identifier dividers. Useful with sort or time filters',
+            '  Ex: --combine'
+
+          ],
+
+          case: [
+            '--case'.pastel(:green),
+            '  Exact case match. Defaults to case insensitive',
+            '  Ex: --case; --name=Jon, --name=jane --case'
+
+          ],
+
+          exact: [
+            '--exact'.pastel(:green),
+            '  Exact parameter/value match. Defaults to partial match',
+            '  Ex: --field=CommonPartial --exact'
+
+          ],
+
+          start: [
+            '--start'.pastel(:green),
+            '  Show events after specified time. Filtered by the `time` field',
+            '  Use with `--end` for between selections',
+            '  Ex: log filter --start="2021-06-22 14:44  UTC" --end="2021-06-22 14:45 UTC"'
+
+          ],
+
+          end: [
+            '--end'.pastel(:green),
+            '  Show events before specified time. Filtered by the `time` field',
+            '  Use with `--start` for between selections',
+            '  Ex: log filter --end="2021-06-22"'
+          ],
+
+          time_zone: [
+            '--time_zone'.pastel(:green),
+            '  Manipulate the `time` field into a specific timezone',
+            '  Ex: log filter --time_zone=EDT'
+
+          ],
+
+          text: [
+            '--text'.pastel(:green),
+            '  Full entry text searching (slow)',
+            '    --text="anything here"'
+          ],
+
+          table_style: [
+            '--table_style'.pastel(:green),
+            '  Renderer used for formatted output. basic, ascii, or unicode(default)',
+            '  Ex: log filter --table_style=base'
+          ],
+
+          truncate: [
+            '--truncate'.pastel(:green),
+            '  Truncate field length. On by default (4 rows). Performance issues!',
+            '  Disable with --truncate=0'.pastel(:bright_red),
+            '  Ex: --truncate=200, --truncate=2048"'
+          ],
+
+          field: [
+            'Field Searching'.pastel(:blue),
+            '  --[key]=[value]',
+            '  Search in key for value',
+            '  Example: --path=mirror/pull'
+          ],
+
+          ls: [
+            'Search specific logs'.pastel(:blue),
+            '  Any non dash parameters will be the log list to search from',
+            "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.pastel(:yellow)}` for log names"
+          ],
+
+          examples: [
+            'Example Queries'.pastel(:blue),
+            "  Also see #{'examples'.pastel(:bright_blue)} for even more examples",
+            '  log filter --class=BuildFinishedWorker sidekiq/current --slice=time,message',
+            '  log filter gitlab-rails/api_json.log --slice=ua --uniq=ua --ua=gitlab-runner'
+          ]
+
+        }
+      end
+      # rubocop:enable Metrics/MethodLength,Metrics/ModuleLength
     end
   end
 end

data/lib/greenhat/shell/log.rb

@@ -47,6 +47,7 @@ module GreenHat
         puts '  filter'.pastel(:green)
         puts "    Primary way for log searching within greenhat. See #{'filter_help'.pastel(:blue)}"
         puts '    Time, round, slice/except, and/or, stats, uniq, sort'
+        puts "    #{'filter_help'.pastel(:blue)} supports filtering Ex: #{'filter_help stats'.pastel(:blue)}"
         puts
 
         puts '  show'.pastel(:green)
@@ -62,8 +63,16 @@ module GreenHat
         puts "See #{'examples'.pastel(:bright_blue)} for query examples"
       end
 
-      def self.filter_help
-        ShellHelper::Filter.help
+      def self.filter_help(args = {})
+        if args.empty?
+          ShellHelper::Filter.help
+        else
+          list = ShellHelper::Filter.help_index.select do |k, _v|
+            k.to_s.include? args.first
+          end
+
+          puts list.values.map { |x| x.join("\n") }.join("\n\n")
+        end
       end
 
       def self.ls(args = [])

data/lib/greenhat/shell/markdown.rb

@@ -0,0 +1,356 @@
+module GreenHat
+  # Root Level Shell / Report Helper
+  module Shell
+    def self.markdown_report(raw)
+      _files, flags, _args = Args.parse(raw)
+
+      archives = if flags.archive
+                   Archive.all.select do |archive|
+                     flags.archive.any? { |x| archive.name.include? x.to_s }
+                   end
+                 else
+                   Archive.all
+                 end
+
+      ShellHelper.show(archives.map(&:report_markdown).map(&:show).flatten, flags)
+    end
+  end
+end
+
+module GreenHat
+  # Report Generator Helper
+  # rubocop:disable Metrics/ClassLength
+  class ReportMarkdown
+    include ActionView::Helpers::NumberHelper
+
+    attr_accessor :archive, :host, :os_release, :selinux_status, :cpu, :uname,
+                  :timedatectl, :uptime, :meminfo, :gitlab_manifest, :gitlab_status,
+                  :production_log, :api_log, :application_log, :sidekiq_log,
+                  :exceptions_log, :gitaly_log, :free_m, :disk_free
+
+    # Find Needed Files for Report
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+    def initialize(archive)
+      self.archive = archive
+      self.host = archive.things.find { |x| x.name == 'hostname' }
+      self.os_release = archive.things.find { |x| x.name == 'etc/os-release' }
+      self.selinux_status = archive.things.find { |x| x.name == 'sestatus' }
+      self.cpu = archive.things.find { |x| x.name == 'lscpu' }
+      self.uname = archive.things.find { |x| x.name == 'uname' }
+      self.timedatectl = archive.things.find { |x| x.name == 'timedatectl' }
+      self.uptime = archive.things.find { |x| x.name == 'uptime' }
+      self.meminfo = archive.things.find { |x| x.name == 'meminfo' }
+      self.free_m = archive.things.find { |x| x.name == 'free_m' }
+      self.gitlab_manifest = archive.things.find { |x| x.name == 'gitlab/version-manifest.json' }
+      self.gitlab_status = archive.things.find { |x| x.name == 'gitlab_status' }
+      self.production_log = archive.things.find { |x| x.name == 'gitlab-rails/production_json.log' }
+      self.api_log = archive.things.find { |x| x.name == 'gitlab-rails/api_json.log' }
+      self.application_log = archive.things.find { |x| x.name == 'gitlab-rails/application_json.log' }
+      self.exceptions_log = archive.things.find { |x| x.name == 'gitlab-rails/exceptions_json.log' }
+      self.gitaly_log = archive.things.find { |x| x.name == 'gitaly/current' }
+      self.sidekiq_log = archive.things.find { |x| x.name == 'sidekiq/current' }
+      self.disk_free = archive.things.find { |x| x.name == 'df_h' }
+    end
+
+    def show
+      output = [
+        archive.friendly_name,
+        ''
+      ]
+
+      # OS
+      output << "**OS**\n"
+
+      output << collect_host
+      output << ''
+
+      # Memory
+      if meminfo || free_m
+        output << "**Memory**\n"
+        # output << memory_perc if meminfo
+        output << memory_free if free_m
+        output << ''
+      end
+
+      # Disk
+      if disk_free
+        output << disks
+        output << ''
+      end
+
+      # Gitlab
+      output << "**GitLab**\n" if gitlab_manifest || gitlab_status
+      output << gitlab_version if gitlab_manifest
+      output << gitlab_services if gitlab_status
+
+      output << ''
+
+      output << "**Errors**\n" if production_log || api_log || application_log || sidekiq_log
+      output << collect_errors
+
+      # Final Space / Return
+      output << ''
+      output
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+    def collect_host
+      output = []
+      output << hostname if host
+      output << distro if os_release
+      output << selinux if selinux_status
+      # output << arch if cpu
+      output << kernel if uname
+      output << sys_time if timedatectl
+      output << sys_uptime if uptime
+      output << load_average if uptime && cpu
+
+      groups = output.each_slice((output.size / 2.to_f).round).to_a
+
+      table = TTY::Table.new do |t|
+        loop do
+          break if groups.all?(&:empty?)
+
+          t << groups.map(&:shift)
+        end
+      end
+
+      "```\n#{table.render(:basic, padding: [0, 2, 0, 0])}\n```"
+    end
+
+    def collect_errors
+      output = []
+      output << production_errors if production_log
+      output << application_errors if application_log
+      output << sidekiq_errors if sidekiq_log
+      output << api_errors if api_log
+      output << exception_errors if exceptions_log
+      output << gitaly_errors if gitaly_log
+
+      # Keep Alphabetical Sort
+      groups = output.each_slice((output.size / 3.to_f).round).to_a
+
+      table = TTY::Table.new do |t|
+        loop do
+          break if groups.all?(&:empty?)
+
+          t << groups.map(&:shift)
+        end
+      end
+
+      "```\n#{table.render(:basic, padding: [0, 2, 0, 0])}\n```"
+    end
+
+    def exception_errors
+      count = exceptions_log.data.count
+
+      "Exception: #{count}"
+    end
+
+    def gitaly_errors
+      count = gitaly_log.data.count { |x| x.level == 'error' }
+
+      "Gitaly: #{count}"
+    end
+
+    def production_errors
+      count = production_log.data.count { |x| x.status == 500 }
+
+      "Production: #{count}"
+    end
+
+    def api_errors
+      count = api_log.data.count { |x| x.status == 500 }
+
+      "API: #{count}"
+    end
+
+    def application_errors
+      results = ShellHelper.filter_internal([
+        'gitlab-rails/application_json.log',
+        '--message!="Cannot obtain an exclusive lease"',
+        '--severity=error'
+      ].join(' '))
+
+      "Application: #{results.count}"
+    end
+
+    def sidekiq_errors
+      count = sidekiq_log.data.count { |x| x&.severity == 'ERROR' }
+
+      "Sidekiq: #{count}"
+    end
+
+    def gitlab_services
+      [
+        "**Services**\n",
+        "\n",
+        GreenHat::GitLab.services_markdown(archive)
+      ].join
+    rescue StandardError => e
+      LogBot.fatal('GitLab Services', message: e.message, backtrace: e.backtrace.first)
+    end
+
+    def gitlab_version
+      "Version: #{gitlab_manifest.data.build_version}\n"
+    end
+
+    def hostname
+      "Hostname: #{host.data.first}"
+    end
+
+    def distro
+      [
+        "Distro: [#{os_release.data.ID}] ",
+        os_release.data.PRETTY_NAME
+      ].join
+    end
+
+    def selinux
+      status = selinux_status.data['SELinux status']
+
+      [
+        'SeLinux: ',
+        status,
+        ' (',
+        selinux_status.data['Current mode'],
+        ')'
+      ].join
+    end
+
+    def arch
+      [
+        'Arch: ',
+        cpu.data.Architecture
+      ].join
+    end
+
+    def kernel
+      # TODO: Better way to consistently get uname info?
+      value, build = uname.data.first.split[2].split('-')
+      [
+        'Kernel: ',
+        value,
+        " (#{build})"
+      ].join
+    end
+
+    # Helper for finding if NTP is enabled
+    def ntp_keys
+      [
+        'Network time on', 'NTP enabled', 'NTP service', 'System clock synchronized'
+      ]
+    end
+
+    def sys_time
+      # Ignore if Empty
+      return false if timedatectl.data.nil?
+
+      ntp_statuses = timedatectl.data.slice(*ntp_keys).values.compact
+
+      ntp_status = ntp_statuses.first
+
+      # Fall Back
+      ntp_status ||= 'unknown'
+
+      [
+        'Sys Time: ',
+        timedatectl.data['Local time'],
+        "(ntp: #{ntp_status})"
+      ].join
+    end
+
+    # Strip/Simplify Uptime
+    def sys_uptime
+      init = uptime.data.first.split(',  load average').first.strip
+
+      "Uptime: #{init.split('up ', 2).last}"
+    end
+
+    def load_average
+      cpu_count = cpu.data['CPU(s)'].to_i
+      intervals = uptime.data.first.split('load average: ', 2).last.split(', ').map(&:to_f)
+
+      # Generate Colorized Text for Output
+      intervals_text = intervals.map do |interval|
+        value = percent(interval, cpu_count)
+
+        "#{interval} (#{value}%)"
+      end
+
+      [
+        'LoadAvg: ',
+        "[CPU #{cpu_count}] ",
+        intervals_text.join(', ')
+      ].join
+    end
+
+    # def memory_perc
+    #   total = ShellHelper.human_size_to_number(meminfo.data['MemTotal'])
+    #   free = ShellHelper.human_size_to_number(meminfo.data['MemFree'])
+    #   used = percent((total - free), total)
+
+    #   [
+    #     title('Usage'),
+    #     '  ['.pastel(:bright_black),
+    #     '='.pastel(:green) * (used / 2),
+    #     ' ' * (50 - used / 2),
+    #     ']'.pastel(:bright_black),
+    #     " #{100 - percent(free, total)}%".pastel(:green) # Inverse
+    #   ].join
+    # end
+
+    def memory_free
+      free = free_m.data.find { |x| x.kind == 'Mem' }
+
+      return unless free
+
+      pad = 6
+      list = [
+        "#{title('Total', pad)} #{number_to_human_size(free.total.to_i * 1024**2)}",
+        "#{title('Used', pad)} #{number_to_human_size(free.used.to_i * 1024**2)}",
+        "#{title('Free', pad)} #{number_to_human_size(free.free.to_i * 1024**2)}",
+        "#{title('Avail', pad)} #{number_to_human_size(free.available.to_i * 1024**2)}"
+      ]
+
+      # Keep Alphabetical Sort
+      groups = list.each_slice((list.size / 2.to_f).round).to_a
+
+      table = TTY::Table.new do |t|
+        loop do
+          break if groups.all?(&:empty?)
+
+          t << groups.map(&:shift)
+        end
+      end
+
+      "```\n#{table.render(:basic, padding: [0, 2, 0, 0])}\n```"
+    end
+
+    def disks
+      # GreenHat::Disk.df({archive: []})
+      file = GreenHat::Disk.df({ archive: [archive.name] })
+
+      disk_list = GreenHat::Disk.markdown_format(file.first, false, 3)
+
+      # Preapre / Indent List
+      [
+        '**Disks**',
+        "\n\n```\n#{disk_list.join("\n")}\n```"
+      ].join
+    end
+
+    # ----------------------------
+    # Helpers
+    # ----------------------------
+    def percent(value, total)
+      ((value / total.to_f) * 100).round
+    end
+
+    # Helper to Make Cyan Titles
+    def title(name, ljust = 16)
+      "#{name}:".ljust(ljust)
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/greenhat/shell/report.rb

@@ -25,7 +25,8 @@ module GreenHat
 
     attr_accessor :archive, :host, :os_release, :selinux_status, :cpu, :uname,
                   :timedatectl, :uptime, :meminfo, :gitlab_manifest, :gitlab_status,
-                  :production_log, :api_log, :application_log, :sidekiq_log, :free_m, :disk_free
+                  :production_log, :api_log, :application_log, :sidekiq_log,
+                  :exceptions_log, :gitaly_log, :free_m, :disk_free
 
     # Find Needed Files for Report
     # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
@@ -45,6 +46,8 @@ module GreenHat
       self.production_log = archive.things.find { |x| x.name == 'gitlab-rails/production_json.log' }
       self.api_log = archive.things.find { |x| x.name == 'gitlab-rails/api_json.log' }
       self.application_log = archive.things.find { |x| x.name == 'gitlab-rails/application_json.log' }
+      self.exceptions_log = archive.things.find { |x| x.name == 'gitlab-rails/exceptions_json.log' }
+      self.gitaly_log = archive.things.find { |x| x.name == 'gitaly/current' }
       self.sidekiq_log = archive.things.find { |x| x.name == 'sidekiq/current' }
       self.disk_free = archive.things.find { |x| x.name == 'df_h' }
     end
@@ -86,9 +89,11 @@ module GreenHat
       output << gitlab_services if gitlab_status
       output << title('Errors') if production_log || api_log || application_log || sidekiq_log
       output << production_errors if production_log
-      output << api_errors if api_log
       output << application_errors if application_log
       output << sidekiq_errors if sidekiq_log
+      output << api_errors if api_log
+      output << exception_errors if exceptions_log
+      output << gitaly_errors if gitaly_log
 
       # Final Space / Return
       output << ''
@@ -96,6 +101,26 @@ module GreenHat
     end
     # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
 
+    def exception_errors
+      count = exceptions_log.data.count
+      color = count.zero? ? :green : :red
+
+      [
+        title('  Exception', :bright_red, 18),
+        count.to_s.pastel(color)
+      ].join
+    end
+
+    def gitaly_errors
+      count = gitaly_log.data.count { |x| x.level == 'error' }
+      color = count.zero? ? :green : :red
+
+      [
+        title('  Gitaly', :bright_red, 18),
+        count.to_s.pastel(color)
+      ].join
+    end
+
     def production_errors
       count = production_log.data.count { |x| x.status == 500 }
       color = count.zero? ? :green : :red
@@ -117,7 +142,13 @@ module GreenHat
     end
 
     def application_errors
-      count = application_log.data.count { |x| x&.severity == 'ERROR' }
+      results = ShellHelper.filter_internal([
+        'gitlab-rails/application_json.log',
+        '--message!="Cannot obtain an exclusive lease"',
+        '--severity=error'
+      ].join(' '))
+
+      count = results.count { |x| x&.severity == 'ERROR' }
       color = count.zero? ? :green : :red
 
       [

data/lib/greenhat/shell/shell_helper.rb

@@ -150,8 +150,18 @@ module GreenHat
       format_table_entry(flags, val)
     end
 
+    # Internal Query Helper
+    # query = 'gitlab-rails/application_json.log --message!="Cannot obtain an exclusive lease" --severity=error'
+    # ShellHelper.filter_internal(query)
+    def self.filter_internal(search = '')
+      files, flags, args = Args.parse(Shellwords.split(search))
+      flags[:combine] = true
+
+      ShellHelper.filter_start(files, flags, args)
+    end
+
     # Main Entry Point for Filtering
-    def self.filter_start(files, flags, args)
+    def self.filter_start(files, flags = {}, args = {})
       # Convert to Things
       logs = ShellHelper.find_things(files, flags).select(&:processed?)
 
@@ -189,7 +199,7 @@ module GreenHat
       # Experimenting with deep clone
       results = Marshal.load(Marshal.dump(data))
       results.select! do |row|
-        args.send(flags.logic) do |arg|
+        args.send(flags.logic || :all?) do |arg|
           filter_row_key(row, arg, flags)
         end
       end

data/lib/greenhat/thing/file_types.rb

@@ -43,6 +43,13 @@ module GreenHat
             /dmesg/
           ]
         },
+        'lets-encrypt/renewal' => {
+          format: :bracket_log,
+          log: true,
+          pattern: [
+            %r{lets-encrypt/renewal}
+          ]
+        },
         'repmgrd/current' => {
           format: :bracket_log,
           log: true,
@@ -225,7 +232,8 @@ module GreenHat
           format: :json,
           log: true,
           pattern: [
-            %r{gitlab-rails/audit_json.log}
+            %r{gitlab-rails/audit_json.log},
+            %r{webservice.log/audit_json.log}
           ]
         },
         'gitlab-rails/auth.log' => {
@@ -336,6 +344,13 @@ module GreenHat
             %r{nginx/current}
           ]
         },
+        'nginx-ingress.log' => {
+          format: :kube_nginx,
+          log: true,
+          pattern: [
+            /nginx-ingress.log/
+          ]
+        },
         'nginx/gitlab_pages_access.log' => {
           format: :nginx,
           log: true,
@@ -422,7 +437,7 @@ module GreenHat
           ]
         },
         'registry/current' => {
-          format: :time_shellwords,
+          format: :time_registry,
           log: true,
           pattern: [
             %r{registry/current},
@@ -444,16 +459,18 @@ module GreenHat
         },
         'puma/puma_stderr.log' => {
           format: :raw,
-          log: true,
+          log: false,
           pattern: [
-            %r{puma/puma_stderr.log}
+            %r{puma/puma_stderr.log},
+            %r{webservice.log/puma.stderr.log}
           ]
         },
         'puma/puma_stdout.log' => {
           format: :json,
           log: true,
           pattern: [
-            %r{puma/puma_stdout.log}
+            %r{puma/puma_stdout.log},
+            %r{webservice.log/puma.stdout.log}
           ]
         },
         'gitlab-pages/current' => {
@@ -745,13 +762,15 @@ module GreenHat
         'gitlab-rails/application.log' => {
           format: :raw,
           pattern: [
-            %r{gitlab-rails/application.log}
+            %r{gitlab-rails/application.log},
+            %r{webservice.log/application.log}
           ]
         },
         'gitlab-rails/production.log' => {
           format: :raw,
           pattern: [
-            %r{gitlab-rails/production.log}
+            %r{gitlab-rails/production.log},
+            %r{webservice.log/production.log}
           ]
         },
         'gitlab/version-manifest.txt' => {
@@ -765,6 +784,72 @@ module GreenHat
           pattern: [
             %r{sidekiq/perf.data}
           ]
+        },
+
+        # ======================================================================
+        # KubeSoS TODO Section
+        # Things I am going to shortcut and set to raw for now
+        # ======================================================================
+        # Attempted Parsing
+        'kubesos_json' => {
+          log: true,
+          format: :kube_json,
+          pattern: [
+            /gitaly.log/
+          ]
+        },
+
+        'kube_webservice' => {
+          log: false,
+          format: :kube_webservice,
+          pattern: [
+            /^webservice\.log$/
+          ]
+        },
+
+        'kubesos' => {
+          format: :raw,
+          pattern: [
+            /all_values.yaml/,
+            %r{webservice.log/sidekiq_client.log},
+            /chart-version/,
+            /configmaps/,
+            /describe_deployments/,
+            /describe_ingress/,
+            /describe_nodes/,
+            /describe_pods/,
+            /describe_pv/,
+            /describe_pvc/,
+            /events/,
+            /get_deployments/,
+            /get_endpoints/,
+            /get_jobs/,
+            /get_pods/,
+            /get_pv/,
+            /get_pvc/,
+            /get_services/,
+            /gitaly.log/,
+            /gitlab-exporter.log/,
+            /gitlab-pages.log/,
+            /gitlab-shell.log/,
+            /grafana.log/,
+            /helm-version/,
+            /kubectl-check/,
+            /migrations.log/,
+            /minio.log/,
+            /nfs-client-provisioner.log/,
+            /operator.log/,
+            /postgresql.log/,
+            /prometheus.log/,
+            /redis.log/,
+            /registry.log/,
+            /secrets/,
+            /sidekiq.log/,
+            /task-runner.log/,
+            /top_nodes/,
+            /top_pods/,
+            /user_supplied_values.yaml/
+          ]
         }
       }
     end

data/lib/greenhat/thing/formatters/kube_json.rb

@@ -0,0 +1,36 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # K8s logs seem to have a lot of non-json entries
+    # Extract all json, ignore everything else
+    # ==========================================================================
+    def format_kube_json
+      self.result = raw.map do |row|
+        # Skip all non-json
+        next unless row.first == '{'
+
+        result = begin
+          Oj.load row
+        rescue EncodingError
+          puts
+          next
+        end
+
+        # Parsing Time
+        format_json_traverse result
+
+        result.sort.to_h
+      rescue StandardError => e
+        # TODO: Background Logger?
+        e.message
+        LogBot.warn('JSON Parse', e.message)
+        next
+      end.compact
+
+      :ok
+    end
+    # =========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/kube_nginx.rb

@@ -0,0 +1,48 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # ==========================================================================
+    def format_kube_nginx
+      self.result = raw.map do |row|
+        ip, _sym, remote_user, rest = row.split(' ', 4)
+
+        time, rest = rest.split(']', 2)
+        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+
+        verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
+
+        method, path, http_version = verb.split
+
+        {
+          remote_addr: ip,
+          remote_user: remote_user,
+          method: method,
+          path: path,
+          status: status,
+          bytes: bytes,
+          http_version: http_version,
+          http_referer: http_referer,
+          http_user_agent: http_user_agent,
+          gzip_ratio: gzip_ratio,
+          time: time
+        }
+
+        # Fall back for malformed logs
+      rescue StandardError
+        { message: row }
+      end
+
+      :ok
+    end
+
+    # rubocop:disable Layout/LineLength
+    # NGINX Conf: /var/opt/gitlab/nginx/conf/nginx.conf
+    # log_format gitlab_access '$remote_addr - $remote_user [$time_local] "$request_method $filtered_request_uri $server_protocol" $status $body_bytes_sent "$filtered_http_referer" "$http_user_agent" $gzip_ratio';
+    # rubocop:enable Layout/LineLength
+
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/kube_webservice.rb

@@ -0,0 +1,51 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Gitlab Tail Formatter
+    # ==========================================================================
+    def format_kube_webservice
+      # Revert to raw for cats
+      self.kind = :raw
+
+      output = {}
+      current_log = nil
+
+      raw.each do |line|
+        next if line.blank?
+
+        if line.include? '*** '
+          current_log = /\*\*\* (.+?) \*\*\*/.match(line).captures.first
+        else
+          output[current_log] ||= [] unless current_log.nil?
+          output[current_log].push line unless current_log.nil?
+        end
+      end
+
+      # Remove Empty Entries
+      output.reject { |_k, v| v.empty? }
+
+      # Root Dir
+      root_dir = "#{$TMP}/#{name}"
+      Dir.mkdir(root_dir)
+
+      # Write Files / Create Things
+      output.each do |k, v|
+        file_name = k.gsub('/var/log/gitlab/', '')
+
+        dir = "#{root_dir}/gitlab-rails"
+        Dir.mkdir(dir) unless File.exist?(dir)
+
+        File.write("#{root_dir}/#{file_name}", v.join("\n"))
+
+        # Thing Setup
+        archive.things_create(file: "#{root_dir}/#{file_name}").setup
+      end
+
+      # Link
+      self.result = raw
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/registry.rb

@@ -0,0 +1,47 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Registry Split
+    # ==========================================================================
+    # https://docs.docker.com/registry/configuration/#log
+    # Formatters text, json
+    # registry['log_formatter'] = "json"
+    # TODO: Logstash (Not working in 14.3)
+
+    def format_time_registry
+      self.result = raw.map do |row|
+        result = row[0] == '{' ? registry_json(row) : registry_shell_words(row)
+
+        # Timestamp Parsing
+        result.ts = Time.parse result.ts if result.key? 'ts'
+        result.time = Time.parse(result.time)
+
+        result
+      end
+    end
+
+    def registry_shell_words(row)
+      time, msg = row.split(' ', 2)
+
+      output = Shellwords.split(msg).each_with_object({}) do |x, h|
+        key, value = x.split('=')
+        next if value.nil?
+
+        h[key.to_sym] = value.numeric? ? value.to_f : value
+      end
+
+      output[:time] = time
+
+      output
+    end
+
+    def registry_json(msg)
+      Oj.load(msg)
+    rescue EncodingError
+      { message: msg }
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.3.3'.freeze
+  VERSION = '0.3.4'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: greenhat
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Davin Walker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-24 00:00:00.000000000 Z
+date: 2021-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -425,6 +425,7 @@ files:
 - lib/greenhat/shell/help.rb
 - lib/greenhat/shell/list.rb
 - lib/greenhat/shell/log.rb
+- lib/greenhat/shell/markdown.rb
 - lib/greenhat/shell/memory.rb
 - lib/greenhat/shell/network.rb
 - lib/greenhat/shell/page.rb
@@ -444,9 +445,13 @@ files:
 - lib/greenhat/thing/formatters/gitlab_status.rb
 - lib/greenhat/thing/formatters/json.rb
 - lib/greenhat/thing/formatters/json_shellwords.rb
+- lib/greenhat/thing/formatters/kube_json.rb
+- lib/greenhat/thing/formatters/kube_nginx.rb
+- lib/greenhat/thing/formatters/kube_webservice.rb
 - lib/greenhat/thing/formatters/multiline_json.rb
 - lib/greenhat/thing/formatters/nginx.rb
 - lib/greenhat/thing/formatters/raw.rb
+- lib/greenhat/thing/formatters/registry.rb
 - lib/greenhat/thing/formatters/shellwords.rb
 - lib/greenhat/thing/formatters/syslog.rb
 - lib/greenhat/thing/formatters/table.rb