greenhat 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6b9e87957f5dfc919379519552041ce83b8daa5923b0e89fc64424e050d8981
-  data.tar.gz: 412b2c4db6613c707c8aad3e39e41a3fb018d1834f5ef17000b11b3af582c639
+  metadata.gz: 79572c7389cb4cd519007f185a7fc249a771c08fa1fcbde8645d99786db42736
+  data.tar.gz: 6904a95f759a1dc0e866733f0a6220c62b47f3985998f82e2efd4619fdabe390
 SHA512:
-  metadata.gz: da4b45b6433d2c4c786bca4b46dc0cfad9b879796e6040b9efceb5ba5181f329c7d3792aef458fde54698791f03830168aad17b9b7d31191aa2e717407444662
-  data.tar.gz: 3d53ed5c45398acc3a317cc41a5573b82ab1c263e947c94cc419f235efd5bc527905f4484867968535f12aed15c3fbb79abe1d63093402f845c572557b7e288d
+  metadata.gz: d44e38727ce532ddbdcf8cc9aad1f5d7ee26a113ee589489bb5f15a99e616d06f66d304dc1e7735a679a707ed99323a45184234779066ca9f1f2cdea3e1b87e9
+  data.tar.gz: 6fb959d970482e244bea30f39ccd7357ce442fbc886df0084f425c3540b662a86972889213273ed5e734cace0f6b2b710c356c5585f98f76ffc4763ce2c68b10

data/lib/greenhat.rb

@@ -76,5 +76,6 @@ require 'greenhat/pry_helpers'
 require 'greenhat/tty/line'
 require 'greenhat/tty/reader'
 require 'greenhat/tty/custom_line'
+require 'greenhat/tty/columns'
 
 Warning.ignore(/The table size exceeds the currently set width/)

data/lib/greenhat/accessors/disk.rb

@@ -20,8 +20,64 @@ module GreenHat
       end
     end
 
-    def self.df
-      Thing.where(name: 'df_h')
+    def self.df(args = {})
+      things = Thing.where(name: 'df_h')
+
+      # Host / Archive
+      things.select! { |x| x.archive? args.archive } if args.archive
+
+      things
+    end
+
+    # Unified Output Handler
+    # rubocop:disable Metrics/MethodLength
+    def self.format_output(file, name = false, limit = nil, filter = %w[tmpfs loop])
+      output = []
+
+      output << file.friendly_name if name
+
+      # Reject TMPFS
+      disks = file.data.sort_by { |x| x.use.to_i }.reverse
+
+      # Filter
+      disks.reject! { |x| filter.any? { |y| x.filesystem.include? y } }
+
+      disks = disks[0..limit - 1] if limit
+
+      pad_mount, pad_size, pad_used, pad_avail = GreenHat::Disk.padding(disks)
+
+      # Headers
+      output << [
+        'Mount'.ljust(pad_mount).colorize(:blue),
+        'Size'.ljust(pad_size).colorize(:magenta),
+        'Used'.ljust(pad_used).colorize(:cyan),
+        'Avail'.ljust(pad_avail).colorize(:white),
+        '% Use'.ljust(pad_avail).colorize(:green)
+      ].join
+
+      # Table Summary
+      disks.map do |disk|
+        # Pretty Disk Use
+        use = [
+          disk.use.rjust(5).ljust(5).colorize(:green),
+          '  ['.colorize(:blue),
+          ('=' * (disk.use.to_i / 2)).colorize(:green),
+          ' ' * (50 - disk.use.to_i / 2),
+          ']'.colorize(:blue)
+        ].join
+
+        # Whole Thing
+        output << [
+          disk.mounted_on.ljust(pad_mount).colorize(:blue),
+          disk[:size].to_s.ljust(pad_size).colorize(:magenta),
+          disk.used.to_s.ljust(pad_used).colorize(:cyan),
+          disk.avail.to_s.ljust(pad_avail).colorize(:white),
+          use
+        ].join
+      end
+
+      output
     end
+    # rubocop:enable Metrics/MethodLength
   end
 end

data/lib/greenhat/accessors/process.rb

@@ -1,8 +1,13 @@
 module GreenHat
   # Sidekiq Log Helpers
   module Ps
-    def self.ps
-      Thing.where(name: 'ps')
+    def self.ps(args)
+      things = Thing.where(name: 'ps')
+
+      # Host / Archive
+      things.select! { |x| x.archive? args.archive } if args.archive
+
+      things
     end
   end
 end

data/lib/greenhat/cli.rb

@@ -379,11 +379,7 @@ module GreenHat
           next
         end
 
-        if line =~ /^exit/i
-          Settings.cmd_write
-
-          break
-        end
+        break if line =~ /^exit/i
 
         Settings.cmd_add(line) unless line.blank?
 

data/lib/greenhat/settings.rb

@@ -4,13 +4,41 @@ module GreenHat
   module Settings
     def self.settings
       @settings ||= {
-        history: []
+        history: [],
+
+        # round: [2],
+        # page: [:true] Automatic,
+        truncate: [TTY::Screen.width * 4]
+
       }
     end
 
+    # Load User Settings and drop them into settings
+    def self.settings_load
+      return true unless File.exist?(settings_file)
+
+      Oj.load(File.read(settings_file)).each do |key, value|
+        settings[key] = value
+      end
+    end
+
+    def self.settings_file
+      "#{dir}/settings.json"
+    end
+
+    # Set any Log Arguments that weren't set otherwise
+    def self.default_log_args(args, skip_args)
+      args[:round] ||= settings.round unless skip_args.include?(:round)
+      args[:page] ||= settings.page unless skip_args.include?(:page)
+      args[:truncate] ||= settings.truncate unless skip_args.include?(:truncate)
+    end
+
     def self.start
       Dir.mkdir dir unless Dir.exist? dir
 
+      # Load User Settings
+      settings_load
+
       # CMD History Loading / Tracking
       File.write(cmd_file, "\n") unless File.exist? cmd_file
     end
@@ -33,8 +61,8 @@ module GreenHat
       File.read(cmd_file).split("\n")
     end
 
-    def self.cmd_write
-      File.write(cmd_file, cmd_history_clean.join("\n"))
+    def self.cmd_history_clear
+      File.write(cmd_file, "\n")
     end
 
     def self.cmd_add(line)

data/lib/greenhat/shell.rb

@@ -9,8 +9,8 @@ module GreenHat
       Disk.df
     end
 
-    def self.ps
-      Process.ps
+    def self.ps(raw = {})
+      Process.ps raw
     end
 
     def self.netstat
@@ -63,11 +63,11 @@ module GreenHat
     end
 
     def self.history_clear
-      File.write(GreenHat::Cli.history_file, "\n")
+      Settings.cmd_history_clear
     end
 
     def self.history
-      File.read(GreenHat::Cli.history_file).split("\n").each_with_index do |line, i|
+      Settings.cmd_history_clean.each_with_index do |line, i|
         puts "#{i.to_s.ljust(3).colorize(:magenta)} #{line}"
       end
     end

data/lib/greenhat/shell/color_string.rb

@@ -0,0 +1,43 @@
+module GreenHat
+  module ShellHelper
+    # Helper to colorize and make outtput easier to read
+    module StringColor
+      def self.do(key, entry)
+        LogBot.debug('Unknown Format', entry.class) if ENV['DEBUG'] && !entry.instance_of?(String)
+
+        # Other Helpful colorizers
+        if colorize?(key)
+          colorize(key, entry)
+        else
+          entry.to_s
+        end
+      end
+
+      # Add Color?
+      def self.colorize?(key)
+        [:severity].any? key
+      end
+
+      # General Key/Value Coloring
+      def self.colorize(key, value)
+        case key
+        when :severity then severity(value)
+        else
+          value.to_s
+        end
+      end
+      # ----
+
+      def self.severity(value)
+        case value.to_s.downcase.to_sym
+        when :debug then value.colorize(:blue)
+        when :info then value.colorize(:cyan)
+        when :warn then value.colorize(:yellow)
+        when :fatal, :error then value.colorize(:light_red)
+        else
+          value.to_s
+        end
+      end
+    end
+  end
+end

data/lib/greenhat/shell/disk.rb

@@ -13,55 +13,16 @@ module GreenHat
         ShellHelper.file_output GreenHat::Disk.df
       end
 
-      # rubocop:disable Metrics/MethodLength,Metrics/BlockLength
-      def self.free
-        GreenHat::Disk.df.each do |file|
-          # Reject TMPFS
-          puts file.friendly_name
-          disks = file.data.sort_by { |x| x.use.to_i }.reverse.reject { |x| x.filesystem.include? 'tmpfs' }
+      def self.free(raw = {})
+        _log_list, _1opts, args = ShellHelper.param_parse(raw)
 
-          # pad_mount = GreenHat::Disk.max_padding(disks, :mounted_on)
-          # pad_size = GreenHat::Disk.max_padding(disks, :size)
-          # pad_used = GreenHat::Disk.max_padding(disks, :used)
-          # pad_avail = GreenHat::Disk.max_padding(disks, :avail)
-
-          pad_mount, pad_size, pad_used, pad_avail = GreenHat::Disk.padding(disks)
-
-          # Headers
-          puts [
-            'Mount'.ljust(pad_mount).colorize(:blue),
-            'Size'.ljust(pad_size).colorize(:magenta),
-            'Used'.ljust(pad_used).colorize(:cyan),
-            'Avail'.ljust(pad_avail).colorize(:white),
-            '% Use'.ljust(pad_avail).colorize(:green)
-          ].join
-
-          # Table Summary
-          disks.map do |disk|
-            # Pretty Disk Use
-            use = [
-              disk.use.rjust(5).ljust(5).colorize(:green),
-              '  ['.colorize(:blue),
-              ('=' * (disk.use.to_i / 2)).colorize(:green),
-              ' ' * (50 - disk.use.to_i / 2),
-              ']'.colorize(:blue)
-            ].join
-
-            # Whole Thing
-            puts [
-              disk.mounted_on.ljust(pad_mount).colorize(:blue),
-              disk[:size].to_s.ljust(pad_size).colorize(:magenta),
-              disk.used.to_s.ljust(pad_used).colorize(:cyan),
-              disk.avail.to_s.ljust(pad_avail).colorize(:white),
-              use
-            ].join
-          end
+        GreenHat::Disk.df(args).each do |file|
+          puts GreenHat::Disk.format_output(file, true)
 
           # File End Loop / Break
           puts
         end
       end
-      # rubocop:enable Metrics/MethodLength,Metrics/BlockLength
       # ------------------------------------------------------------------------
     end
   end

data/lib/greenhat/shell/faststats.rb

@@ -111,11 +111,8 @@ module GreenHat
       # --sort=count,fail,max,median,min,p95,p99,rps,score
       # ========================================================================
       def self.top(log_list)
-        # Extract Args
-        log_list, opts, args = ShellHelper.param_parse(log_list)
-
-        # Default to color output
-        args['color-output'] ||= true
+        # Extract Args,  No Round / Truncate for Fast Stats
+        log_list, opts, args = ShellHelper.param_parse(log_list, %i[page round truncate])
 
         cmd = opts.map { |opt| "--#{opt.field}=#{opt.value}" }.join(' ')
         cmd += args.keys.map { |arg| "--#{arg}" }.join(' ')
@@ -142,8 +139,8 @@ module GreenHat
       # ===== [ Fast Stats - Errors ] ====================
       # ========================================================================
       def self.errors(log_list)
-        # Extract Args
-        log_list, opts, args = ShellHelper.param_parse(log_list)
+        # Extract Args,  No Round / Truncate for Fast Stats
+        log_list, opts, args = ShellHelper.param_parse(log_list, %i[page round truncate])
 
         # Default to color output
         args['color-output'] ||= true
@@ -157,6 +154,8 @@ module GreenHat
         # Convert to Things
         files = ShellHelper.find_things(log_list)
 
+        LogBot.debug('FastStats CMD', cmd) if ENV['DEBUG']
+
         results = ShellHelper.file_process(files) do |file|
           [
             file.friendly_name,

data/lib/greenhat/shell/filter.rb

@@ -0,0 +1,128 @@
+module GreenHat
+  # CLI Helper
+  module ShellHelper
+    # Unify Filter / Filter Help
+    module Filter
+      # rubocop:disable Metrics/MethodLength
+      def self.help
+        puts "\u2500".colorize(:cyan) * 20
+        puts 'Filter'.colorize(:yellow)
+        puts "\u2500".colorize(:cyan) * 20
+
+        puts 'Options'.colorize(:blue)
+        puts '--raw'.colorize(:green)
+        puts '  Do not use less/paging'
+        puts
+
+        puts '--page'.colorize(:green)
+        puts '  Specifically enable or disable paging'
+        puts '  E.g. --page (default to true), --page=true, --page=false'
+        puts
+
+        puts '--round'.colorize(:green)
+        puts '  Attempt to round all integers. Default: 2.'
+        puts '  E.g. --round, --round=3, --round=0'
+        puts
+
+        puts '--truncate'.colorize(:green)
+        puts '  Limit output values. Defaults to 4 lines. 0 to disable'
+        puts '  E.g. --truncate=100 or --truncate=0'
+        puts
+
+        puts '--json'.colorize(:green)
+        puts '  Print output back into JSON'
+        puts
+
+        puts '--or'.colorize(:green)
+        puts '  Filters will use OR instead of AND'
+        puts
+
+        puts '--total'.colorize(:green)
+        puts '  Print only total count of matching entries'
+        puts
+
+        puts '--slice'.colorize(:green)
+        puts '  Extract specific fields from entries (slice multiple with comma)'
+        puts '  Ex: --slice=path or --slice=path,params'
+        puts
+
+        puts '--except'.colorize(:green)
+        puts '  Exclude specific fields (except multiple with comma)'
+        puts '  Ex: --except=params --except=params,path'
+        puts
+
+        puts '--stats'.colorize(:green)
+        puts '  Order/Count occurrances by field'
+        puts '  Ex: --stats=params --except=params,path'
+        puts
+
+        puts '--uniq'.colorize(:green)
+        puts '  Show unique values only'
+        puts '  Ex: --uniq=params --uniq=params,path'
+        puts
+
+        puts '--pluck'.colorize(:green)
+        puts '  Extract values from entries'
+        puts '  Ex: --pluck=params --pluck=params,path'
+        puts
+
+        puts '--archive'.colorize(:green)
+        puts '  Limit to specific archvie name (inclusive). Matching SOS tar.gz name'
+        puts '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
+        puts
+
+        puts '--sort'.colorize(:green)
+        puts '  Sort by multiple fields'
+        puts '  Ex: --sort=duration_s,db_duration_s'
+        puts
+
+        puts '--reverse'.colorize(:green)
+        puts '  Reverse all results'
+        puts '  Ex: --reverse'
+        puts
+
+        puts '--combine'.colorize(:green)
+        puts '  Omit archive identifier dividers. Useful with sort or time filters'
+        puts '  Ex: --combine'
+        puts
+
+        puts '--start'.colorize(:green)
+        puts '  Show events after specified time. Filtered by the `time` field'
+        puts '  Use with `--end` for between selections'
+        puts '  Ex: log filter --start="2021-06-22 14:44 UTC" --end="2021-06-22 14:45 UTC"'
+        puts
+
+        puts '--end'.colorize(:green)
+        puts '  Show events before specified time. Filtered by the `time` field'
+        puts '  Use with `--start` for between selections'
+        puts '  Ex: log filter --end="2021-06-22"'
+        puts
+
+        puts '--truncate'.colorize(:green)
+        puts '  Truncate field length. On by default (4 rows). Performance issues!'
+        puts '  Disable with --truncate=0'.colorize(:light_red)
+        puts '  Ex: --truncate=200, --truncate=2048"'
+        puts
+
+        puts 'Field Searching'.colorize(:blue)
+        puts '  --[key]=[value]'
+        puts '  Search in key for value'
+        puts '  Example: --path=mirror/pull'
+        puts
+
+        puts 'Search specific logs'.colorize(:blue)
+        puts '  Any non dash parameters will be the log list to search from'
+        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.colorize(:yellow)}` for log names"
+        puts
+
+        puts 'Example Queries'.colorize(:blue)
+        puts "  Also see #{'examples'.colorize(:light_blue)} for even more examples"
+        puts '  log filter --class=BuildFinishedWorker sidekiq/current --slice=time,message'
+        puts '  log filter gitlab-rails/api_json.log --slice=ua --uniq=ua --ua=gitlab-runner'
+
+        puts
+      end
+      # rubocop:enable Metrics/MethodLength
+    end
+  end
+end

data/lib/greenhat/shell/helper.rb

@@ -3,7 +3,7 @@ module GreenHat
   # rubocop:disable Metrics/ModuleLength
   module ShellHelper
     # Generic Parameter Parsing
-    def self.param_parse(params)
+    def self.param_parse(params, skip_args = [])
       # Turn Params into Hash
       opts = params.flat_map { |param| param_opt_scan(param) }
 
@@ -23,6 +23,9 @@ module GreenHat
         opt_field_remove?(opts, param) || arg_field_remove?(args, param)
       end
 
+      # Update other user defaults
+      Settings.default_log_args(args, skip_args)
+
       [params, opts, args]
     end
 
@@ -57,7 +60,7 @@ module GreenHat
 
     def self.param_special_opts
       %i[
-        slice except stats uniq pluck round archive start end sort limit
+        slice except stats uniq pluck round archive start end sort limit truncate page case
       ]
     end
 
@@ -75,8 +78,10 @@ module GreenHat
     # Arg Defaults
     def self.param_arg_defaults(field)
       case field
-      when :round then 2
+      when :round then [2]
       when :limit then [TTY::Screen.height / 2]
+      when :truncate then [TTY::Screen.width * 4]
+      when :page, :case then [:true]
       when *param_special_opts then []
       else
         true
@@ -128,15 +133,15 @@ module GreenHat
       end
 
       # Check if content needs to paged, or if auto_height is off
-      if !args.page && count_rows(data)
-        puts data.map { |x| entry_show(x, args) }.compact.join("\n")
+      if Page.skip?(args, data)
+        puts data.map { |entry| entry_show(args, entry) }.compact.join("\n")
         return true
       end
 
       # Default Pager
       TTY::Pager.page do |pager|
         data.each do |entry|
-          output = entry_show(entry, args)
+          output = entry_show(args, entry)
 
           next if output.blank?
 
@@ -145,63 +150,76 @@ module GreenHat
       end
     end
 
-    # Array/Hash and String pagination check. Don't unncessarily loop through everything
-    def self.count_rows(data)
-      height = TTY::Screen.height
-      size = 0
-
-      data.each do |entry|
-        size += case entry
-                when Hash then entry.keys.count
-                when Array then entry.count
-                else
-                  1
-                end
-
-        break if size > height
-      end
-
-      height > size
-    end
-
-    # Entry Shower
-    def self.entry_show(entry, args)
+    # Entry Shower / Top Level
+    def self.entry_show(args, entry, key = nil)
+      LogBot.debug('Entry Show', entry.class) if ENV['DEBUG']
       case entry
       when Hash then render_table(entry, args)
-      when String, Float, Integer then entry
-      when Array
-        entry.map { |x| x.ai(multiline: false) }.join("\n")
+      when Float, Integer, Array
+        format_table_entry(args, entry, key)
+      # Ignore Special Formatting for Strings / Usually already formatted
+      when String
+        entry
       else
         LogBot.warn('Shell Show', "Unknown #{entry.class}")
         nil
       end
     end
 
+    # Format Table Entries
+    def self.format_table_entry(args, entry, key = nil)
+      formatted_entry = case entry
+                        # Rounding
+                        when Float, Integer || entry.numeric?
+                          args.round ? entry.to_f.round(args.round.first.to_s.to_i).ai : entry.ai
+
+                          # General Inspecting
+                        when Hash then entry.ai(ruby19_syntax: true)
+
+                        # Arrays often contain Hashes. Dangerous Recursive?
+                        when Array
+                          entry.map { |x| format_table_entry(args, x) }.join("\n")
+
+                        when Time
+                          entry.to_s.colorize(:light_white)
+
+                        # Default String Formatting
+                        else
+                          StringColor.do(key, entry)
+                        end
+
+      if args[:truncate]
+        entry_truncate(formatted_entry, args[:truncate].join.to_i)
+      else
+        formatted_entry
+      end
+    rescue StandardError => e
+      binding.pry
+    end
+
     # Print the Table in a Nice way
     def self.render_table(entry, args)
+      entry = entry.map { |k, v| [k, format_table_entry(args, v, k)] }.to_h
+      # Pre-format Entry
+
       table = TTY::Table.new(header: entry.keys, rows: [entry], orientation: :vertical)
 
-      table.render(:unicode, padding: [0, 1, 0, 1]) do |renderer|
+      LogBot.debug('Rendering Entries') if ENV['DEBUG']
+      table.render(:unicode, padding: [0, 1, 0, 1], multiline: true) do |renderer|
         renderer.border.style = :cyan
 
-        renderer.filter = lambda do |val, _row_index, col_index|
-          if col_index == 1
-            if val.numeric?
-              # TODO:  Better Casting?
-              val = val.to_f.round(args.round.first.to_s.to_i) if args.round
-              val.to_s.colorize(:blue)
-            else
-              val.to_s
-            end
-          else
-            val.to_s
-          end
-        end
+        # renderer.filter = lambda do |val, _row_index, col_index|
+        #   render_table_entry(val, col_index, args)
+        # end
       end
 
+      # LogBot.debug('Finish Render Table') if ENV['DEBUG']
       # Fall Back to Amazing Inspect
     rescue StandardError => e
-      LogBot.warn('Table', message: e.message, backtrace: e.backtrace.first) if ENV['DEBUG']
+      if ENV['DEBUG']
+        LogBot.warn('Table', message: e.message)
+        ap e.backtrace
+      end
 
       [
         entry.ai,
@@ -210,6 +228,12 @@ module GreenHat
       ].join("\n")
     end
 
+    def self.render_table_entry(val, col_index, args)
+      return val.to_s unless col_index == 1
+
+      format_table_entry(args, val)
+    end
+
     # Main Entry Point for Filtering
     def self.filter_start(log_list, filter_type, args, opts)
       # Convert to Things
@@ -239,7 +263,7 @@ module GreenHat
       results = data.clone.flatten.compact
       results.select! do |row|
         opts.send(type) do |opt|
-          filter_row_key(row, opt)
+          filter_row_key(row, opt, args)
         end
       end
 
@@ -273,9 +297,12 @@ module GreenHat
       # Pluck
       results = filter_pluck(results, args[:pluck]) if args.key?(:pluck)
 
+      # Truncate
+      # filter_truncate(results, args[:truncate].join.to_i) if args.key?(:truncate)
+
       # Limit
       if args[:limit]
-        results[0..args[:limit].map(&:to_s).join.to_i]
+        results[0..args[:limit].map(&:to_s).join.to_i - 1]
       else
         results
       end
@@ -332,6 +359,17 @@ module GreenHat
       results.map { |row| row.except(*except) }
     end
 
+    def self.entry_truncate(entry, truncate)
+      # Ignore if Truncation Off
+      return entry if truncate.zero?
+
+      # Only truncate large strings
+      return entry unless entry.instance_of?(String) && entry.size > truncate
+
+      # Include  '...' to indicate truncation
+      "#{entry.to_s[0..truncate]} #{'...'.colorize(:light_blue)}"
+    end
+
     def self.filter_slice(results, slice)
       # Avoid Empty Results
       if slice.empty?
@@ -397,13 +435,18 @@ module GreenHat
     end
 
     # Break out filter row logic into separate method
-    def self.filter_row_key(row, param)
+    def self.filter_row_key(row, param, args)
       # Ignore Other Logic if Field isn't even included
       return false unless row.key? param.field
 
-      # Not Included Param
-      included = row[param.field].to_s.include? param.value.to_s
+      # Sensitivity Check / Check for Match
+      included = if args.key?(:case)
+                   row[param.field].to_s.include? param.value.to_s
+                 else
+                   row[param.field].to_s.downcase.include? param.value.to_s.downcase
+                 end
 
+      # Pivot of off include vs exclude
       if param.bang
         !included
       else

data/lib/greenhat/shell/log.rb

@@ -2,7 +2,6 @@ module GreenHat
   # CLI Helper
   module Shell
     # Logs
-    # rubocop:disable Metrics/ModuleLength
     module Log
       def self.help
         puts "\u2500".colorize(:cyan) * 20
@@ -14,16 +13,17 @@ module GreenHat
         puts '    Just print selected logs'
         puts '  filter'.colorize(:green)
         puts '    Key/Field Filtering'
+        puts '    - See `filter_help`'
         puts '  search'.colorize(:green)
         puts '    General text by entry searching'
+        puts '    - See `search_help`'
         puts '  ls'.colorize(:green)
         puts '    List available files'
         puts
+      end
 
-        filter_help
-
-        puts
-        search_help
+      def self.filter_help
+        ShellHelper::Filter.help
       end
 
       # List Files Helpers
@@ -113,122 +113,15 @@ module GreenHat
         end
 
         # log filter --path='cloud/gitlab-automation' --path='/pull' --all
-        # cloud/gitlab-automation
         # log filter --project=thingy --other_filter=asdf *
       rescue StandardError => e
-        binding.pry
         LogBot.fatal('Filter', message: e.message, backtrace: e.backtrace.first)
       end
       # ========================================================================
 
-      # rubocop:disable Metrics/MethodLength
-      def self.filter_help
-        puts "\u2500".colorize(:cyan) * 20
-        puts 'Log Filter'.colorize(:yellow)
-        puts "\u2500".colorize(:cyan) * 20
-
-        puts 'Options'.colorize(:blue)
-        puts '--raw'.colorize(:green)
-        puts '  Do not use less/paging'
-        puts
-
-        puts '--round'.colorize(:green)
-        puts '  Attempt to round all integers. Default: 2.'
-        puts '  E.g. --round, --round=3, --round=0'
-        puts
-
-        puts '--json'.colorize(:green)
-        puts '  Print output back into JSON'
-        puts
-
-        puts '--or'.colorize(:green)
-        puts '  Filters will use OR instead of AND'
-        puts
-
-        puts '--total'.colorize(:green)
-        puts '  Print only total count of matching entries'
-        puts
-
-        puts '--slice'.colorize(:green)
-        puts '  Extract specific fields from entries (slice multiple with comma)'
-        puts '  Ex: --slice=path or --slice=path,params'
-        puts
-
-        puts '--except'.colorize(:green)
-        puts '  Exclude specific fields (except multiple with comma)'
-        puts '  Ex: --except=params --except=params,path'
-        puts
-
-        puts '--stats'.colorize(:green)
-        puts '  Order/Count occurrances by field'
-        puts '  Ex: --stats=params --except=params,path'
-        puts
-
-        puts '--uniq'.colorize(:green)
-        puts '  Show unique values only'
-        puts '  Ex: --uniq=params --uniq=params,path'
-        puts
-
-        puts '--pluck'.colorize(:green)
-        puts '  Extract values from entries'
-        puts '  Ex: --pluck=params --pluck=params,path'
-        puts
-
-        puts '--archive'.colorize(:green)
-        puts '  Limit to specific archvie name (inclusive). Matching SOS tar.gz name'
-        puts '  Ex: --archive=dev-gitlab_20210622154626, --archive=202106,202107'
-        puts
-
-        puts '--sort'.colorize(:green)
-        puts '  Sort by multiple fields'
-        puts '  Ex: --sort=duration_s,db_duration_s'
-        puts
-
-        puts '--reverse'.colorize(:green)
-        puts '  Reverse all results'
-        puts '  Ex: --reverse'
-        puts
-
-        puts '--combine'.colorize(:green)
-        puts '  Omit archive identifier dividers. Useful with sort or time filters'
-        puts '  Ex: --combine'
-        puts
-
-        puts '--start'.colorize(:green)
-        puts '  Show events after specified time. Filtered by the `time` field'
-        puts '  Use with `--end` for between selections'
-        puts '  Ex: log filter --start="2021-06-22 14:44 UTC" --end="2021-06-22 14:45 UTC"'
-        puts
-
-        puts '--end'.colorize(:green)
-        puts '  Show events before specified time. Filtered by the `time` field'
-        puts '  Use with `--start` for between selections'
-        puts '  Ex: log filter --end="2021-06-22"'
-        puts
-
-        puts 'Field Searching'.colorize(:blue)
-        puts '  --[key]=[value]'
-        puts '  Search in key for value'
-        puts '  Example: --path=mirror/pull'
-        puts
-
-        puts 'Search specific logs'.colorize(:blue)
-        puts '  Any non dash parameters will be the log list to search from'
-        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.colorize(:yellow)}` for log names"
-        puts
-
-        puts 'Example Queries'.colorize(:blue)
-        puts "  Also see #{'filter_examples'.colorize(:light_blue)} for even more examples"
-        puts 'log filter --class=BuildFinishedWorker sidekiq/current --slice=time,message'
-        puts 'log filter gitlab-rails/api_json.log --slice=ua --uniq=ua --ua=gitlab-runner'
-
-        puts
-      end
-      # rubocop:enable Metrics/MethodLength
-
       # rubocop:disable Layout/LineLength
       # TODO: Add a lot more examples
-      def self.filter_examples
+      def self.examples
         puts 'Find `done` job for sidekiq, sort by duration, only duration, and show longest first'.colorize(:light_green)
         puts 'log filter sidekiq/current --job_status=done --sort=duration_s,db_duration_s --slice=duration_s,db_duration_s --reverse'
         puts
@@ -236,8 +129,8 @@ module GreenHat
         puts 'log filter --status=500 --slice=exception.message gitlab-rails/production_json.log'
         puts
       end
-      # rubocop:enable Layout/LineLength
 
+      # rubocop:enable Layout/LineLength
       # ========================================================================
       # Search (Full Text / String Search)
       # ========================================================================
@@ -325,7 +218,7 @@ module GreenHat
 
         puts 'Search specific logs'.colorize(:blue)
         puts '  Any non dash parameters will be the log list to search from'
-        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.colorize(:yellow)}` for log names"
+        puts "  Ex: log filter --path=api sidekiq/current (hint: use  `#{'ls'.colorize(:yellow)}` for log names)"
         puts
 
         puts 'Example Queries'.colorize(:blue)
@@ -337,7 +230,6 @@ module GreenHat
 
       # ------------------------------------------------------------------------
     end
-    # rubocop:enable Metrics/ModuleLength
   end
 end
 

data/lib/greenhat/shell/page.rb

@@ -0,0 +1,33 @@
+module GreenHat
+  module ShellHelper
+    # Helper to organize page check / methods
+    module Page
+      # Check if paging should be skipped
+      def self.skip?(args, data)
+        return count_rows(data) unless args[:page]
+
+        args[:page].first == :false
+      end
+
+      # Array/Hash and String pagination check. Don't unncessarily loop through everything
+      def self.count_rows(data)
+        height = TTY::Screen.height
+        size = 0
+
+        data.each do |entry|
+          size += case entry
+                  when Hash then entry.keys.count
+                  when Array then entry.count
+                  else
+                    1
+                  end
+
+          break if size > height
+        end
+
+        size < height
+      end
+    end
+    # ----
+  end
+end

data/lib/greenhat/shell/process.rb

@@ -4,8 +4,51 @@ module GreenHat
     # Logs
     module Process
       # Easy Show All
-      def self.ps
-        ShellHelper.file_output GreenHat::Ps.ps
+      # Filter --archive
+
+      def self.ls
+        help
+      end
+
+      def self.help
+        puts "\u2500".colorize(:cyan) * 20
+        puts "#{'Process'.colorize(:yellow)} -  ps helper"
+        puts "\u2500".colorize(:cyan) * 20
+
+        puts 'Command Summary'.colorize(:blue)
+        puts '  ps'.colorize(:green)
+        puts '    Raw `ps`'
+        puts '  filter'.colorize(:green)
+        puts '    Key/Field Filtering'
+        puts '    - See `filter_help`'
+        puts
+      end
+
+      def self.filter_help
+        ShellHelper::Filter.help
+      end
+
+      def self.ps(raw = {})
+        _log_list, _1opts, args = ShellHelper.param_parse(raw)
+        ShellHelper.file_output GreenHat::Ps.ps(args)
+      end
+
+      def self.filter(raw = {})
+        _log_list, opts, args = ShellHelper.param_parse(raw)
+
+        # AND / OR Filtering
+        filter_type = args.or ? :any? : :all?
+
+        # ShellHelper.file_output
+        results = ShellHelper.filter_start(GreenHat::Ps.ps(args).map(&:name), filter_type, args, opts)
+
+        # Check Search Results
+        if results.instance_of?(Hash) && results.values.flatten.empty?
+          puts 'No results'.colorize(:red)
+        else
+          # This causes the key 'colorized' output to also be included
+          ShellHelper.show(results.to_a.compact.flatten, args)
+        end
       end
     end
   end

data/lib/greenhat/shell/report.rb

@@ -19,7 +19,7 @@ module GreenHat
 
     attr_accessor :archive, :host, :os_release, :selinux_status, :cpu, :uname,
                   :timedatectl, :uptime, :meminfo, :gitlab_manifest, :gitlab_status,
-                  :production_log, :api_log, :application_log, :sidekiq_log, :free_m
+                  :production_log, :api_log, :application_log, :sidekiq_log, :free_m, :disk_free
 
     # Find Needed Files for Report
     # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
@@ -40,6 +40,7 @@ module GreenHat
       self.api_log = archive.things.find { |x| x.name == 'gitlab-rails/api_json.log' }
       self.application_log = archive.things.find { |x| x.name == 'gitlab-rails/application_json.log' }
       self.sidekiq_log = archive.things.find { |x| x.name == 'sidekiq/current' }
+      self.disk_free = archive.things.find { |x| x.name == 'df_h' }
     end
 
     def show
@@ -62,11 +63,17 @@ module GreenHat
       # Memory
       if meminfo || free_m
         output << 'Memory'.colorize(:light_yellow)
-        output << memory if meminfo
+        output << memory_perc if meminfo
         output << memory_free if free_m
         output << ''
       end
 
+      # Disk
+      if disk_free
+        output << disks
+        output << ''
+      end
+
       # Gitlab
       output << 'GitLab'.colorize(:light_yellow) if gitlab_manifest
       output << gitlab_version if gitlab_manifest
@@ -209,6 +216,9 @@ module GreenHat
     end
 
     def sys_time
+      # Ignore if Empty
+      return false if timedatectl.data.nil?
+
       ntp_statuses = timedatectl.data.slice(*ntp_keys).values.compact
 
       enabled = %w[active yes] & ntp_statuses
@@ -260,26 +270,18 @@ module GreenHat
       ].join
     end
 
-    def memory
+    def memory_perc
       total = ShellHelper.human_size_to_number(meminfo.data['MemTotal'])
       free = ShellHelper.human_size_to_number(meminfo.data['MemFree'])
       used = percent((total - free), total)
+
       [
-        title('MemUsed'),
-        '['.colorize(:light_black),
+        title('Usage'),
+        '  ['.colorize(:light_black),
         '='.colorize(:green) * (used / 2),
         ' ' * (50 - used / 2),
         ']'.colorize(:light_black),
-        " #{100 - percent(free, total)}%".colorize(:green), # Inverse
-        "\n",
-        title('Total'),
-        number_to_human_size(total).colorize(:light_white),
-        "\n",
-        title('Used'),
-        number_to_human_size(total - free),
-        "\n",
-        title('Free'),
-        number_to_human_size(free)
+        " #{100 - percent(free, total)}%".colorize(:green) # Inverse
       ].join
     end
 
@@ -288,6 +290,8 @@ module GreenHat
 
       return unless free
 
+      formatted_mem = free_m.data.map { |x| GreenHat::Memory.memory_row x }
+
       [
         title('Total', :cyan, 14),
         number_to_human_size(free.total.to_i * 1024**2),
@@ -299,7 +303,23 @@ module GreenHat
         number_to_human_size(free.free.to_i * 1024**2),
         "\n",
         title('Available', :green, 14),
-        number_to_human_size(free.available.to_i * 1024**2)
+        number_to_human_size(free.available.to_i * 1024**2),
+        "\n\n",
+        formatted_mem.map { |x| x.prepend ' ' * 2 }.join("\n")
+      ].join
+    end
+
+    def disks
+      # GreenHat::Disk.df({archive: []})
+      file = GreenHat::Disk.df({ archive: [archive.name] })
+
+      disk_list = GreenHat::Disk.format_output(file.first, false, 3)
+
+      # Preapre / Indent List
+      [
+        'Disks(Top % Usage)'.colorize(:light_yellow),
+        "\n",
+        disk_list.each { |x| x.prepend(' ' * 4) }.join("\n")
       ].join
     end
 

data/lib/greenhat/thing/file_types.rb

@@ -44,7 +44,7 @@ module GreenHat
           ]
         },
         'log/syslog' => {
-          format: :bracket_log,
+          format: :syslog,
           log: true,
           pattern: [
             %r{log/syslog}
@@ -95,6 +95,13 @@ module GreenHat
             %r{consul/current}
           ]
         },
+        'consul/failover_pgbouncer.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{consul/failover_pgbouncer.log}
+          ]
+        },
         'pgbouncer/current' => {
           format: :time_space,
           log: true,
@@ -179,6 +186,13 @@ module GreenHat
             %r{logrotate/current}
           ]
         },
+        'gitlab-rails/grpc.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{gitlab-rails/grpc.log}
+          ]
+        },
         'gitlab-rails/api_json.log' => {
           format: :api_json,
           log: true,
@@ -301,6 +315,13 @@ module GreenHat
             /mount/
           ]
         },
+        'nginx/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{nginx/current}
+          ]
+        },
         'nginx/gitlab_pages_access.log' => {
           format: :raw,
           log: true,
@@ -671,7 +692,7 @@ module GreenHat
           ]
         },
         'log/messages' => {
-          format: :raw,
+          format: :syslog,
           log: true,
           pattern: [
             %r{log/messages}

data/lib/greenhat/thing/formatters/syslog.rb

@@ -0,0 +1,39 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Formatters for bracket logs (dmesg, sos)
+    def format_syslog
+      self.result = raw.map do |row|
+        next if row.empty? || row == "\n"
+
+        format_syslog_row(row)
+      end
+
+      result.compact!
+    end
+
+    # Split / Parse Time
+    # TODO: Better sys log parsing? Cannot find ready-made regex/ruby parser
+    def format_syslog_row(row)
+      month, day, time, device, service, message = row.split(' ', 6)
+      service.gsub!(':', '')
+      pid = service.match(/\[(.*?)\]/)&.captures&.first
+
+      {
+        time: format_time_parse("#{month} #{day} #{time}"),
+        device: device,
+        service: service,
+        message: message,
+        pid: pid
+      }
+
+    # Return everything incase of error
+    rescue StandardError => e
+      LogBot.warn('SysLog Parse', e.message)
+      {
+        message: row
+      }
+    end
+  end
+end

data/lib/greenhat/tty/columns.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module TTY
+  class Table
+    # A module for calculating table data column widths
+    #
+    # Used by {Table} to manage column sizing.
+    #
+    # @api private
+    module Columns
+      # Converts column widths to array format or infers default widths
+      #
+      # @param [TTY::Table] table
+      #
+      # @param [Array, Numeric, NilClass] column_widths
+      #
+      # @return [Array[Integer]]
+      #
+      # @api public
+      def widths_from(table, column_widths = nil)
+        case column_widths
+        when Array
+          assert_widths(column_widths, table.columns_size)
+          Array(column_widths).map(&:to_i)
+        when Numeric
+          Array.new(table.columns_size, column_widths)
+        when NilClass
+          # THE SHIM! Strings that are too large fail to render correctly to do an empty result
+          # Set the maximum table width to half the screen size. Can't be full size due to the table headers
+          LogBot.debug('TTY Column Width') if ENV['DEBUG']
+          extract_widths(table.data).map { |x| x >= TTY::Screen.width ? (TTY::Screen.width * 3 / 4) : x }
+
+        else
+          raise TypeError, 'Invalid type for column widths'
+        end
+      end
+      module_function :widths_from
+    end
+  end
+end

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.1.5'.freeze
+  VERSION = '0.2.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: greenhat
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Davin Walker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-08 00:00:00.000000000 Z
+date: 2021-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -399,14 +399,17 @@ files:
 - lib/greenhat/settings.rb
 - lib/greenhat/shell.rb
 - lib/greenhat/shell/cat.rb
+- lib/greenhat/shell/color_string.rb
 - lib/greenhat/shell/disk.rb
 - lib/greenhat/shell/faststats.rb
+- lib/greenhat/shell/filter.rb
 - lib/greenhat/shell/gitlab.rb
 - lib/greenhat/shell/help.rb
 - lib/greenhat/shell/helper.rb
 - lib/greenhat/shell/log.rb
 - lib/greenhat/shell/memory.rb
 - lib/greenhat/shell/network.rb
+- lib/greenhat/shell/page.rb
 - lib/greenhat/shell/process.rb
 - lib/greenhat/shell/report.rb
 - lib/greenhat/thing.rb
@@ -425,6 +428,7 @@ files:
 - lib/greenhat/thing/formatters/multiline_json.rb
 - lib/greenhat/thing/formatters/raw.rb
 - lib/greenhat/thing/formatters/shellwords.rb
+- lib/greenhat/thing/formatters/syslog.rb
 - lib/greenhat/thing/formatters/table.rb
 - lib/greenhat/thing/formatters/time_json.rb
 - lib/greenhat/thing/formatters/time_shellwords.rb
@@ -435,6 +439,7 @@ files:
 - lib/greenhat/thing/kind.rb
 - lib/greenhat/thing/spinner.rb
 - lib/greenhat/thing/super_log.rb
+- lib/greenhat/tty/columns.rb
 - lib/greenhat/tty/custom_line.rb
 - lib/greenhat/tty/line.rb
 - lib/greenhat/tty/reader.rb