greenhat 0.3.4 → 0.5.0

data/lib/greenhat/shell/shell_helper.rb

@@ -55,12 +55,14 @@ module GreenHat
       # Default Pager
       TTY::Pager.page do |pager|
         data.each do |entry|
-          output = entry_show(flags, entry)
+          output = "\n#{entry_show(flags, entry)}"
+
+          # output += "\n" if flags[:table_style] == 'basic'
 
           # Breaks any intentional spaces
           # next if output.blank?
 
-          pager.write("\n#{output}") # write line to the pager
+          pager.write(output) # write line to the pager
         end
       end
     end
@@ -72,7 +74,7 @@ module GreenHat
       when Hash then render_table(entry, flags)
       when Float, Integer, Array
         format_table_entry(flags, entry, key)
-      # Ignore Special Formatting for Strings / Usually already formatted
+        # Ignore Special Formatting for Strings / Usually already formatted
       when String
         entry
       else
@@ -88,13 +90,12 @@ module GreenHat
                         when Float, Integer || entry.numeric?
                           flags.key?(:round) ? entry.to_f.round(flags.round).ai : entry.ai
 
-                          # General Inspecting
+                        # General Inspecting
                         when Hash then entry.ai(ruby19_syntax: true)
 
                         # Arrays often contain Hashes. Dangerous Recursive?
                         when Array
                           entry.map { |x| format_table_entry(flags, x) }.join("\n")
-
                         when Time
                           entry.to_s.pastel(:bright_white)
 
@@ -103,7 +104,8 @@ module GreenHat
                           StringColor.do(key, entry)
                         end
 
-      if flags[:truncate]
+      # Stats truncation handled separately
+      if flags[:truncate] && !flags[:stats]
         entry_truncate(formatted_entry, flags[:truncate])
       else
         formatted_entry
@@ -117,7 +119,7 @@ module GreenHat
 
     # Print the Table in a Nice way
     def self.render_table(entry, flags)
-      entry = entry.map { |k, v| [k, format_table_entry(flags, v, k)] }.to_h
+      entry = entry.to_h { |k, v| [k, format_table_entry(flags, v, k)] }
       # Pre-format Entry
 
       table_style = flags[:table_style]&.to_sym || :unicode
@@ -125,10 +127,15 @@ module GreenHat
       table = TTY::Table.new(header: entry.keys, rows: [entry], orientation: :vertical)
 
       LogBot.debug('Rendering Entries') if ENV['DEBUG']
-      table.render(table_style, padding: [0, 1, 0, 1], multiline: true) do |renderer|
+      output = table.render(table_style, padding: [0, 1, 0, 1], multiline: true) do |renderer|
         renderer.border.style = :cyan
       end
 
+      # Line breaks for basic tables
+      output += "\n" if flags[:table_style] == 'basic'
+
+      output
+
       # LogBot.debug('Finish Render Table') if ENV['DEBUG']
       # Fall Back to Amazing Inspect
     rescue StandardError => e
@@ -157,171 +164,10 @@ module GreenHat
       files, flags, args = Args.parse(Shellwords.split(search))
       flags[:combine] = true
 
-      ShellHelper.filter_start(files, flags, args)
-    end
-
-    # Main Entry Point for Filtering
-    def self.filter_start(files, flags = {}, args = {})
-      # Convert to Things
-      logs = ShellHelper.find_things(files, flags).select(&:processed?)
-
-      # Ignore Archive/Host Dividers
-      if flags[:combine]
-        results = logs.reject(&:blank?).map(&:data).flatten.compact
-        ShellHelper.filter(results, flags, args)
-      else
-        # Iterate and Preserve Archive/Host Index
-        logs.each_with_object({}) do |log, obj|
-          # Ignore Empty Results / No Thing
-          next if log&.blank?
-
-          # Include Total Count in Name
-          results = ShellHelper.filter(log.data, flags, args)
-          title = [
-            log.friendly_name,
-            " #{results.count}".pastel(:bright_black)
-          ]
-
-          # Save unless empty
-          obj[title.join] = results unless results.count.zero?
-
-          obj
-        end
-      end
-    end
-
-    # Filter Logic
-    # TODO: Simplify
-    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-    def self.filter(data, flags = {}, args = {})
-      # results = data.clone.flatten.compact
-
-      # Experimenting with deep clone
-      results = Marshal.load(Marshal.dump(data))
-      results.select! do |row|
-        args.send(flags.logic || :all?) do |arg|
-          filter_row_key(row, arg, flags)
-        end
-      end
-
-      # Ensure presecense of a specific field
-      results = filter_exists(results, flags[:exists]) if flags.key?(:exists)
-
-      # Time Zone
-      results = filter_modify_timezone(results, flags[:time_zone]) if flags.key?(:time_zone)
-
-      # Time Filtering
-      results = filter_time(results, flags) if flags.key?(:start) || flags.key?(:end)
-
-      # Strip Results if Slice is defined
-      results = filter_slice(results, flags[:slice]) if flags.key?(:slice)
-
-      # Strip Results if Except is defined
-      results = filter_except(results, flags[:except]) if flags.key?(:except)
-
-      # Remove Blank from either slice or except
-      results.reject!(&:empty?)
-
-      # Sort
-      results.sort_by! { |x| x.slice(*flags[:sort]).values } if flags.key?(:sort)
-
-      # JSON Formatting
-      results = results.map { |x| Oj.dump(x) } if flags.key?(:json)
-
-      # Show Unique Only
-      results = filter_uniq(results, flags[:uniq]) if flags.key?(:uniq)
-
-      # Reverse
-      results.reverse! if flags[:reverse]
-
-      # Count occurrences / Skip Results
-      return filter_stats(results, flags) if flags.key?(:stats)
-
-      # Limit before Pluck / Flattening
-      results = filter_limit(results, flags[:limit]) if flags.key?(:limit)
-
-      # Pluck
-      results = filter_pluck(results, flags[:pluck]) if flags.key?(:pluck)
-
-      results
-    end
-    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-
-    # Limit / Ensure Exists and Valid Number
-    def self.filter_limit(results, limit)
-      return results unless limit.integer? && limit.positive?
-
-      results.shift limit
-    end
-
-    def self.filter_modify_timezone(results, time_zone)
-      results.each do |x|
-        next unless x.key? :time
+      # Default to everything
+      files = Thing.all.map(&:name) if files.empty?
 
-        x[:time] = x[:time].in_time_zone time_zone
-      end
-
-      results
-    end
-
-    # Filter Start and End Times
-    # rubocop:disable Metrics/MethodLength
-    # TODO: This is a bit icky, simplify/dry
-    def self.filter_time(results, flags)
-      if flags.key?(:start)
-        begin
-          time_start = Time.parse(flags[:start])
-
-          results.select! do |x|
-            if x.time
-              time_start < x.time
-            else
-              true
-            end
-          end
-        rescue StandardError
-          puts 'Unable to Process Start Time Filter'.pastel(:red)
-        end
-      end
-
-      if flags.key?(:end)
-        begin
-          time_start = Time.parse(flags[:end])
-
-          results.select! do |x|
-            if x.time
-              time_start > x.time
-            else
-              true
-            end
-          end
-        rescue StandardError
-          puts 'Unable to Process End Time Filter'.pastel(:red)
-        end
-      end
-
-      results
-    end
-    # rubocop:enable Metrics/MethodLength
-
-    def self.filter_except(results, except)
-      # Avoid Empty Results
-      if except.empty?
-        filter_empty_arg('except')
-        return results
-      end
-
-      results.map { |row| row.except(*except) }
-    end
-
-    def self.filter_exists(results, exists)
-      # Avoid Empty Results
-      if exists.empty?
-        filter_empty_arg('exists')
-        return results
-      end
-
-      results.select { |row| (exists - row.keys).empty? }
+      Query.start(files, flags, args)
     end
 
     def self.entry_truncate(entry, truncate)
@@ -335,156 +181,31 @@ module GreenHat
       "#{entry.to_s[0..truncate]} #{'...'.pastel(:bright_blue)}"
     end
 
-    def self.filter_slice(results, slice)
-      # Avoid Empty Results
-      if slice.empty?
-        filter_empty_arg('slice')
-        return results
-      end
-
-      results.map { |row| row.slice(*slice) }
-    end
-
-    def self.filter_pluck(results, pluck)
-      # Avoid Empty Results
-      if pluck.empty?
-        filter_empty_arg('pluck')
-        return results
-      end
-
-      results.map { |x| x.slice(*pluck).values }.flatten
-    end
-
-    def self.filter_uniq(results, unique)
-      # Avoid Empty Results
-      if unique.empty?
-        filter_empty_arg('uniq')
-        return results
-      end
-
-      unique.map do |field|
-        results.uniq { |x| x[field] }
-      end.inject(:&)
-    end
-
-    def self.filter_stats(results, flags)
-      stats = flags[:stats]
-
-      # Avoid Empty Results
-      if stats.empty?
-        filter_empty_arg('stats')
-        return results
-      end
-
-      # Loop through Stats, Separate Hash/Tables
-      stats.map do |field|
-        occurrences = filter_count_occurrences(results, field, flags)
-
-        # Total Occurences
-        total = occurrences.values.sum
-
-        # Percs
-        occurrences.transform_values! do |count|
-          [
-            count,
-            " #{percent(count, total)}%".pastel(:bright_black)
-          ]
-        end
-
-        # Sort by total occurances / New Variable for Total
-        output = occurrences.sort_by(&:last).to_h.transform_values!(&:join).to_a
-
-        # Append Header / Total with field name
-        output.unshift([field.to_s.pastel(:bright_black), total])
-
-        # Use Truncate For Long Keys
-        if flags[:truncate]
-          output.map! do |key, value|
-            [key.to_s[0..flags[:truncate]], value]
-          end
-        end
-
-        # Format
-        output.to_h
-      end
-    end
-
-    # Percent Helper
-    def self.percent(value, total)
-      ((value / total.to_f) * 100).round
-    end
-
-    # Helper to Count occurrences
-    def self.filter_count_occurrences(results, field, flags = {})
-      results.each_with_object(Hash.new(0)) do |entry, counts|
-        if entry.key? field
-          # Rounding in pagination breaks stats
-          key = if flags.key?(:round) && entry[field].numeric?
-                  entry[field].to_f.round(flags.round)
-                else
-                  entry[field]
-                end
-
-          counts[key] += 1
-        else
-          counts['None'.pastel(:bright_black)] += 1
+    # Total Count Helper
+    def self.total_count(results, flags)
+      results.map do |k, v|
+        puts k
+        # puts "Total: #{v.count.to_s.pastel(:blue)}"
+        # puts "Time Covered: #{calculate_duration(v).pastel(:cyan)}"
+
+        output = {}
+        output[:total] = v.count
+        output[:duration] = Query.calculate_duration(v)
+
+        # Sort / Get first and Last
+        list = v.select(&:time).map(&:time)
+        unless list.blank?
+          output[:start] = list.first
+          output[:end] = list.last
         end
 
-        counts
-      end
-    end
-
-    def self.filter_empty_arg(arg)
-      puts [
-        'Ignoring'.pastel(:bright_yellow),
-        "--#{arg}".pastel(:cyan),
-        'it requires an argument'.pastel(:red)
-      ].join(' ')
-    end
-
-    # Break out filter row logic into separate method
-
-    def self.filter_row_key(row, arg, flags)
-      # Ignore Other Logic if Field isn't even included / Full Text Searching
-      return false unless row.key?(arg[:field]) || arg[:field] == :text
-
-      # Sensitivity Check / Check for Match / Full Text Searching
-      included = if arg[:field] == :text
-                   filter_row_entry(row.to_s, arg, flags)
-                 else
-                   filter_row_entry(row[arg.field].to_s, arg, flags)
-                 end
+        puts render_table(output, flags)
 
-      # Pivot of off include vs exclude
-      if arg.bang
-        !included
-      else
-        included
-      end
-    end
-
-    # Field Partial / Case / Exact search
-    def self.filter_row_entry(entry, arg, flags)
-      # Exact Matching / Unless doing full text search
-      return entry.to_s == arg.value.to_s if flags.key?(:exact) && arg.field != :text
-
-      if flags.key?(:case)
-        entry.include? arg.value.to_s
-      else
-        entry.downcase.include? arg.value.to_s.downcase
-      end
-    end
-
-    # Total Count Helper
-    def self.total_count(results)
-      results.each do |k, v|
-        puts k
-        puts "Total: #{v.count.to_s.pastel(:blue)}"
         puts
       end
     end
 
-    # Total Count Helper
+    # Table Printer Helper
     def self.fields_print(results)
       results.each do |k, v|
         puts k
@@ -522,8 +243,8 @@ module GreenHat
     end
 
     # Total Log List Manipulator
-    def self.prepare_list(log_list, base_list = nil, _flags = {})
-      base_list ||= GreenHat::ShellHelper::Log.list
+    def self.prepare_list(log_list, base_list = nil)
+      base_list ||= Thing.list
 
       # Assume all
       log_list.push '*' if log_list.empty?
@@ -534,9 +255,9 @@ module GreenHat
       log_list
     end
 
-    # Fuzzy match for things
+    # Fuzzy match for things / List used for processable (Hash Entries)
     def self.thing_list
-      @thing_list ||= Thing.all.map(&:name)
+      @thing_list ||= Thing.list.map(&:name)
 
       @thing_list
     end
@@ -548,7 +269,7 @@ module GreenHat
         return file if file.instance_of?(Thing)
 
         if flags.fuzzy_file_match
-          Thing.all.select { |x| x.name.include? file }
+          Thing.list.select { |x| x.name.include?(file) || x.type.include?(file) }
         else
           Thing.where name: file
         end
@@ -560,62 +281,6 @@ module GreenHat
       things
     end
 
-    # Main Entry Point for Searching
-    # def self.search_start(log_list, filter_type, args, opts)
-    def self.search_start(files, flags, args)
-      # Convert to Things
-      logs = ShellHelper.find_things(files, flags)
-
-      logs.each_with_object({}) do |log, obj|
-        # Ignore Empty Results / No Thing
-        next if log&.data.blank?
-
-        obj[log.friendly_name] = ShellHelper.search(log.data, flags, args)
-
-        obj
-      end
-    end
-
-    # Generic Search Helper / String/Regex
-    def self.search(data, flags = {}, args = {})
-      results = data.clone.flatten.compact
-      results.select! do |row|
-        args.send(flags.logic) do |arg|
-          search_row(row, arg, flags)
-        end
-      end
-
-      # Strip Results if Slice is defined
-      results.map! { |row| row.slice(*flags[:slice]) } if flags[:slice]
-
-      # Strip Results if Except is defined
-      results.map! { |row| row.except(*flags[:except]) } if flags[:except]
-
-      # Remove Blank from either slice or except
-      results.reject!(&:empty?)
-
-      results
-    end
-
-    # Break out filter row logic into separate method
-    def self.search_row(row, arg, flags)
-      # Sensitivity Check / Check for Match
-      included = filter_row_entry(row.to_s, arg, flags)
-
-      # Pivot of off include vs exclude
-      if arg.bang
-        !included
-      else
-        included
-      end
-    end
-
-    # TODO: Remove?
-    # Color Reader Helper
-    # def self.pastel
-    #   @pastel ||= Pastel.new
-    # end
-
     # Number Helper
     # https://gitlab.com/zedtux/human_size_to_number/-/blob/master/lib/human_size_to_number/helper.rb
     def self.human_size_to_number(string)
@@ -637,23 +302,6 @@ module GreenHat
       number.round
     end
 
-    # TODO: Needed?
-    def self.filter_and(data, params = {})
-      result = data.clone.flatten.compact
-      params.each do |k, v|
-        result.select! do |row|
-          if row.key? k.to_sym
-            row[k.to_sym].include? v
-          else
-            false
-          end
-        end
-        next
-      end
-
-      result
-    end
-
     # General Helper for `show`
     def self.common_opts
       puts 'Common Options'.pastel(:blue)
@@ -667,5 +315,6 @@ module GreenHat
       puts
     end
   end
+
   # rubocop:enable Metrics/ModuleLength
 end

data/lib/greenhat/shell.rb

@@ -7,10 +7,25 @@ module GreenHat
       # rubocop:enable Lint/Debugger
     end
 
+    def self.settings
+      Settings.configure_settings
+    end
+
+    def self.web
+      # Load Required Files
+      require 'greenhat/web'
+
+      GreenHat::Web.toggle
+    end
+
     def self.df
       Disk.df
     end
 
+    def self.quit(_raw = {})
+      exit 0
+    end
+
     def self.ps(raw = {})
       Process.ps raw
     end
@@ -23,12 +38,12 @@ module GreenHat
       Memory.free
     end
 
-    def self.load_local
-      return false unless TTY::Prompt.new.yes?('Load local Omnibus GitLab Instance files?')
+    def self.load_local(prompt = true)
+      return false if prompt && !TTY::Prompt.new.yes?('Load local Omnibus GitLab Instance files?')
 
       archive_path = "#{$TMP}/#{Time.now.to_i}_local"
       Dir.mkdir(archive_path)
-      # archive = Archive.new(name: 'local')
+
       archive = Archive.new(name: archive_path, path: archive_path)
 
       file_list = Dir['/var/log/gitlab/*/current'] + Dir['/var/log/gitlab/*/*.log']
@@ -53,7 +68,7 @@ module GreenHat
     end
 
     def self.quiet
-      Cli.quiet!
+      Cli.quiet_toggle
 
       if Cli.quiet
         puts "GreenHat Quiet Logging #{'Off'.pastel(:red)}"

data/lib/greenhat/thing/file_types.rb

@@ -67,7 +67,8 @@ module GreenHat
         'etc/fstab' => {
           format: :clean_raw,
           pattern: [
-            %r{etc/fstab}
+            %r{etc/fstab},
+            /fstab/
           ]
         },
         'etc/os-release' => {
@@ -94,6 +95,12 @@ module GreenHat
             /getenforce/
           ]
         },
+        'geo-logcursor/current' => {
+          format: :raw,
+          pattern: [
+            %r{geo-logcursor/current}
+          ]
+        },
         'gitaly/current' => {
           format: :json_shell,
           log: true,
@@ -102,6 +109,13 @@ module GreenHat
             %r{gitaly/@.*}
           ]
         },
+        'gitaly/gitaly_hooks.log' => {
+          format: :shellwords,
+          log: true,
+          pattern: [
+            %r{gitaly/gitaly_hooks\.log}
+          ]
+        },
         'consul/current' => {
           format: :time_space,
           log: true,
@@ -351,6 +365,13 @@ module GreenHat
             /nginx-ingress.log/
           ]
         },
+        'nginx/access.log' => {
+          format: :nginx,
+          log: true,
+          pattern: [
+            %r{nginx/access.log}
+          ]
+        },
         'nginx/gitlab_pages_access.log' => {
           format: :nginx,
           log: true,
@@ -358,6 +379,13 @@ module GreenHat
             %r{nginx/gitlab_pages_access.log}
           ]
         },
+        'nginx/gitlab_pages_error.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{nginx/gitlab_pages_error.log}
+          ]
+        },
         'nginx/gitlab_access.log' => {
           format: :nginx,
           log: true,
@@ -546,6 +574,20 @@ module GreenHat
             %r{unicorn/current}
           ]
         },
+        'unicorn_stats' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            /unicorn_stats/
+          ]
+        },
+        'unicorn/unicorn_stdout.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{unicorn/unicorn_stdout.log}
+          ]
+        },
         'running_swappiness' => {
           format: :raw,
           pattern: [
@@ -583,6 +625,7 @@ module GreenHat
           log: true,
           pattern: [
             %r{sidekiq/current},
+            /sidekiq_current/,
             %r{sidekiq/@.*}
           ]
         },
@@ -646,6 +689,12 @@ module GreenHat
             /uname/
           ]
         },
+        'gitlab_socket_stats' => {
+          format: :gitlab_socket_stats,
+          pattern: [
+            /gitlab_socket_stats/
+          ]
+        },
         'uptime' => {
           format: :raw,
           pattern: [
@@ -769,6 +818,7 @@ module GreenHat
         'gitlab-rails/production.log' => {
           format: :raw,
           pattern: [
+            /production.log/,
             %r{gitlab-rails/production.log},
             %r{webservice.log/production.log}
           ]