greenhat 0.4.0 → 0.5.0

data/lib/greenhat/shell/shell_helper.rb

@@ -74,7 +74,7 @@ module GreenHat
       when Hash then render_table(entry, flags)
       when Float, Integer, Array
         format_table_entry(flags, entry, key)
-      # Ignore Special Formatting for Strings / Usually already formatted
+        # Ignore Special Formatting for Strings / Usually already formatted
       when String
         entry
       else
@@ -90,13 +90,12 @@ module GreenHat
                         when Float, Integer || entry.numeric?
                           flags.key?(:round) ? entry.to_f.round(flags.round).ai : entry.ai
 
-                          # General Inspecting
+                        # General Inspecting
                         when Hash then entry.ai(ruby19_syntax: true)
 
                         # Arrays often contain Hashes. Dangerous Recursive?
                         when Array
                           entry.map { |x| format_table_entry(flags, x) }.join("\n")
-
                         when Time
                           entry.to_s.pastel(:bright_white)
 
@@ -120,7 +119,7 @@ module GreenHat
 
     # Print the Table in a Nice way
     def self.render_table(entry, flags)
-      entry = entry.map { |k, v| [k, format_table_entry(flags, v, k)] }.to_h
+      entry = entry.to_h { |k, v| [k, format_table_entry(flags, v, k)] }
       # Pre-format Entry
 
       table_style = flags[:table_style]&.to_sym || :unicode
@@ -168,201 +167,7 @@ module GreenHat
       # Default to everything
       files = Thing.all.map(&:name) if files.empty?
 
-      ShellHelper.filter_start(files, flags, args)
-    end
-
-    # Main Entry Point for Filtering
-    def self.filter_start(files, flags = {}, args = {})
-      # Convert to Things
-      logs = ShellHelper.find_things(files, flags).select(&:processed?)
-
-      # Ignore Archive/Host Dividers
-      if flags[:combine]
-        results = logs.reject(&:blank?).map(&:data).flatten.compact
-        ShellHelper.filter(results, flags, args)
-      else
-        # Iterate and Preserve Archive/Host Index
-        logs.each_with_object({}) do |log, obj|
-          # Ignore Empty Results / No Thing
-          next if log&.blank?
-
-          # Include Total Count in Name
-          results = ShellHelper.filter(log.data, flags, args)
-          duration = calculate_duration(results)
-
-          title = [
-            log.friendly_name,
-            " #{results.count}".pastel(:bright_black)
-
-          ]
-
-          # Append Duration
-          title.push(" #{duration.pastel(:cyan, :dim)}") unless duration.blank?
-
-          # Save unless empty
-          obj[title.join] = results unless results.count.zero?
-
-          obj
-        end
-      end
-    end
-
-    def self.calculate_duration(results)
-      # Skip for Pluck
-      only_with_time = results.select { |x| x.instance_of?(Hash) && x.key?(:time) }
-
-      # If slice is used ignore
-      return nil if only_with_time.empty?
-
-      sorted = only_with_time.map(&:time).sort
-      humanize_time(sorted.first, sorted.last)
-    end
-
-    # Replace TimeDifference with https://stackoverflow.com/a/4136485/1678507
-    def self.humanize_time(time_start, time_end, increments = 2)
-      miliseconds = (time_end - time_start) * 1000
-
-      list = [[1000, :ms], [60, :s], [60, :m], [24, :h]].map do |count, name|
-        next unless miliseconds.positive?
-
-        miliseconds, n = miliseconds.divmod(count)
-
-        "#{n.to_i}#{name}" unless n.to_i.zero?
-      end
-
-      list.compact.reverse[0..increments - 1].join(' ')
-    end
-
-    # Filter Logic
-    # TODO: Simplify
-    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-    def self.filter(data, flags = {}, args = {})
-      # results = data.clone.flatten.compact
-
-      # Experimenting with deep clone
-      results = Marshal.load(Marshal.dump(data))
-      results.select! do |row|
-        args.send(flags.logic || :all?) do |arg|
-          filter_row_key(row, arg, flags)
-        end
-      end
-
-      # Ensure presecense of a specific field
-      results = filter_exists(results, flags[:exists]) if flags.key?(:exists)
-
-      # Time Zone
-      results = filter_modify_timezone(results, flags[:time_zone]) if flags.key?(:time_zone)
-
-      # Time Filtering
-      results = filter_time(results, flags) if flags.key?(:start) || flags.key?(:end)
-
-      # Strip Results if Slice is defined
-      results = filter_slice(results, flags[:slice]) if flags.key?(:slice)
-
-      # Strip Results if Except is defined
-      results = filter_except(results, flags[:except]) if flags.key?(:except)
-
-      # Remove Blank from either slice or except
-      results.reject!(&:empty?)
-
-      # Sort
-      results.sort_by! { |x| x.slice(*flags[:sort]).values } if flags.key?(:sort)
-
-      # JSON Formatting
-      results = results.map { |x| Oj.dump(x) } if flags.key?(:json)
-
-      # Show Unique Only
-      results = filter_uniq(results, flags[:uniq]) if flags.key?(:uniq)
-
-      # Reverse
-      results.reverse! if flags[:reverse]
-
-      # Count occurrences / Skip Results
-      return filter_stats(results, flags) if flags.key?(:stats)
-
-      # Limit before Pluck / Flattening
-      results = filter_limit(results, flags[:limit]) if flags.key?(:limit)
-
-      # Pluck
-      results = filter_pluck(results, flags[:pluck]) if flags.key?(:pluck)
-
-      results
-    end
-    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-
-    # Limit / Ensure Exists and Valid Number
-    def self.filter_limit(results, limit)
-      return results unless limit.integer? && limit.positive?
-
-      results.shift limit
-    end
-
-    def self.filter_modify_timezone(results, time_zone)
-      results.each do |x|
-        next unless x.key? :time
-
-        x[:time] = x[:time].in_time_zone time_zone
-      end
-
-      results
-    end
-
-    # Filter Start and End Times
-    # TODO: This is a bit icky, simplify/dry
-    def self.filter_time(results, flags)
-      if flags.key?(:start)
-        begin
-          time_start = Time.parse(flags[:start])
-
-          results.select! do |x|
-            if x.time
-              time_start < x.time
-            else
-              true
-            end
-          end
-        rescue StandardError
-          puts 'Unable to Process Start Time Filter'.pastel(:red)
-        end
-      end
-
-      if flags.key?(:end)
-        begin
-          time_start = Time.parse(flags[:end])
-
-          results.select! do |x|
-            if x.time
-              time_start > x.time
-            else
-              true
-            end
-          end
-        rescue StandardError
-          puts 'Unable to Process End Time Filter'.pastel(:red)
-        end
-      end
-
-      results
-    end
-
-    def self.filter_except(results, except)
-      # Avoid Empty Results
-      if except.empty?
-        filter_empty_arg('except')
-        return results
-      end
-
-      results.map { |row| row.except(*except) }
-    end
-
-    def self.filter_exists(results, exists)
-      # Avoid Empty Results
-      if exists.empty?
-        filter_empty_arg('exists')
-        return results
-      end
-
-      results.select { |row| (exists - row.keys).empty? }
+      Query.start(files, flags, args)
     end
 
     def self.entry_truncate(entry, truncate)
@@ -376,162 +181,6 @@ module GreenHat
       "#{entry.to_s[0..truncate]} #{'...'.pastel(:bright_blue)}"
     end
 
-    def self.filter_slice(results, slice)
-      # Avoid Empty Results
-      if slice.empty?
-        filter_empty_arg('slice')
-        return results
-      end
-
-      results.map { |row| row.slice(*slice) }
-    end
-
-    def self.filter_pluck(results, pluck)
-      # Avoid Empty Results
-      if pluck.empty?
-        filter_empty_arg('pluck')
-        return results
-      end
-
-      results.map { |x| x.slice(*pluck).values }.flatten
-    end
-
-    def self.filter_uniq(results, unique)
-      # Avoid Empty Results
-      if unique.empty?
-        filter_empty_arg('uniq')
-        return results
-      end
-
-      unique.map do |field|
-        results.uniq { |x| x[field] }
-      end.inject(:&)
-    end
-
-    def self.filter_stats(results, flags)
-      stats = flags[:stats]
-
-      # Avoid Empty Results
-      if stats.empty?
-        filter_empty_arg('stats')
-        return results
-      end
-
-      # Loop through Stats, Separate Hash/Tables
-      stats.map do |field|
-        occurrences = filter_count_occurrences(results, field, flags)
-
-        # Use Truncate For Long Keys
-        occurrences.transform_keys! { |key| key.to_s[0..flags[:truncate]] } if flags[:truncate]
-
-        # Total Occurences
-        total = occurrences.values.sum
-
-        # Percs
-        occurrences.transform_values! do |count|
-          [
-            count,
-            " #{percent(count, total)}%".pastel(:bright_black)
-          ]
-        end
-
-        # Sort by total occurances / New Variable for Total
-        output = occurrences.sort_by(&:last).to_h.transform_values!(&:join).to_a
-
-        # Append Header / Total with field name
-        output.unshift([field.to_s.pastel(:bright_black), total])
-
-        # Format
-        output.to_h
-      end
-    end
-
-    # Percent Helper
-    def self.percent(value, total)
-      ((value / total.to_f) * 100).round
-    end
-
-    # Helper to Count occurrences
-    def self.filter_count_occurrences(results, field, flags = {})
-      results.each_with_object(Hash.new(0)) do |entry, counts|
-        if entry.key? field
-          # Rounding in pagination breaks stats
-          key = if flags.key?(:round) && entry[field].numeric?
-                  entry[field].to_f.round(flags.round)
-                else
-                  entry[field]
-                end
-
-          counts[key] += 1
-        else
-          counts['None'.pastel(:bright_black)] += 1
-        end
-
-        counts
-      end
-    end
-
-    def self.filter_empty_arg(arg)
-      puts [
-        'Ignoring'.pastel(:bright_yellow),
-        "--#{arg}".pastel(:cyan),
-        'it requires an argument'.pastel(:red)
-      ].join(' ')
-    end
-
-    # Break out filter row logic into separate method
-    def self.filter_row_key(row, arg, flags)
-      # Ignore Other Logic if Field isn't even included / Full Text Searching
-      return false unless row.key?(arg[:field]) || arg[:field] == :text
-
-      # Sensitivity Check / Check for Match / Full Text Searching
-      search_data = arg[:field] == :text ? row : row[arg.field]
-      match = filter_row_entry(search_data.to_s, arg, flags)
-
-      # Pivot of off include vs exclude
-      if arg.bang
-        !match
-      else
-        match
-      end
-    end
-
-    # Field Partial / Case / Exact search
-    def self.filter_row_entry(entry, arg, flags)
-      # Exact Matching / Unless doing full text search
-      return entry == arg.value.to_s if flags.key?(:exact) && arg.field != :text
-
-      # Cast to String/Integer Helper
-      entry, value = filter_entry_cast(entry, arg, flags)
-
-      entry.send(arg.logic, value)
-    end
-
-    # Handle casting to strings or integers
-    def self.filter_entry_cast(entry, arg, flags)
-      # Cast to String
-      value = arg.value.to_s
-
-      case arg.logic
-      when :include?
-
-        # Exact Case argument
-        unless flags.key?(:case)
-          entry.downcase!
-          value.downcase!
-        end
-      when :>=, :<=
-        entry = entry.to_i if entry.numeric?
-        value = value.to_i if value&.numeric?
-      end
-
-      [entry, value]
-    end
-
-    def self.filter_entry_logic(entry, arg)
-      entry.send(arg.logic, arg.value)
-    end
-
     # Total Count Helper
     def self.total_count(results, flags)
       results.map do |k, v|
@@ -541,7 +190,7 @@ module GreenHat
 
         output = {}
         output[:total] = v.count
-        output[:duration] = calculate_duration(v)
+        output[:duration] = Query.calculate_duration(v)
 
         # Sort / Get first and Last
         list = v.select(&:time).map(&:time)
@@ -594,8 +243,8 @@ module GreenHat
     end
 
     # Total Log List Manipulator
-    def self.prepare_list(log_list, base_list = nil, _flags = {})
-      base_list ||= GreenHat::ShellHelper::Log.list
+    def self.prepare_list(log_list, base_list = nil)
+      base_list ||= Thing.list
 
       # Assume all
       log_list.push '*' if log_list.empty?
@@ -606,9 +255,9 @@ module GreenHat
       log_list
     end
 
-    # Fuzzy match for things
+    # Fuzzy match for things / List used for processable (Hash Entries)
     def self.thing_list
-      @thing_list ||= Thing.all.map(&:name)
+      @thing_list ||= Thing.list.map(&:name)
 
       @thing_list
     end
@@ -620,7 +269,7 @@ module GreenHat
         return file if file.instance_of?(Thing)
 
         if flags.fuzzy_file_match
-          Thing.all.select { |x| x.name.include?(file) || x.type.include?(file) }
+          Thing.list.select { |x| x.name.include?(file) || x.type.include?(file) }
         else
           Thing.where name: file
         end
@@ -666,5 +315,6 @@ module GreenHat
       puts
     end
   end
+
   # rubocop:enable Metrics/ModuleLength
 end

data/lib/greenhat/shell.rb

@@ -7,6 +7,10 @@ module GreenHat
       # rubocop:enable Lint/Debugger
     end
 
+    def self.settings
+      Settings.configure_settings
+    end
+
     def self.web
       # Load Required Files
       require 'greenhat/web'
@@ -18,6 +22,10 @@ module GreenHat
       Disk.df
     end
 
+    def self.quit(_raw = {})
+      exit 0
+    end
+
     def self.ps(raw = {})
       Process.ps raw
     end
@@ -60,7 +68,7 @@ module GreenHat
     end
 
     def self.quiet
-      Cli.quiet!
+      Cli.quiet_toggle
 
       if Cli.quiet
         puts "GreenHat Quiet Logging #{'Off'.pastel(:red)}"

data/lib/greenhat/thing/file_types.rb

@@ -379,6 +379,13 @@ module GreenHat
             %r{nginx/gitlab_pages_access.log}
           ]
         },
+        'nginx/gitlab_pages_error.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{nginx/gitlab_pages_error.log}
+          ]
+        },
         'nginx/gitlab_access.log' => {
           format: :nginx,
           log: true,

data/lib/greenhat/thing/formatters/api_json.rb

@@ -11,9 +11,10 @@ module GreenHat
         result = Oj.load row
 
         # Parsing Time
-        format_json_traverse result
+        format_time(result)
 
-        flatten_hash(result).sort.to_h
+        # flatten_hash(result).sort.to_h
+        result
       rescue StandardError => e
         # TODO: Background Logger?
         e.message
@@ -29,6 +30,7 @@ module GreenHat
         v.is_a?(Hash) ? a.merge(flatten_hash(v, "#{prefix}#{k}.")) : a.merge("#{prefix}#{k}".to_sym => v)
       end
     end
+
     # ==========================================================================
   end
 end

data/lib/greenhat/thing/formatters/bracket_log.rb

@@ -25,7 +25,7 @@ module GreenHat
         result
       rescue StandardError => e
         # TODO: Background logger
-        LogBot.fatal('dmesg', "Unable to Parse, #{row}:#{e.message}")
+        LogBot.warn('dmesg', "Unable to Parse, #{row}:#{e.message}")
       end
 
       self.result.compact!

data/lib/greenhat/thing/formatters/colon_split_strip.rb

@@ -4,9 +4,9 @@ module GreenHat
   module Formatters
     # Formatters for single json blobs in entire file
     def format_colon_split_strip
-      self.result = raw.map do |row|
+      self.result = raw_full.to_h do |row|
         row.split(':', 2).map(&:strip)
-      end.to_h
+      end
     end
   end
 end

data/lib/greenhat/thing/formatters/dotenv.rb

@@ -4,7 +4,7 @@ module GreenHat
   module Formatters
     # Formatters for Dmesg
     def format_dotenv
-      self.result = Dotenv::Parser.new(raw.join("\n")).call
+      self.result = Dotenv::Parser.new(raw_full.join("\n")).call
     end
   end
 end

data/lib/greenhat/thing/formatters/format.rb

@@ -1,12 +1 @@
-module GreenHat
-  # Formatter
-  module Formatters
-    def log_format
-      self.result = send(SuperLog.type?(path)) if log_type
-    end
 
-    def log_type
-      SuperLog.type?(path)
-    end
-  end
-end

data/lib/greenhat/thing/formatters/free_m.rb

@@ -5,7 +5,7 @@ module GreenHat
     # Get Split Memory Table
     def format_free_m
       # Headers to Readable Symbol
-      headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
+      headers = raw_full.first.split.map(&:downcase).map do |x|
         x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
       end.map(&:to_sym)
 
@@ -15,7 +15,7 @@ module GreenHat
       final = []
 
       # Put fields into a Hash based on Location/Key
-      raw[1..].map(&:split).each do |row|
+      raw_full[1..].map(&:split).each do |row|
         result = {}
         row.each_with_index do |detail, i|
           result[headers[i]] = detail.split(':').first

data/lib/greenhat/thing/formatters/json.rb

@@ -14,10 +14,10 @@ module GreenHat
           { message: row }
         end
 
-        # Parsing Time
-        format_json_traverse result
+        # Parsing Time / Recusrive is really slow
+        format_time(result)
 
-        result.sort.to_h
+        result
       rescue StandardError => e
         # TODO: Background Logger?
         e.message
@@ -27,11 +27,12 @@ module GreenHat
 
       :ok
     end
+
     # ==========================================================================
 
-    # Recursively Navigate
+    # Recursively Navigate -- Recusrive is really slow
     def format_json_traverse(result)
-      format_json_time(result)
+      format_time(result)
 
       result.each do |_key, value|
         next unless value.instance_of? Hash
@@ -40,27 +41,50 @@ module GreenHat
       end
     end
 
+    def format_time_fields
+      %i[time timestamp created_at enqueued_at completed_at]
+    end
+
+    # Loop through potential fields and parse as needed
+    def format_time(result)
+      format_time_fields.each do |field|
+        result[field] = format_time_parse(result[field]) if result.key?(field)
+      rescue StandardError => e
+        LogBot.warn('JSON Format Time', e.message)
+        next
+      end
+    end
+
     # Check for Common Fields
-    def format_json_time(result)
-      result.time = format_time_parse(result.time) if result.key? :time
+    # def format_json_time(result)
+    #   if result.key? :time
+    #     result.time = format_time_parse(result.time)
+    #   elsif result.key?(:timestamp)
+    #     result.time = format_time_parse(result.timestamp)
+    #   end
 
-      # If timestamp exists but time doesn't, add time
-      result.time = format_time_parse(result.timestamp) if result.key?(:timestamp) && !result.key?(:time)
+    #   result.created_at = format_time_parse(result.created_at) if result.key? :created_at
+    #   result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
+    # rescue StandardError => e
+    #   LogBot.warn("JSON Time Parse", e.message)
+    #   true
+    # end
 
-      result.created_at = format_time_parse(result.created_at) if result.key? :created_at
-      result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
-    rescue StandardError => e
-      LogBot.warn('JSON Time Parse', e.message)
-      true
+    def format_time_parse(time)
+      Time.parse time
+    rescue StandardError
+      # Handle Epoch Timestamps as well as string timestamps
+      format_time_numeric(time)
     end
 
-    # Handle Epoch Timestamps as well as string timestamps
-    def format_time_parse(time)
+    def format_time_numeric(time)
       if time.numeric?
         Time.at time
       else
-        Time.parse time
+        time
       end
+    rescue StandardError => e
+      LogBot.warn('JSON Time Parse', e.message)
     end
     # ---
   end

data/lib/greenhat/thing/formatters/json_shellwords.rb

@@ -14,9 +14,9 @@ module GreenHat
         end
 
         # Parsing Time
-        format_json_traverse result
+        format_time(result)
 
-        result.sort.to_h
+        result
       rescue StandardError => e
         LogBot.warn('JSON Parse', e.message)
         next
@@ -36,6 +36,7 @@ module GreenHat
 
       result
     end
+
     # ==========================================================================
   end
 end