greenhat 0.4.0 → 0.6.1

data/lib/greenhat/thing/file_types.rb

@@ -27,7 +27,8 @@ module GreenHat
         'df_h' => {
           format: :table,
           pattern: [
-            /df_h/
+            /df_h/,
+            /df_hT/
           ]
         },
         'df_inodes' => {
@@ -64,6 +65,13 @@ module GreenHat
             %r{log/syslog}
           ]
         },
+        'gitlab-runner.log' => {
+          format: :runner_log,
+          log: true,
+          pattern: [
+            /gitlab-runner.log/
+          ]
+        },
         'etc/fstab' => {
           format: :clean_raw,
           pattern: [
@@ -123,6 +131,13 @@ module GreenHat
             %r{consul/current}
           ]
         },
+        'gitlab-kas/current' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            %r{gitlab-kas/current}
+          ]
+        },
         'sentinel/current' => {
           format: :time_space,
           log: true,
@@ -151,6 +166,13 @@ module GreenHat
             %r{mailroom/mail_room_json.log}
           ]
         },
+        'gitlab_migrations' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            /gitlab_migrations/
+          ]
+        },
         'gitlab-rails/git_json.log' => {
           format: :json,
           log: true,
@@ -379,6 +401,13 @@ module GreenHat
             %r{nginx/gitlab_pages_access.log}
           ]
         },
+        'nginx/gitlab_pages_error.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{nginx/gitlab_pages_error.log}
+          ]
+        },
         'nginx/gitlab_access.log' => {
           format: :nginx,
           log: true,

data/lib/greenhat/thing/formatters/api_json.rb

@@ -11,9 +11,10 @@ module GreenHat
         result = Oj.load row
 
         # Parsing Time
-        format_json_traverse result
+        format_time(result)
 
-        flatten_hash(result).sort.to_h
+        # flatten_hash(result).sort.to_h
+        result
       rescue StandardError => e
         # TODO: Background Logger?
         e.message
@@ -29,6 +30,7 @@ module GreenHat
         v.is_a?(Hash) ? a.merge(flatten_hash(v, "#{prefix}#{k}.")) : a.merge("#{prefix}#{k}".to_sym => v)
       end
     end
+
     # ==========================================================================
   end
 end

data/lib/greenhat/thing/formatters/bracket_log.rb

@@ -25,7 +25,7 @@ module GreenHat
         result
       rescue StandardError => e
         # TODO: Background logger
-        LogBot.fatal('dmesg', "Unable to Parse, #{row}:#{e.message}")
+        LogBot.warn('dmesg', "Unable to Parse, #{row}:#{e.message}")
       end
 
       self.result.compact!

data/lib/greenhat/thing/formatters/clean_raw.rb

@@ -4,7 +4,7 @@ module GreenHat
   module Formatters
     # Remove Comments / Empty Lines
     def format_clean_raw
-      staging = raw.clone
+      staging = raw_full.clone
 
       # Empty
       staging.reject!(&:empty?)

data/lib/greenhat/thing/formatters/colon_split_strip.rb

@@ -4,9 +4,9 @@ module GreenHat
   module Formatters
     # Formatters for single json blobs in entire file
     def format_colon_split_strip
-      self.result = raw.map do |row|
+      self.result = raw_full.to_h do |row|
         row.split(':', 2).map(&:strip)
-      end.to_h
+      end
     end
   end
 end

data/lib/greenhat/thing/formatters/dotenv.rb

@@ -4,7 +4,7 @@ module GreenHat
   module Formatters
     # Formatters for Dmesg
     def format_dotenv
-      self.result = Dotenv::Parser.new(raw.join("\n")).call
+      self.result = Dotenv::Parser.new(raw_full.join("\n")).call
     end
   end
 end

data/lib/greenhat/thing/formatters/format.rb

@@ -1,12 +1 @@
-module GreenHat
-  # Formatter
-  module Formatters
-    def log_format
-      self.result = send(SuperLog.type?(path)) if log_type
-    end
 
-    def log_type
-      SuperLog.type?(path)
-    end
-  end
-end

data/lib/greenhat/thing/formatters/free_m.rb

@@ -5,7 +5,7 @@ module GreenHat
     # Get Split Memory Table
     def format_free_m
       # Headers to Readable Symbol
-      headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
+      headers = raw_full.first.split.map(&:downcase).map do |x|
         x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
       end.map(&:to_sym)
 
@@ -15,7 +15,7 @@ module GreenHat
       final = []
 
       # Put fields into a Hash based on Location/Key
-      raw[1..].map(&:split).each do |row|
+      raw_full[1..].map(&:split).each do |row|
         result = {}
         row.each_with_index do |detail, i|
           result[headers[i]] = detail.split(':').first

data/lib/greenhat/thing/formatters/json.rb

@@ -14,10 +14,10 @@ module GreenHat
           { message: row }
         end
 
-        # Parsing Time
-        format_json_traverse result
+        # Parsing Time / Recusrive is really slow
+        format_time(result)
 
-        result.sort.to_h
+        result
       rescue StandardError => e
         # TODO: Background Logger?
         e.message
@@ -27,11 +27,12 @@ module GreenHat
 
       :ok
     end
+
     # ==========================================================================
 
-    # Recursively Navigate
+    # Recursively Navigate -- Recusrive is really slow
     def format_json_traverse(result)
-      format_json_time(result)
+      format_time(result)
 
       result.each do |_key, value|
         next unless value.instance_of? Hash
@@ -40,27 +41,50 @@ module GreenHat
       end
     end
 
+    def format_time_fields
+      %i[time timestamp created_at enqueued_at completed_at]
+    end
+
+    # Loop through potential fields and parse as needed
+    def format_time(result)
+      format_time_fields.each do |field|
+        result[field] = format_time_parse(result[field]) if result.key?(field)
+      rescue StandardError => e
+        LogBot.warn('JSON Format Time', e.message)
+        next
+      end
+    end
+
     # Check for Common Fields
-    def format_json_time(result)
-      result.time = format_time_parse(result.time) if result.key? :time
+    # def format_json_time(result)
+    #   if result.key? :time
+    #     result.time = format_time_parse(result.time)
+    #   elsif result.key?(:timestamp)
+    #     result.time = format_time_parse(result.timestamp)
+    #   end
 
-      # If timestamp exists but time doesn't, add time
-      result.time = format_time_parse(result.timestamp) if result.key?(:timestamp) && !result.key?(:time)
+    #   result.created_at = format_time_parse(result.created_at) if result.key? :created_at
+    #   result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
+    # rescue StandardError => e
+    #   LogBot.warn("JSON Time Parse", e.message)
+    #   true
+    # end
 
-      result.created_at = format_time_parse(result.created_at) if result.key? :created_at
-      result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
-    rescue StandardError => e
-      LogBot.warn('JSON Time Parse', e.message)
-      true
+    def format_time_parse(time)
+      Time.parse time
+    rescue StandardError
+      # Handle Epoch Timestamps as well as string timestamps
+      format_time_numeric(time)
     end
 
-    # Handle Epoch Timestamps as well as string timestamps
-    def format_time_parse(time)
+    def format_time_numeric(time)
       if time.numeric?
         Time.at time
       else
-        Time.parse time
+        time
       end
+    rescue StandardError => e
+      LogBot.warn('JSON Time Parse', e.message)
     end
     # ---
   end

data/lib/greenhat/thing/formatters/json_shellwords.rb

@@ -14,9 +14,9 @@ module GreenHat
         end
 
         # Parsing Time
-        format_json_traverse result
+        format_time(result)
 
-        result.sort.to_h
+        result
       rescue StandardError => e
         LogBot.warn('JSON Parse', e.message)
         next
@@ -36,6 +36,7 @@ module GreenHat
 
       result
     end
+
     # ==========================================================================
   end
 end

data/lib/greenhat/thing/formatters/kube_json.rb

@@ -19,9 +19,9 @@ module GreenHat
         end
 
         # Parsing Time
-        format_json_traverse result
+        format_time(result)
 
-        result.sort.to_h
+        result
       rescue StandardError => e
         # TODO: Background Logger?
         e.message
@@ -31,6 +31,7 @@ module GreenHat
 
       :ok
     end
+
     # =========================================================================
   end
 end

data/lib/greenhat/thing/formatters/multiline_json.rb

@@ -4,7 +4,7 @@ module GreenHat
   module Formatters
     # Formatters for single json blobs in entire file
     def format_multiline_json
-      self.result = Oj.load raw.join
+      self.result = Oj.load raw_full.join
     end
   end
 end

data/lib/greenhat/thing/formatters/nginx.rb

@@ -6,7 +6,9 @@ module GreenHat
     # Formatters
     # ==========================================================================
     def format_nginx
-      self.result = raw.map do |row|
+      output = raw.map do |row|
+        next if row.chomp.empty?
+
         conn_data, rest = row.split(' [', 2)
         ip, _sym, remote_user = conn_data.split(' ', 3)
 
@@ -35,6 +37,8 @@ module GreenHat
         { msg: row }
       end
 
+      # Remove Empty Entries
+      self.result = output.compact
       :ok
     end
 

data/lib/greenhat/thing/formatters/runner_log.rb

@@ -0,0 +1,70 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Attempt to make runner logs easier to parse
+    def format_runner_log
+      self.result = raw.map do |row|
+        next if row.empty? || row == "\n"
+        next if row.include? '-- Logs begin at' # Skip Header
+
+        process_runner_log_row(row)
+      end
+
+      result.compact!
+    end
+
+    # Split / Parse Time
+    # TODO: Better sys log parsing? Cannot find ready-made regex/ruby parser
+    def process_runner_log_row(row)
+      month, day, time, device, service, message = row.split(' ', 6)
+      service&.gsub!(':', '')
+      pid = service&.match(/\[(.*?)\]/)&.captures&.first
+
+      entry = {
+        time: format_time_parse("#{month} #{day} #{time}"),
+        device: device,
+        service: service,
+        message: message,
+        pid: pid,
+        row: row # Insert full row for different timestamp formats
+      }
+
+      # Remove Empty Keys
+      entry.compact!
+
+      # Shellwords split - Skip Blank
+      process_runner_row_message(message, entry) unless message.blank?
+
+      entry
+
+    # Return everything incase of error
+    rescue StandardError => e
+      LogBot.warn('Runner Log Parse', e.message)
+      {
+        message: row
+      }
+    end
+
+    def process_runner_row_message(message, entry)
+      # Insert Keywords / Format Time if Possible
+      Shellwords.split(message).select { |x| x.include? '=' }.each do |x|
+        key, value = x.split('=', 2)
+
+        # Don't overwrite stuff
+        case key
+        when 'time'
+          entry["#{key}-msg".to_sym] = format_time_parse value
+        when 'message', :message
+          entry["#{key}-msg".to_sym] = value
+        else
+          entry[key.to_sym] = value
+        end
+      end
+    rescue StandardError
+      # LogBot.debug('Runner Log Parse', e.message)
+      nil
+    end
+    # --
+  end
+end

data/lib/greenhat/thing/formatters/table.rb

@@ -2,17 +2,31 @@
 module GreenHat
   # Log
   module Formatters
-    # Formatters for Dmesg
+    # Looks like this may be different per Type
+    def header_count
+      case name
+      when 'df_hT' then 7
+      end
+    end
+
+    # Table Formatting
     def format_table
+      # Default to split by space unless specifically needed
+      headers_raw = if header_count
+                      raw_full.first.split(' ', header_count)
+                    else
+                      raw_full.first.split
+                    end
+
       # Headers to Readable Symbol
-      headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
+      headers = headers_raw.map(&:downcase).map do |x|
         x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
       end.map(&:to_sym)
 
       final = []
 
       # Put fields into a Hash based on Location/Key
-      raw[1..].map(&:split).each do |row|
+      raw_full.map(&:split)[1..].each do |row|
         result = {}
         row.each_with_index do |detail, i|
           result[headers[i]] = detail

data/lib/greenhat/thing/formatters/time_json.rb

@@ -10,10 +10,21 @@ module GreenHat
       self.result = raw.map do |row|
         time, msg = row.split(' ', 2)
 
-        result =  Oj.load msg
+        # Be mo fault tolerant
+        result =  begin
+          Oj.load msg
+        rescue StandardError
+          { msg: msg }
+        end
+
         result.time = time
 
         result
+      rescue StandardError => e
+        # TODO: Background Logger?
+        e.message
+        LogBot.warn('Time JSON Parse', e.message)
+        next
       end
     end
     # ==========================================================================

data/lib/greenhat/thing/helpers.rb

@@ -25,17 +25,6 @@ module GreenHat
         tmp_path.split('/').last(2).join(divider)
       end
     end
-
-    # Check what kind of file we have
-    def type_check
-      if info?
-        :info
-      elsif SuperLog.type?(path)
-        :log
-      else
-        :raw
-      end
-    end
   end
 end
 

data/lib/greenhat/thing/info_format.rb

@@ -64,15 +64,15 @@ module GreenHat
       conns, sockets = raw.split { |x| x.include? 'Active' }.reject(&:empty?)
 
       formatted_conns = conns[1..].map do |entry|
-        entry.split(' ', 7).map(&:strip).each_with_index.map do |field, idx|
+        entry.split(' ', 7).map(&:strip).each_with_index.to_h do |field, idx|
           [format_netstat_conn_headers[idx], field]
-        end.to_h
+        end
       end
 
       formatted_sockets = sockets[1..].map do |entry|
-        entry.split('  ').map(&:strip).reject(&:blank?).each_with_index.map do |field, idx|
+        entry.split('  ').map(&:strip).reject(&:blank?).each_with_index.to_h do |field, idx|
           [format_netstat_socket_headers[idx], field]
-        end.to_h
+        end
       end
 
       {

data/lib/greenhat/thing/kind.rb

@@ -84,7 +84,7 @@ module GreenHat
       option = prompt.select('Wat is this?', prompt_list.keys.sort_by(&:length), filter: true)
 
       # Store for later
-      ThingHistory.add(name, option)
+      ThingHistory.add(name.to_sym, option)
 
       self.type = option
     end

data/lib/greenhat/thing.rb

@@ -24,8 +24,7 @@ class Thing < Teron
 
   # Preloader Flags
   field :parsed # Flag for Parsing
-  field :result # Processed Data
-  field :raw_result # Flag for reading raw data
+  field :result, dump: false # Processed Data
   field :result_fields # All available fields
 
   def friendly_name
@@ -57,10 +56,14 @@ class Thing < Teron
     result_fields
   end
 
+  # Readlines rather than putting whole thing into memory
   def raw
-    raw_read if raw_result.nil?
+    File.foreach(file)
+  end
 
-    raw_result
+  # Where full read is needed
+  def raw_full
+    File.read(file).split("\n")
   end
 
   # Filter / Archive Pattern Matching
@@ -73,32 +76,27 @@ class Thing < Teron
     data.blank?
   end
 
-  def raw_read
-    spin_start("Read  #{name.pastel(:blue)} #{size.pastel(:bright_black)}")
-    self.raw_result = File.read(file).split("\n")
-  rescue StandardError => e
-    LogBot.fatal('Raw Read', message: e.message, backtrace: e.backtrace.first)
-    self.raw_result = ''
-  ensure
-    spin_done
-  end
-
   def output(print_it = true)
     if print_it
-      puts raw.join("\n")
+      puts raw.map(&:to_s).map(&:chomp)
     else
-      raw
+      raw.map(&:to_s).map(&:chomp)
     end
   end
 
-  # Log Identifier Helper
-  def log?
-    GreenHat::SuperLog.log? kind
+  # Hashed values searching
+  def process?
+    kind != :raw && methods.include?(formatter)
   end
 
-  # Hashed values searching
-  def processed?
-    kind != :raw
+  # Things that can be queried (Query Helper)
+  def query?
+    data.instance_of?(Array)
+  end
+
+  # Helper for all things that can be hash/value filtered
+  def self.list
+    all.select.select(&:process?)
   end
 
   # Helper Formatter Method
@@ -107,9 +105,6 @@ class Thing < Teron
   end
 
   def process
-    # Read First if Needed (Spinner)
-    raw_read if raw_result.nil?
-
     if methods.include? formatter
       spin_start("Parse #{name.pastel(:blue)} #{kind.to_s.pastel(:bright_black)} ")
       begin
@@ -148,6 +143,7 @@ class Thing < Teron
     self.log = true
     self.parsed = true
     self.type = 'json'
+    self.kind = :json
     self.result_fields = field_processing
     save!
   end

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.4.0'.freeze
+  VERSION = '0.6.1'.freeze
 end

data/lib/greenhat.rb

@@ -24,6 +24,7 @@ require 'tty-table'
 require 'tty-which'
 require 'warning'
 require 'ruby_native_statistics'
+require 'gitlab_chronic_duration'
 
 # Custom Gem
 require 'teron'
@@ -54,8 +55,12 @@ require 'greenhat/logbot'
 require 'greenhat/settings'
 require 'greenhat/color'
 
+# Printer Helpers
+require 'greenhat/paper/paper_helper'
+require 'greenhat/paper/flag_helper'
+require 'greenhat/paper'
+
 # Formatters - Loads Required Files Automatically
-require 'greenhat/thing/super_log'
 require 'greenhat/thing/file_types'
 require 'greenhat/thing/kind'
 require 'greenhat/thing/history'
@@ -68,14 +73,7 @@ require 'greenhat/thing'
 # Shell - Loads Required Files Automatically
 require 'greenhat/shell'
 
-# TODO: Confirm
-# require 'greenhat/host'
-# require 'greenhat/web'
-
-require 'greenhat/pry_helpers'
-
 ## TTY Shims
-
 require 'greenhat/tty/line'
 require 'greenhat/tty/reader'
 require 'greenhat/tty/custom_line'