greenhat 0.1.4

data/lib/greenhat/thing/formatters/api_json.rb

@@ -0,0 +1,34 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # ==========================================================================
+    # Primarily for gitlab-rails/api_json.log
+    def format_api_json
+      self.result = raw.map do |row|
+        result = Oj.load row
+
+        # Parsing Time
+        format_json_traverse result
+
+        flatten_hash(result).sort.to_h
+      rescue StandardError => e
+        # TODO: Background Logger?
+        e.message
+        LogBot.warn('JSON Parse', e.message)
+        next
+      end
+
+      :ok
+    end
+
+    def flatten_hash(param, prefix = nil)
+      param.each_pair.reduce({}) do |a, (k, v)|
+        v.is_a?(Hash) ? a.merge(flatten_hash(v, "#{prefix}#{k}.")) : a.merge("#{prefix}#{k}".to_sym => v)
+      end
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/bracket_log.rb

@@ -0,0 +1,44 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Formatters for bracket logs (dmesg, sos)
+    def format_bracket_log
+      self.result = raw.map do |row|
+        next if row.empty? || row == "\n"
+
+        result = {}
+        time, output = row.split(']', 2)
+        result[:time] = Time.parse time.split('[', 2).last.strip
+
+        if output.include? ': '
+          category, raw = output.split(': ', 2)
+          result[:category] = category.strip
+
+          result.merge! dmesg_split(raw) if raw.include? '='
+
+          result[:message] = raw.strip
+        else
+          result[:message] = output
+        end
+
+        result
+      rescue StandardError => e
+        # TODO: Background logger
+        LogBot.fatal('dmesg', "Unable to Parse, #{row}:#{e.message}")
+      end
+
+      self.result.compact!
+    end
+
+    def dmesg_split(raw)
+      Shellwords.split(raw).each_with_object({}) do |x, h|
+        key, value = x.split('=')
+        next if value.nil?
+
+        h[key] = value.numeric? ? value.to_f : value
+        h[key] = 0.0 if h[key].numeric? && h[key].zero?
+      end
+    end
+  end
+end

data/lib/greenhat/thing/formatters/clean_raw.rb

@@ -0,0 +1,23 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Remove Comments / Empty Lines
+    def format_clean_raw
+      staging = raw.clone
+
+      # Empty
+      staging.reject!(&:empty?)
+
+      # Commented
+      staging.reject! { |x| x =~ /#.*$/ }
+
+      self.result = if staging.empty?
+                      raw
+                    else
+                      staging
+                    end
+    end
+  end
+  # ----------------------------------------
+end

data/lib/greenhat/thing/formatters/colon_split_strip.rb

@@ -0,0 +1,12 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Formatters for single json blobs in entire file
+    def format_colon_split_strip
+      self.result = raw.map do |row|
+        row.split(':', 2).map(&:strip)
+      end.to_h
+    end
+  end
+end

data/lib/greenhat/thing/formatters/dotenv.rb

@@ -0,0 +1,10 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Formatters for Dmesg
+    def format_dotenv
+      self.result = Dotenv::Parser.new(raw.join("\n")).call
+    end
+  end
+end

data/lib/greenhat/thing/formatters/format.rb

@@ -0,0 +1,12 @@
+module GreenHat
+  # Formatter
+  module Formatters
+    def log_format
+      self.result = send(SuperLog.type?(path)) if log_type
+    end
+
+    def log_type
+      SuperLog.type?(path)
+    end
+  end
+end

data/lib/greenhat/thing/formatters/free_m.rb

@@ -0,0 +1,29 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Get Split Memory Table
+    def format_free_m
+      # Headers to Readable Symbol
+      headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
+        x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
+      end.map(&:to_sym)
+
+      # Add Kind
+      headers.unshift(:kind)
+
+      final = []
+
+      # Put fields into a Hash based on Location/Key
+      raw[1..].map(&:split).each do |row|
+        result = {}
+        row.each_with_index do |detail, i|
+          result[headers[i]] = detail.split(':').first
+        end
+        final.push result
+      end
+
+      self.result = final
+    end
+  end
+end

data/lib/greenhat/thing/formatters/gitlab_ctl_tail.rb

@@ -0,0 +1,51 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Gitlab Tail Formatter
+    # ==========================================================================
+    def format_gitlab_tail
+      # Revert to raw for cats
+      self.kind = :raw
+
+      output = {}
+      current_log = nil
+
+      raw.each do |line|
+        next if line.blank?
+
+        if line.include? '==>'
+          current_log = /==> (.+?) <==/.match(line).captures.first
+        else
+          output[current_log] ||= []
+          output[current_log].push line
+        end
+      end
+
+      # Remove Empty Entries
+      output.reject { |_k, v| v.empty? }
+
+      # Root Dir
+      root_dir = "#{$TMP}/#{name}"
+      Dir.mkdir(root_dir)
+
+      # Write Files / Create Things
+      output.each do |k, v|
+        file_name = k.gsub('/var/log/gitlab/', '')
+
+        dir = "#{root_dir}/#{file_name.split('/').first}"
+        Dir.mkdir(dir) unless File.exist?(dir)
+
+        File.write("#{root_dir}/#{file_name}", v.join("\n"))
+
+        # Thing Setup
+        archive.things_create(file: "#{root_dir}/#{file_name}").setup
+      end
+
+      # Link
+      self.result = raw
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/gitlab_status.rb

@@ -0,0 +1,26 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Remove Comments / Empty Lines
+    def format_gitlab_status
+      final = {}
+      raw.each do |row|
+        list = row.split('; ').map do |entry|
+          status, service, pid_uptime = entry.split(': ')
+
+          {
+            status: status,
+            name: service,
+            pid_uptime: pid_uptime
+          }
+        end
+
+        final[list.first.name] = list
+      end
+
+      self.result = final
+    end
+  end
+  # ----------------------------------------
+end

data/lib/greenhat/thing/formatters/json.rb

@@ -0,0 +1,63 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # Fallback to String
+    # ==========================================================================
+    def format_json
+      self.result = raw.map do |row|
+        result = begin
+          Oj.load row
+        rescue EncodingError
+          { message: row }
+        end
+
+        # Parsing Time
+        format_json_traverse result
+
+        result.sort.to_h
+      rescue StandardError => e
+        # TODO: Background Logger?
+        e.message
+        LogBot.warn('JSON Parse', e.message)
+        next
+      end
+
+      :ok
+    end
+    # ==========================================================================
+
+    # Recursively Navigate
+    def format_json_traverse(result)
+      format_json_time(result)
+
+      result.each do |_key, value|
+        next unless value.instance_of? Hash
+
+        format_json_traverse(value)
+      end
+    end
+
+    # Check for Common Fields
+    def format_json_time(result)
+      result.time = format_time_parse(result.time) if result.key? :time
+      result.created_at = format_time_parse(result.created_at) if result.key? :created_at
+      result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
+    rescue StandardError => e
+      LogBot.warn('JSON Time Parse', e.message)
+      true
+    end
+
+    # Handle Epoch Timestamps as well as string timestamps
+    def format_time_parse(time)
+      if time.numeric?
+        Time.at time
+      else
+        Time.parse time
+      end
+    end
+    # ---
+  end
+end

data/lib/greenhat/thing/formatters/json_shellwords.rb

@@ -0,0 +1,44 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # ==========================================================================
+    def format_json_shell
+      self.result = raw.map do |row|
+        result = begin
+          Oj.load row
+        rescue EncodingError
+          json_shellwords_fallback row
+        end
+
+        # Parsing Time
+        format_json_traverse result
+
+        result.sort.to_h
+      rescue StandardError => e
+        binding.pry
+        # TODO: Background Logger?
+        e.message
+        LogBot.warn('JSON Parse', e.message)
+        next
+      end
+
+      :ok
+    end
+
+    def json_shellwords_fallback(row)
+      result = Shellwords.split(row).each_with_object({}) do |x, h|
+        key, value = x.split('=')
+        next if value.nil?
+
+        h[key] = value.numeric? ? value.to_f : value
+      end
+      result.time = Time.parse result.time if result.key? 'time'
+
+      result
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/multiline_json.rb

@@ -0,0 +1,10 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Formatters for single json blobs in entire file
+    def format_multiline_json
+      self.result = Oj.load raw.join
+    end
+  end
+end

data/lib/greenhat/thing/formatters/raw.rb

@@ -0,0 +1,18 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters Not Handled
+    # ==========================================================================
+    def mia
+      # TODO: Background Logger
+      LogBot.warn('Log Format', "No Formatter for #{name}::#{path}")
+    end
+
+    def format_raw
+      self.result = raw
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/shellwords.rb

@@ -0,0 +1,23 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters Not Handled
+    # ==========================================================================
+    def format_shellwords
+      self.result = raw.map do |row|
+        result = Shellwords.split(row).each_with_object({}) do |x, h|
+          key, value = x.split('=')
+          next if value.nil?
+
+          h[key] = value.numeric? ? value.to_f : value
+        end
+        result.time = Time.parse result.time if result.key? 'time'
+
+        result
+      end
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/table.rb

@@ -0,0 +1,26 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # Formatters for Dmesg
+    def format_table
+      # Headers to Readable Symbol
+      headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
+        x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
+      end.map(&:to_sym)
+
+      final = []
+
+      # Put fields into a Hash based on Location/Key
+      raw[1..].map(&:split).each do |row|
+        result = {}
+        row.each_with_index do |detail, i|
+          result[headers[i]] = detail
+        end
+        final.push result
+      end
+
+      self.result = final
+    end
+  end
+end