greenhat 0.3.4 → 0.5.0

data/lib/greenhat/thing/formatters/api_json.rb

@@ -11,9 +11,10 @@ module GreenHat
         result = Oj.load row
 
         # Parsing Time
-        format_json_traverse result
+        format_time(result)
 
-        flatten_hash(result).sort.to_h
+        # flatten_hash(result).sort.to_h
+        result
       rescue StandardError => e
         # TODO: Background Logger?
         e.message
@@ -29,6 +30,7 @@ module GreenHat
         v.is_a?(Hash) ? a.merge(flatten_hash(v, "#{prefix}#{k}.")) : a.merge("#{prefix}#{k}".to_sym => v)
       end
     end
+
     # ==========================================================================
   end
 end

data/lib/greenhat/thing/formatters/bracket_log.rb

@@ -25,7 +25,7 @@ module GreenHat
         result
       rescue StandardError => e
         # TODO: Background logger
-        LogBot.fatal('dmesg', "Unable to Parse, #{row}:#{e.message}")
+        LogBot.warn('dmesg', "Unable to Parse, #{row}:#{e.message}")
       end
 
       self.result.compact!

data/lib/greenhat/thing/formatters/colon_split_strip.rb

@@ -4,9 +4,9 @@ module GreenHat
   module Formatters
     # Formatters for single json blobs in entire file
     def format_colon_split_strip
-      self.result = raw.map do |row|
+      self.result = raw_full.to_h do |row|
         row.split(':', 2).map(&:strip)
-      end.to_h
+      end
     end
   end
 end

data/lib/greenhat/thing/formatters/dotenv.rb

@@ -4,7 +4,7 @@ module GreenHat
   module Formatters
     # Formatters for Dmesg
     def format_dotenv
-      self.result = Dotenv::Parser.new(raw.join("\n")).call
+      self.result = Dotenv::Parser.new(raw_full.join("\n")).call
     end
   end
 end

data/lib/greenhat/thing/formatters/format.rb

@@ -1,12 +1 @@
-module GreenHat
-  # Formatter
-  module Formatters
-    def log_format
-      self.result = send(SuperLog.type?(path)) if log_type
-    end
 
-    def log_type
-      SuperLog.type?(path)
-    end
-  end
-end

data/lib/greenhat/thing/formatters/free_m.rb

@@ -5,7 +5,7 @@ module GreenHat
     # Get Split Memory Table
     def format_free_m
       # Headers to Readable Symbol
-      headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
+      headers = raw_full.first.split.map(&:downcase).map do |x|
         x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
       end.map(&:to_sym)
 
@@ -15,7 +15,7 @@ module GreenHat
       final = []
 
       # Put fields into a Hash based on Location/Key
-      raw[1..].map(&:split).each do |row|
+      raw_full[1..].map(&:split).each do |row|
         result = {}
         row.each_with_index do |detail, i|
           result[headers[i]] = detail.split(':').first

data/lib/greenhat/thing/formatters/json.rb

@@ -14,10 +14,10 @@ module GreenHat
           { message: row }
         end
 
-        # Parsing Time
-        format_json_traverse result
+        # Parsing Time / Recusrive is really slow
+        format_time(result)
 
-        result.sort.to_h
+        result
       rescue StandardError => e
         # TODO: Background Logger?
         e.message
@@ -27,11 +27,12 @@ module GreenHat
 
       :ok
     end
+
     # ==========================================================================
 
-    # Recursively Navigate
+    # Recursively Navigate -- Recusrive is really slow
     def format_json_traverse(result)
-      format_json_time(result)
+      format_time(result)
 
       result.each do |_key, value|
         next unless value.instance_of? Hash
@@ -40,23 +41,50 @@ module GreenHat
       end
     end
 
-    # Check for Common Fields
-    def format_json_time(result)
-      result.time = format_time_parse(result.time) if result.key? :time
-      result.created_at = format_time_parse(result.created_at) if result.key? :created_at
-      result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
-    rescue StandardError => e
-      LogBot.warn('JSON Time Parse', e.message)
-      true
+    def format_time_fields
+      %i[time timestamp created_at enqueued_at completed_at]
+    end
+
+    # Loop through potential fields and parse as needed
+    def format_time(result)
+      format_time_fields.each do |field|
+        result[field] = format_time_parse(result[field]) if result.key?(field)
+      rescue StandardError => e
+        LogBot.warn('JSON Format Time', e.message)
+        next
+      end
     end
 
-    # Handle Epoch Timestamps as well as string timestamps
+    # Check for Common Fields
+    # def format_json_time(result)
+    #   if result.key? :time
+    #     result.time = format_time_parse(result.time)
+    #   elsif result.key?(:timestamp)
+    #     result.time = format_time_parse(result.timestamp)
+    #   end
+
+    #   result.created_at = format_time_parse(result.created_at) if result.key? :created_at
+    #   result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
+    # rescue StandardError => e
+    #   LogBot.warn("JSON Time Parse", e.message)
+    #   true
+    # end
+
     def format_time_parse(time)
+      Time.parse time
+    rescue StandardError
+      # Handle Epoch Timestamps as well as string timestamps
+      format_time_numeric(time)
+    end
+
+    def format_time_numeric(time)
       if time.numeric?
         Time.at time
       else
-        Time.parse time
+        time
       end
+    rescue StandardError => e
+      LogBot.warn('JSON Time Parse', e.message)
     end
     # ---
   end

data/lib/greenhat/thing/formatters/json_shellwords.rb

@@ -14,9 +14,9 @@ module GreenHat
         end
 
         # Parsing Time
-        format_json_traverse result
+        format_time(result)
 
-        result.sort.to_h
+        result
       rescue StandardError => e
         LogBot.warn('JSON Parse', e.message)
         next
@@ -36,6 +36,7 @@ module GreenHat
 
       result
     end
+
     # ==========================================================================
   end
 end

data/lib/greenhat/thing/formatters/kube_json.rb

@@ -19,9 +19,9 @@ module GreenHat
         end
 
         # Parsing Time
-        format_json_traverse result
+        format_time(result)
 
-        result.sort.to_h
+        result
       rescue StandardError => e
         # TODO: Background Logger?
         e.message
@@ -31,6 +31,7 @@ module GreenHat
 
       :ok
     end
+
     # =========================================================================
   end
 end

data/lib/greenhat/thing/formatters/multiline_json.rb

@@ -4,7 +4,7 @@ module GreenHat
   module Formatters
     # Formatters for single json blobs in entire file
     def format_multiline_json
-      self.result = Oj.load raw.join
+      self.result = Oj.load raw_full.join
     end
   end
 end

data/lib/greenhat/thing/formatters/nginx.rb

@@ -6,11 +6,14 @@ module GreenHat
     # Formatters
     # ==========================================================================
     def format_nginx
-      self.result = raw.map do |row|
-        ip, _sym, remote_user, rest = row.split(' ', 4)
+      output = raw.map do |row|
+        next if row.chomp.empty?
+
+        conn_data, rest = row.split(' [', 2)
+        ip, _sym, remote_user = conn_data.split(' ', 3)
 
         time, rest = rest.split(']', 2)
-        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+        time = Time.strptime(time, '%d/%b/%Y:%H:%M:%S %z')
 
         verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
 
@@ -29,8 +32,13 @@ module GreenHat
           gzip_ratio: gzip_ratio,
           time: time
         }
+      rescue StandardError => e
+        LogBot.warn('NGINX Parse', e.message)
+        { msg: row }
       end
 
+      # Remove Empty Entries
+      self.result = output.compact
       :ok
     end
 

data/lib/greenhat/thing/formatters/table.rb

@@ -2,17 +2,17 @@
 module GreenHat
   # Log
   module Formatters
-    # Formatters for Dmesg
+    # Table Formatting
     def format_table
       # Headers to Readable Symbol
-      headers = raw.first.split(' ', 6).map(&:downcase).map do |x|
+      headers = raw_full.first.split(' ', 6).map(&:downcase).map do |x|
         x.gsub(/\s+/, '_').gsub(/[^0-9A-Za-z_]/, '')
       end.map(&:to_sym)
 
       final = []
 
       # Put fields into a Hash based on Location/Key
-      raw[1..].map(&:split).each do |row|
+      raw_full.map(&:split)[1..].each do |row|
         result = {}
         row.each_with_index do |detail, i|
           result[headers[i]] = detail

data/lib/greenhat/thing/formatters/time_space.rb

@@ -18,19 +18,3 @@ module GreenHat
     # ==========================================================================
   end
 end
-
-# def time_space
-#   IO.foreach(file) do |row|
-#     time, msg = row.split(' ', 2)
-#     result = {
-#       time: parse_datetime(time), msg: msg, host: thing.host,
-#       source: path_format
-#     }
-
-#     result[:time] ||= datetime
-#     post result
-#   rescue StandardError => e
-#     puts "Unable to Parse, #{name}:#{e.message}"
-#     post(time: datetime, msg: row, source: path_format, host: thing.host)
-#   end
-# end

data/lib/greenhat/thing/helpers.rb

@@ -25,17 +25,6 @@ module GreenHat
         tmp_path.split('/').last(2).join(divider)
       end
     end
-
-    # Check what kind of file we have
-    def type_check
-      if info?
-        :info
-      elsif SuperLog.type?(path)
-        :log
-      else
-        :raw
-      end
-    end
   end
 end
 
@@ -69,3 +58,15 @@ class FalseClass
     0
   end
 end
+
+# https://stackoverflow.com/a/449322/1678507
+class Time
+  # Time#round already exists with different meaning in Ruby 1.9
+  def round_off(seconds = 60)
+    Time.at((to_f / seconds).round * seconds).utc
+  end
+
+  def floor(seconds = 60)
+    Time.at((to_f / seconds).floor * seconds).utc
+  end
+end

data/lib/greenhat/thing/info_format.rb

@@ -64,15 +64,15 @@ module GreenHat
       conns, sockets = raw.split { |x| x.include? 'Active' }.reject(&:empty?)
 
       formatted_conns = conns[1..].map do |entry|
-        entry.split(' ', 7).map(&:strip).each_with_index.map do |field, idx|
+        entry.split(' ', 7).map(&:strip).each_with_index.to_h do |field, idx|
           [format_netstat_conn_headers[idx], field]
-        end.to_h
+        end
       end
 
       formatted_sockets = sockets[1..].map do |entry|
-        entry.split('  ').map(&:strip).reject(&:blank?).each_with_index.map do |field, idx|
+        entry.split('  ').map(&:strip).reject(&:blank?).each_with_index.to_h do |field, idx|
           [format_netstat_socket_headers[idx], field]
-        end.to_h
+        end
       end
 
       {

data/lib/greenhat/thing/kind.rb

@@ -73,6 +73,11 @@ module GreenHat
         end
       end
 
+      if Settings.assume_raw?
+        self.type = 'raw'
+        return true
+      end
+
       puts "Unable to determine file type for #{name.pastel(:yellow)}"
       puts "Use '#{'json'.pastel(:cyan)}' or '#{'raw'.pastel(:cyan)}' if there are no matches (see file_types.rb)"
 

data/lib/greenhat/thing/super_log.rb

@@ -1,102 +1 @@
-# TODO: Deprecate, was used specifically for Elk
-# rubocop:disable  /AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/
-module GreenHat
-  # Log Identifier
-  module SuperLog
-    def self.log?(kind)
-      [
-        'gitaly/current',
-        'gitaly/gitaly_ruby_json.log',
-        'gitlab-rails/api_json.log',
-        'gitlab-rails/application_json.log',
-        'gitlab-rails/audit_json.log',
-        'gitlab-rails/graphql_json.log',
-        'gitlab-rails/integrations_json.log',
-        'gitlab-rails/production_json.log',
-        'gitlab-rails/sidekiq_client.log',
-        'gitlab-shell/gitlab-shell.log',
-        'gitlab-workhorse/current',
-        'puma/puma_stdout.log',
-        'sidekiq/current',
-        'gitlab-rails/importer.log',
-        'gitlabsos.log',
-        'nginx/gitlab_access.log',
-        'patroni/current'
-      ].any? { |x| x.include? kind.to_s }
 
-      true
-    end
-
-    # ==========================================================================
-    # File Pattern Matching
-    # ==========================================================================
-    def self.api_json_format?(path)
-      ['rails_api_json_log'].any? { |x| path.include? x }
-    end
-
-    def self.unicorn_stderr_format?(path)
-      %w[
-        unicorn_stderr
-      ].any? { |x| path.include? x }
-    end
-
-    def self.reconfigure_format?(path)
-      %w[
-        reconfigure
-      ].any? { |x| path.include? x }
-    end
-
-    def self.shellwords?(path)
-      %w[
-        gitlab_pages_current
-        alertmanager_current
-        registry_current
-        prometheus_current
-
-      ].any? { |x| path.include? x }
-    end
-
-    def self.time_space?(path)
-      %w[
-        postgresql_current
-        redis_current
-        unicorn_current
-        gitlab_monitor_current
-        sidekiq_exporter_log
-      ].any? { |x| path.include? x }
-    end
-
-    def self.json_format?(path)
-      %w[
-        production_json_log
-        gitaly_current
-        geo_log
-        sidekiq_current
-        sidekiq_log
-        gitlab_shell_gitlab_shell_log
-        gitlab_rails_audit_json_log
-        gitlab_rails_application_json_log
-      ].any? { |x| path.include? x }
-    end
-
-    def self.dmesg_format?(path)
-      %w[
-        dmesg
-      ].any? { |x| path.include? x }
-    end
-    # ==========================================================================
-
-    # Identify Formatter
-    def self.type?(path)
-      return :api_json_format if api_json_format?(path)
-      return :unicorn_stderr_format if unicorn_stderr_format?(path)
-      return :reconfigure_format if reconfigure_format?(path)
-      return :shellwords if shellwords?(path)
-      return :time_space if time_space?(path)
-      return :json_format if json_format?(path)
-      return :dmesg_format if dmesg_format?(path)
-
-      nil
-    end
-  end
-end

data/lib/greenhat/thing.rb

@@ -24,8 +24,7 @@ class Thing < Teron
 
   # Preloader Flags
   field :parsed # Flag for Parsing
-  field :result # Processed Data
-  field :raw_result # Flag for reading raw data
+  field :result, dump: false # Processed Data
   field :result_fields # All available fields
 
   def friendly_name
@@ -57,10 +56,14 @@ class Thing < Teron
     result_fields
   end
 
+  # Readlines rather than putting whole thing into memory
   def raw
-    raw_read if raw_result.nil?
+    File.foreach(file)
+  end
 
-    raw_result
+  # Where full read is needed
+  def raw_full
+    File.read(file).split("\n")
   end
 
   # Filter / Archive Pattern Matching
@@ -73,32 +76,27 @@ class Thing < Teron
     data.blank?
   end
 
-  def raw_read
-    spin_start("Read  #{name.pastel(:blue)} #{size.pastel(:bright_black)}")
-    self.raw_result = File.read(file).split("\n")
-  rescue StandardError => e
-    LogBot.fatal('Raw Read', message: e.message, backtrace: e.backtrace.first)
-    self.raw_result = ''
-  ensure
-    spin_done
-  end
-
   def output(print_it = true)
     if print_it
-      puts raw.join("\n")
+      puts raw.map(&:to_s).map(&:chomp)
     else
-      raw
+      raw.map(&:to_s).map(&:chomp)
     end
   end
 
-  # Log Identifier Helper
-  def log?
-    GreenHat::SuperLog.log? kind
+  # Hashed values searching
+  def process?
+    kind != :raw && methods.include?(formatter)
   end
 
-  # Hashed values searching
-  def processed?
-    kind != :raw
+  # Things that can be queried (Query Helper)
+  def query?
+    data.instance_of?(Array)
+  end
+
+  # Helper for all things that can be hash/value filtered
+  def self.list
+    all.select.select(&:process?)
   end
 
   # Helper Formatter Method
@@ -107,9 +105,6 @@ class Thing < Teron
   end
 
   def process
-    # Read First if Needed (Spinner)
-    raw_read if raw_result.nil?
-
     if methods.include? formatter
       spin_start("Parse #{name.pastel(:blue)} #{kind.to_s.pastel(:bright_black)} ")
       begin
@@ -140,4 +135,15 @@ class Thing < Teron
 
     []
   end
+
+  def query_save(results, name)
+    self.archive = Archive.first
+    self.result = results
+    self.name = name
+    self.log = true
+    self.parsed = true
+    self.type = 'json'
+    self.result_fields = field_processing
+    save!
+  end
 end

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.3.4'.freeze
+  VERSION = '0.5.0'.freeze
 end

data/lib/greenhat/views/api.slim

@@ -0,0 +1,55 @@
+== slim :chart_template
+  .row
+    .chart-menu
+      ul.section.table-of-contents
+        li
+          a href="#top"  Top
+        li
+          a href="#path"  Path
+        li
+          a href="#time"  Time
+
+    h3.white-text.center API
+    
+    h4.col.s12.white-text.center id="top" Top
+    .col.s12.card.padding.z-depth-4
+      == bubble_chart(faststats_top('gitlab-rails/api_json.log', :project_stats),  xtitle:'Duration', ytitle:'% Count', label: "Total", title: 'by Project')
+      .col.s12.padding
+        .legend X: Duration
+        .legend.red Y: % of total calls
+        .legend.blue Size: Total Calls
+
+
+    .col.s12.card.padding.z-depth-4
+      == bubble_chart(faststats_top('gitlab-rails/api_json.log', :user_stats),  xtitle:'Duration', ytitle:'% Count', label: "Total", title: 'by User')
+      .col.s12.padding
+        .legend X: Duration
+        .legend.red Y: % of total calls
+        .legend.blue Size: Total Calls
+
+    .col.s12.card.padding.z-depth-4
+      == pie_chart(api_duration_pie, title: 'Duration Total')
+
+    h4.col.s12.white-text.center id="path" Path
+    .col.s12.card.padding.z-depth-4
+      == pie_chart(faststats_field('gitlab-rails/api_json.log', :count), title: 'Total by Path')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(faststats_field('gitlab-rails/api_json.log',:p95), title: 'p95', ytitle: 'ms')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(faststats_field('gitlab-rails/api_json.log',:median), title: 'Median', ytitle: 'ms')
+
+    h4.col.s12.white-text.center id="time" Over Time
+    
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_avg_path_duration_series, title: 'Path Duration', ytitle: 'seconds')
+
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_avg_duration_series, title: 'Avg Duration', ytitle: 'ms')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(api_duration_series_stacked, stacked: true, title: 'Duration Breakdown', ytitle: 'seconds')
+
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_duration_series_stacked, stacked: true, title: 'Duration Breakdown', ytitle: 'seconds')

data/lib/greenhat/views/chart.slim

@@ -0,0 +1,42 @@
+doctype html
+html
+  head
+    == slim :headers
+  body.grey.darken-4
+    == slim :css
+
+    nav.blue-grey.darken-4
+      .nav-wrapper
+        a.brand-logo.center href="/"  Charts Home
+        ul.left.white-text 
+          li
+            a href ='/'
+              .btn-flat.white-text
+                i.fa.fa-home.left.line-height
+                | Home
+    
+          li
+            a href ='/chart'
+              .btn-flat.white-text
+                i.fa.fa-chart-pie.left.line-height
+                | Charts
+          li
+            a href ='/chart/time'
+              .btn-flat.white-text
+                i.fa.fa-chart-line.left.line-height
+                | Time Query
+
+    .container.padding
+      .row        
+        h5.white-text.center Dashboards
+      .row
+        .col.s4.padding
+          a.blue.btn-large.chart-btn href="/charts/gitaly" Gitaly
+        .col.s4.padding
+          a.green.btn-large.chart-btn href="/charts/sidekiq" Sidekiq
+        .col.s4.padding
+          a.red.btn-large.chart-btn href="/charts/api" Api
+        .col.s4.padding
+          a.purple.btn-large.chart-btn href="/charts/production" Production
+        .col.s4.padding
+          a.teal.btn-large.chart-btn href="/charts/workhorse" Workhorse