greenhat 0.3.5 → 0.3.6

data/lib/greenhat/shell/shell_helper.rb

@@ -103,7 +103,8 @@ module GreenHat
                           StringColor.do(key, entry)
                         end
 
-      if flags[:truncate]
+      # Stats truncation handled separately
+      if flags[:truncate] && !flags[:stats]
         entry_truncate(formatted_entry, flags[:truncate])
       else
         formatted_entry
@@ -157,6 +158,9 @@ module GreenHat
       files, flags, args = Args.parse(Shellwords.split(search))
       flags[:combine] = true
 
+      # Default to everything
+      files = Thing.all.map(&:name) if files.empty?
+
       ShellHelper.filter_start(files, flags, args)
     end
 
@@ -177,11 +181,17 @@ module GreenHat
 
           # Include Total Count in Name
           results = ShellHelper.filter(log.data, flags, args)
+          duration = calculate_duration(results)
+
           title = [
             log.friendly_name,
             " #{results.count}".pastel(:bright_black)
+
           ]
 
+          # Append Duration
+          title.push(" #{duration.pastel(:cyan, :dim)}") unless duration.blank?
+
           # Save unless empty
           obj[title.join] = results unless results.count.zero?
 
@@ -190,6 +200,31 @@ module GreenHat
       end
     end
 
+    def self.calculate_duration(results)
+      only_with_time = results.select(&:time)
+
+      # If slice is used ignore
+      return nil if only_with_time.empty?
+
+      sorted = only_with_time.map(&:time).sort
+      humanize_time(sorted.first, sorted.last)
+    end
+
+    # Replace TimeDifference with https://stackoverflow.com/a/4136485/1678507
+    def self.humanize_time(time_start, time_end, increments = 2)
+      miliseconds = (time_end - time_start) * 1000
+
+      list = [[1000, :ms], [60, :s], [60, :m], [24, :h]].map do |count, name|
+        next unless miliseconds.positive?
+
+        miliseconds, n = miliseconds.divmod(count)
+
+        "#{n.to_i}#{name}" unless n.to_i.zero?
+      end
+
+      list.compact.reverse[0..increments - 1].join(' ')
+    end
+
     # Filter Logic
     # TODO: Simplify
     # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
@@ -265,7 +300,6 @@ module GreenHat
     end
 
     # Filter Start and End Times
-    # rubocop:disable Metrics/MethodLength
     # TODO: This is a bit icky, simplify/dry
     def self.filter_time(results, flags)
       if flags.key?(:start)
@@ -302,7 +336,6 @@ module GreenHat
 
       results
     end
-    # rubocop:enable Metrics/MethodLength
 
     def self.filter_except(results, except)
       # Avoid Empty Results
@@ -380,6 +413,9 @@ module GreenHat
       stats.map do |field|
         occurrences = filter_count_occurrences(results, field, flags)
 
+        # Use Truncate For Long Keys
+        occurrences.transform_keys! { |key| key.to_s[0..flags[:truncate]] } if flags[:truncate]
+
         # Total Occurences
         total = occurrences.values.sum
 
@@ -397,13 +433,6 @@ module GreenHat
         # Append Header / Total with field name
         output.unshift([field.to_s.pastel(:bright_black), total])
 
-        # Use Truncate For Long Keys
-        if flags[:truncate]
-          output.map! do |key, value|
-            [key.to_s[0..flags[:truncate]], value]
-          end
-        end
-
         # Format
         output.to_h
       end
@@ -443,48 +472,83 @@ module GreenHat
     end
 
     # Break out filter row logic into separate method
-
     def self.filter_row_key(row, arg, flags)
       # Ignore Other Logic if Field isn't even included / Full Text Searching
       return false unless row.key?(arg[:field]) || arg[:field] == :text
 
       # Sensitivity Check / Check for Match / Full Text Searching
-      included = if arg[:field] == :text
-                   filter_row_entry(row.to_s, arg, flags)
-                 else
-                   filter_row_entry(row[arg.field].to_s, arg, flags)
-                 end
+      search_data = arg[:field] == :text ? row : row[arg.field]
+      match = filter_row_entry(search_data.to_s, arg, flags)
 
       # Pivot of off include vs exclude
       if arg.bang
-        !included
+        !match
       else
-        included
+        match
       end
     end
 
     # Field Partial / Case / Exact search
     def self.filter_row_entry(entry, arg, flags)
       # Exact Matching / Unless doing full text search
-      return entry.to_s == arg.value.to_s if flags.key?(:exact) && arg.field != :text
+      return entry == arg.value.to_s if flags.key?(:exact) && arg.field != :text
 
-      if flags.key?(:case)
-        entry.include? arg.value.to_s
-      else
-        entry.downcase.include? arg.value.to_s.downcase
+      # Cast to String/Integer Helper
+      entry, value = filter_entry_cast(entry, arg, flags)
+
+      entry.send(arg.logic, value)
+    end
+
+    # Handle casting to strings or integers
+    def self.filter_entry_cast(entry, arg, flags)
+      # Cast to String
+      value = arg.value.to_s
+
+      case arg.logic
+      when :include?
+
+        # Exact Case argument
+        unless flags.key?(:case)
+          entry.downcase!
+          value.downcase!
+        end
+      when :>=, :<=
+        entry = entry.to_i if entry.numeric?
+        value = value.to_i if value&.numeric?
       end
+
+      [entry, value]
+    end
+
+    def self.filter_entry_logic(entry, arg)
+      entry.send(arg.logic, arg.value)
     end
 
     # Total Count Helper
-    def self.total_count(results)
-      results.each do |k, v|
+    def self.total_count(results, flags)
+      results.map do |k, v|
         puts k
-        puts "Total: #{v.count.to_s.pastel(:blue)}"
+        # puts "Total: #{v.count.to_s.pastel(:blue)}"
+        # puts "Time Covered: #{calculate_duration(v).pastel(:cyan)}"
+
+        output = {}
+        output[:total] = v.count
+        output[:duration] = calculate_duration(v)
+
+        # Sort / Get first and Last
+        list = v.select(&:time).map(&:time)
+        unless list.blank?
+          output[:start] = list.first
+          output[:end] = list.last
+        end
+
+        puts render_table(output, flags)
+
         puts
       end
     end
 
-    # Total Count Helper
+    # Table Printer Helper
     def self.fields_print(results)
       results.each do |k, v|
         puts k
@@ -548,7 +612,7 @@ module GreenHat
         return file if file.instance_of?(Thing)
 
         if flags.fuzzy_file_match
-          Thing.all.select { |x| x.name.include? file }
+          Thing.all.select { |x| x.name.include?(file) || x.type.include?(file) }
         else
           Thing.where name: file
         end
@@ -560,62 +624,6 @@ module GreenHat
       things
     end
 
-    # Main Entry Point for Searching
-    # def self.search_start(log_list, filter_type, args, opts)
-    def self.search_start(files, flags, args)
-      # Convert to Things
-      logs = ShellHelper.find_things(files, flags)
-
-      logs.each_with_object({}) do |log, obj|
-        # Ignore Empty Results / No Thing
-        next if log&.data.blank?
-
-        obj[log.friendly_name] = ShellHelper.search(log.data, flags, args)
-
-        obj
-      end
-    end
-
-    # Generic Search Helper / String/Regex
-    def self.search(data, flags = {}, args = {})
-      results = data.clone.flatten.compact
-      results.select! do |row|
-        args.send(flags.logic) do |arg|
-          search_row(row, arg, flags)
-        end
-      end
-
-      # Strip Results if Slice is defined
-      results.map! { |row| row.slice(*flags[:slice]) } if flags[:slice]
-
-      # Strip Results if Except is defined
-      results.map! { |row| row.except(*flags[:except]) } if flags[:except]
-
-      # Remove Blank from either slice or except
-      results.reject!(&:empty?)
-
-      results
-    end
-
-    # Break out filter row logic into separate method
-    def self.search_row(row, arg, flags)
-      # Sensitivity Check / Check for Match
-      included = filter_row_entry(row.to_s, arg, flags)
-
-      # Pivot of off include vs exclude
-      if arg.bang
-        !included
-      else
-        included
-      end
-    end
-
-    # TODO: Remove?
-    # Color Reader Helper
-    # def self.pastel
-    #   @pastel ||= Pastel.new
-    # end
-
     # Number Helper
     # https://gitlab.com/zedtux/human_size_to_number/-/blob/master/lib/human_size_to_number/helper.rb
     def self.human_size_to_number(string)
@@ -637,23 +645,6 @@ module GreenHat
       number.round
     end
 
-    # TODO: Needed?
-    def self.filter_and(data, params = {})
-      result = data.clone.flatten.compact
-      params.each do |k, v|
-        result.select! do |row|
-          if row.key? k.to_sym
-            row[k.to_sym].include? v
-          else
-            false
-          end
-        end
-        next
-      end
-
-      result
-    end
-
     # General Helper for `show`
     def self.common_opts
       puts 'Common Options'.pastel(:blue)

data/lib/greenhat/shell.rb

@@ -7,6 +7,13 @@ module GreenHat
       # rubocop:enable Lint/Debugger
     end
 
+    def self.web
+      # Load Required Files
+      require 'greenhat/web'
+
+      GreenHat::Web.toggle
+    end
+
     def self.df
       Disk.df
     end

data/lib/greenhat/thing/file_types.rb

@@ -357,6 +357,13 @@ module GreenHat
             /nginx-ingress.log/
           ]
         },
+        'nginx/access.log' => {
+          format: :nginx,
+          log: true,
+          pattern: [
+            %r{nginx/access.log}
+          ]
+        },
         'nginx/gitlab_pages_access.log' => {
           format: :nginx,
           log: true,
@@ -552,6 +559,20 @@ module GreenHat
             %r{unicorn/current}
           ]
         },
+        'unicorn_stats' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            /unicorn_stats/
+          ]
+        },
+        'unicorn/unicorn_stdout.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{unicorn/unicorn_stdout.log}
+          ]
+        },
         'running_swappiness' => {
           format: :raw,
           pattern: [
@@ -589,6 +610,7 @@ module GreenHat
           log: true,
           pattern: [
             %r{sidekiq/current},
+            /sidekiq_current/,
             %r{sidekiq/@.*}
           ]
         },
@@ -652,6 +674,12 @@ module GreenHat
             /uname/
           ]
         },
+        'gitlab_socket_stats' => {
+          format: :gitlab_socket_stats,
+          pattern: [
+            /gitlab_socket_stats/
+          ]
+        },
         'uptime' => {
           format: :raw,
           pattern: [

data/lib/greenhat/thing/formatters/json.rb

@@ -43,6 +43,10 @@ module GreenHat
     # Check for Common Fields
     def format_json_time(result)
       result.time = format_time_parse(result.time) if result.key? :time
+
+      # If timestamp exists but time doesn't, add time
+      result.time = format_time_parse(result.timestamp) if result.key?(:timestamp) && !result.key?(:time)
+
       result.created_at = format_time_parse(result.created_at) if result.key? :created_at
       result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
     rescue StandardError => e

data/lib/greenhat/thing/formatters/nginx.rb

@@ -7,10 +7,11 @@ module GreenHat
     # ==========================================================================
     def format_nginx
       self.result = raw.map do |row|
-        ip, _sym, remote_user, rest = row.split(' ', 4)
+        conn_data, rest = row.split(' [', 2)
+        ip, _sym, remote_user = conn_data.split(' ', 3)
 
         time, rest = rest.split(']', 2)
-        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+        time = Time.strptime(time, '%d/%b/%Y:%H:%M:%S %z')
 
         verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
 
@@ -29,6 +30,9 @@ module GreenHat
           gzip_ratio: gzip_ratio,
           time: time
         }
+      rescue StandardError => e
+        LogBot.warn('NGINX Parse', e.message)
+        { msg: row }
       end
 
       :ok

data/lib/greenhat/thing/formatters/time_space.rb

@@ -18,19 +18,3 @@ module GreenHat
     # ==========================================================================
   end
 end
-
-# def time_space
-#   IO.foreach(file) do |row|
-#     time, msg = row.split(' ', 2)
-#     result = {
-#       time: parse_datetime(time), msg: msg, host: thing.host,
-#       source: path_format
-#     }
-
-#     result[:time] ||= datetime
-#     post result
-#   rescue StandardError => e
-#     puts "Unable to Parse, #{name}:#{e.message}"
-#     post(time: datetime, msg: row, source: path_format, host: thing.host)
-#   end
-# end

data/lib/greenhat/thing/helpers.rb

@@ -69,3 +69,15 @@ class FalseClass
     0
   end
 end
+
+# https://stackoverflow.com/a/449322/1678507
+class Time
+  # Time#round already exists with different meaning in Ruby 1.9
+  def round_off(seconds = 60)
+    Time.at((to_f / seconds).round * seconds).utc
+  end
+
+  def floor(seconds = 60)
+    Time.at((to_f / seconds).floor * seconds).utc
+  end
+end

data/lib/greenhat/thing.rb

@@ -140,4 +140,14 @@ class Thing < Teron
 
     []
   end
+
+  def query_save(results, name)
+    self.archive = Archive.first
+    self.result = results
+    self.name = name
+    self.log = true
+    self.parsed = true
+    self.result_fields = field_processing
+    save!
+  end
 end

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.3.5'.freeze
+  VERSION = '0.3.6'.freeze
 end

data/lib/greenhat/views/api.slim

@@ -0,0 +1,55 @@
+== slim :chart_template
+  .row
+    .chart-menu
+      ul.section.table-of-contents
+        li
+          a href="#top"  Top
+        li
+          a href="#path"  Path
+        li
+          a href="#time"  Time
+
+    h3.white-text.center API
+    
+    h4.col.s12.white-text.center id="top" Top
+    .col.s12.card.padding.z-depth-4
+      == bubble_chart(faststats_top('gitlab-rails/api_json.log', :project_stats),  xtitle:'Duration', ytitle:'% Count', label: "Total", title: 'by Project')
+      .col.s12.padding
+        .legend X: Duration
+        .legend.red Y: % of total calls
+        .legend.blue Size: Total Calls
+
+
+    .col.s12.card.padding.z-depth-4
+      == bubble_chart(faststats_top('gitlab-rails/api_json.log', :user_stats),  xtitle:'Duration', ytitle:'% Count', label: "Total", title: 'by User')
+      .col.s12.padding
+        .legend X: Duration
+        .legend.red Y: % of total calls
+        .legend.blue Size: Total Calls
+
+    .col.s12.card.padding.z-depth-4
+      == pie_chart(api_duration_pie, title: 'Duration Total')
+
+    h4.col.s12.white-text.center id="path" Path
+    .col.s12.card.padding.z-depth-4
+      == pie_chart(faststats_field('gitlab-rails/api_json.log', :count), title: 'Total by Path')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(faststats_field('gitlab-rails/api_json.log',:p95), title: 'p95', ytitle: 'ms')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(faststats_field('gitlab-rails/api_json.log',:median), title: 'Median', ytitle: 'ms')
+
+    h4.col.s12.white-text.center id="time" Over Time
+    
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_avg_path_duration_series, title: 'Path Duration', ytitle: 'seconds')
+
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_avg_duration_series, title: 'Avg Duration', ytitle: 'ms')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(api_duration_series_stacked, stacked: true, title: 'Duration Breakdown', ytitle: 'seconds')
+
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_duration_series_stacked, stacked: true, title: 'Duration Breakdown', ytitle: 'seconds')

data/lib/greenhat/views/chart.slim

@@ -0,0 +1,42 @@
+doctype html
+html
+  head
+    == slim :headers
+  body.grey.darken-4
+    == slim :css
+
+    nav.blue-grey.darken-4
+      .nav-wrapper
+        a.brand-logo.center href="/"  Charts Home
+        ul.left.white-text 
+          li
+            a href ='/'
+              .btn-flat.white-text
+                i.fa.fa-home.left.line-height
+                | Home
+    
+          li
+            a href ='/chart'
+              .btn-flat.white-text
+                i.fa.fa-chart-pie.left.line-height
+                | Charts
+          li
+            a href ='/chart/time'
+              .btn-flat.white-text
+                i.fa.fa-chart-line.left.line-height
+                | Time Query
+
+    .container.padding
+      .row        
+        h5.white-text.center Dashboards
+      .row
+        .col.s4.padding
+          a.blue.btn-large.chart-btn href="/charts/gitaly" Gitaly
+        .col.s4.padding
+          a.green.btn-large.chart-btn href="/charts/sidekiq" Sidekiq
+        .col.s4.padding
+          a.red.btn-large.chart-btn href="/charts/api" Api
+        .col.s4.padding
+          a.purple.btn-large.chart-btn href="/charts/production" Production
+        .col.s4.padding
+          a.teal.btn-large.chart-btn href="/charts/workhorse" Workhorse

data/lib/greenhat/views/chart_template.slim

@@ -0,0 +1,31 @@
+doctype html
+html
+  head
+    == slim :headers
+
+  body.grey.darken-4
+    == slim :css
+
+    nav.blue-grey.darken-4
+      .nav-wrapper
+        a.brand-logo.center href="/"  Charts Home
+        ul.left.white-text 
+          li
+            a href ='/'
+              .btn-flat.white-text
+                i.fa.fa-home.left.line-height
+                | Home
+    
+          li
+            a href ='/chart'
+              .btn-flat.white-text
+                i.fa.fa-chart-pie.left.line-height
+                | Charts
+          li
+            a href ='/chart/time'
+              .btn-flat.white-text
+                i.fa.fa-chart-line.left.line-height
+                | Time Query
+
+    .container.padding
+      == yield