greenhat 0.3.2 → 0.3.6

data/lib/greenhat/thing/formatters/kube_webservice.rb

@@ -0,0 +1,51 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Gitlab Tail Formatter
+    # ==========================================================================
+    def format_kube_webservice
+      # Revert to raw for cats
+      self.kind = :raw
+
+      output = {}
+      current_log = nil
+
+      raw.each do |line|
+        next if line.blank?
+
+        if line.include? '*** '
+          current_log = /\*\*\* (.+?) \*\*\*/.match(line).captures.first
+        else
+          output[current_log] ||= [] unless current_log.nil?
+          output[current_log].push line unless current_log.nil?
+        end
+      end
+
+      # Remove Empty Entries
+      output.reject { |_k, v| v.empty? }
+
+      # Root Dir
+      root_dir = "#{$TMP}/#{name}"
+      Dir.mkdir(root_dir)
+
+      # Write Files / Create Things
+      output.each do |k, v|
+        file_name = k.gsub('/var/log/gitlab/', '')
+
+        dir = "#{root_dir}/gitlab-rails"
+        Dir.mkdir(dir) unless File.exist?(dir)
+
+        File.write("#{root_dir}/#{file_name}", v.join("\n"))
+
+        # Thing Setup
+        archive.things_create(file: "#{root_dir}/#{file_name}").setup
+      end
+
+      # Link
+      self.result = raw
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/nginx.rb

@@ -7,10 +7,11 @@ module GreenHat
     # ==========================================================================
     def format_nginx
       self.result = raw.map do |row|
-        ip, _sym, remote_user, rest = row.split(' ', 4)
+        conn_data, rest = row.split(' [', 2)
+        ip, _sym, remote_user = conn_data.split(' ', 3)
 
         time, rest = rest.split(']', 2)
-        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+        time = Time.strptime(time, '%d/%b/%Y:%H:%M:%S %z')
 
         verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
 
@@ -29,6 +30,9 @@ module GreenHat
           gzip_ratio: gzip_ratio,
           time: time
         }
+      rescue StandardError => e
+        LogBot.warn('NGINX Parse', e.message)
+        { msg: row }
       end
 
       :ok

data/lib/greenhat/thing/formatters/registry.rb

@@ -0,0 +1,47 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Registry Split
+    # ==========================================================================
+    # https://docs.docker.com/registry/configuration/#log
+    # Formatters text, json
+    # registry['log_formatter'] = "json"
+    # TODO: Logstash (Not working in 14.3)
+
+    def format_time_registry
+      self.result = raw.map do |row|
+        result = row[0] == '{' ? registry_json(row) : registry_shell_words(row)
+
+        # Timestamp Parsing
+        result.ts = Time.parse result.ts if result.key? 'ts'
+        result.time = Time.parse(result.time)
+
+        result
+      end
+    end
+
+    def registry_shell_words(row)
+      time, msg = row.split(' ', 2)
+
+      output = Shellwords.split(msg).each_with_object({}) do |x, h|
+        key, value = x.split('=')
+        next if value.nil?
+
+        h[key.to_sym] = value.numeric? ? value.to_f : value
+      end
+
+      output[:time] = time
+
+      output
+    end
+
+    def registry_json(msg)
+      Oj.load(msg)
+    rescue EncodingError
+      { message: msg }
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/time_space.rb

@@ -18,19 +18,3 @@ module GreenHat
     # ==========================================================================
   end
 end
-
-# def time_space
-#   IO.foreach(file) do |row|
-#     time, msg = row.split(' ', 2)
-#     result = {
-#       time: parse_datetime(time), msg: msg, host: thing.host,
-#       source: path_format
-#     }
-
-#     result[:time] ||= datetime
-#     post result
-#   rescue StandardError => e
-#     puts "Unable to Parse, #{name}:#{e.message}"
-#     post(time: datetime, msg: row, source: path_format, host: thing.host)
-#   end
-# end

data/lib/greenhat/thing/helpers.rb

@@ -69,3 +69,15 @@ class FalseClass
     0
   end
 end
+
+# https://stackoverflow.com/a/449322/1678507
+class Time
+  # Time#round already exists with different meaning in Ruby 1.9
+  def round_off(seconds = 60)
+    Time.at((to_f / seconds).round * seconds).utc
+  end
+
+  def floor(seconds = 60)
+    Time.at((to_f / seconds).floor * seconds).utc
+  end
+end

data/lib/greenhat/thing.rb

@@ -140,4 +140,14 @@ class Thing < Teron
 
     []
   end
+
+  def query_save(results, name)
+    self.archive = Archive.first
+    self.result = results
+    self.name = name
+    self.log = true
+    self.parsed = true
+    self.result_fields = field_processing
+    save!
+  end
 end

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.3.2'.freeze
+  VERSION = '0.3.6'.freeze
 end

data/lib/greenhat/views/api.slim

@@ -0,0 +1,55 @@
+== slim :chart_template
+  .row
+    .chart-menu
+      ul.section.table-of-contents
+        li
+          a href="#top"  Top
+        li
+          a href="#path"  Path
+        li
+          a href="#time"  Time
+
+    h3.white-text.center API
+    
+    h4.col.s12.white-text.center id="top" Top
+    .col.s12.card.padding.z-depth-4
+      == bubble_chart(faststats_top('gitlab-rails/api_json.log', :project_stats),  xtitle:'Duration', ytitle:'% Count', label: "Total", title: 'by Project')
+      .col.s12.padding
+        .legend X: Duration
+        .legend.red Y: % of total calls
+        .legend.blue Size: Total Calls
+
+
+    .col.s12.card.padding.z-depth-4
+      == bubble_chart(faststats_top('gitlab-rails/api_json.log', :user_stats),  xtitle:'Duration', ytitle:'% Count', label: "Total", title: 'by User')
+      .col.s12.padding
+        .legend X: Duration
+        .legend.red Y: % of total calls
+        .legend.blue Size: Total Calls
+
+    .col.s12.card.padding.z-depth-4
+      == pie_chart(api_duration_pie, title: 'Duration Total')
+
+    h4.col.s12.white-text.center id="path" Path
+    .col.s12.card.padding.z-depth-4
+      == pie_chart(faststats_field('gitlab-rails/api_json.log', :count), title: 'Total by Path')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(faststats_field('gitlab-rails/api_json.log',:p95), title: 'p95', ytitle: 'ms')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(faststats_field('gitlab-rails/api_json.log',:median), title: 'Median', ytitle: 'ms')
+
+    h4.col.s12.white-text.center id="time" Over Time
+    
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_avg_path_duration_series, title: 'Path Duration', ytitle: 'seconds')
+
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_avg_duration_series, title: 'Avg Duration', ytitle: 'ms')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(api_duration_series_stacked, stacked: true, title: 'Duration Breakdown', ytitle: 'seconds')
+
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_duration_series_stacked, stacked: true, title: 'Duration Breakdown', ytitle: 'seconds')

data/lib/greenhat/views/chart.slim

@@ -0,0 +1,42 @@
+doctype html
+html
+  head
+    == slim :headers
+  body.grey.darken-4
+    == slim :css
+
+    nav.blue-grey.darken-4
+      .nav-wrapper
+        a.brand-logo.center href="/"  Charts Home
+        ul.left.white-text 
+          li
+            a href ='/'
+              .btn-flat.white-text
+                i.fa.fa-home.left.line-height
+                | Home
+    
+          li
+            a href ='/chart'
+              .btn-flat.white-text
+                i.fa.fa-chart-pie.left.line-height
+                | Charts
+          li
+            a href ='/chart/time'
+              .btn-flat.white-text
+                i.fa.fa-chart-line.left.line-height
+                | Time Query
+
+    .container.padding
+      .row        
+        h5.white-text.center Dashboards
+      .row
+        .col.s4.padding
+          a.blue.btn-large.chart-btn href="/charts/gitaly" Gitaly
+        .col.s4.padding
+          a.green.btn-large.chart-btn href="/charts/sidekiq" Sidekiq
+        .col.s4.padding
+          a.red.btn-large.chart-btn href="/charts/api" Api
+        .col.s4.padding
+          a.purple.btn-large.chart-btn href="/charts/production" Production
+        .col.s4.padding
+          a.teal.btn-large.chart-btn href="/charts/workhorse" Workhorse

data/lib/greenhat/views/chart_template.slim

@@ -0,0 +1,31 @@
+doctype html
+html
+  head
+    == slim :headers
+
+  body.grey.darken-4
+    == slim :css
+
+    nav.blue-grey.darken-4
+      .nav-wrapper
+        a.brand-logo.center href="/"  Charts Home
+        ul.left.white-text 
+          li
+            a href ='/'
+              .btn-flat.white-text
+                i.fa.fa-home.left.line-height
+                | Home
+    
+          li
+            a href ='/chart'
+              .btn-flat.white-text
+                i.fa.fa-chart-pie.left.line-height
+                | Charts
+          li
+            a href ='/chart/time'
+              .btn-flat.white-text
+                i.fa.fa-chart-line.left.line-height
+                | Time Query
+
+    .container.padding
+      == yield