greenhat 0.3.3 → 0.4.0

data/lib/greenhat/thing/file_types.rb

@@ -43,6 +43,13 @@ module GreenHat
             /dmesg/
           ]
         },
+        'lets-encrypt/renewal' => {
+          format: :bracket_log,
+          log: true,
+          pattern: [
+            %r{lets-encrypt/renewal}
+          ]
+        },
         'repmgrd/current' => {
           format: :bracket_log,
           log: true,
@@ -60,7 +67,8 @@ module GreenHat
         'etc/fstab' => {
           format: :clean_raw,
           pattern: [
-            %r{etc/fstab}
+            %r{etc/fstab},
+            /fstab/
           ]
         },
         'etc/os-release' => {
@@ -87,6 +95,12 @@ module GreenHat
             /getenforce/
           ]
         },
+        'geo-logcursor/current' => {
+          format: :raw,
+          pattern: [
+            %r{geo-logcursor/current}
+          ]
+        },
         'gitaly/current' => {
           format: :json_shell,
           log: true,
@@ -95,6 +109,13 @@ module GreenHat
             %r{gitaly/@.*}
           ]
         },
+        'gitaly/gitaly_hooks.log' => {
+          format: :shellwords,
+          log: true,
+          pattern: [
+            %r{gitaly/gitaly_hooks\.log}
+          ]
+        },
         'consul/current' => {
           format: :time_space,
           log: true,
@@ -225,7 +246,8 @@ module GreenHat
           format: :json,
           log: true,
           pattern: [
-            %r{gitlab-rails/audit_json.log}
+            %r{gitlab-rails/audit_json.log},
+            %r{webservice.log/audit_json.log}
           ]
         },
         'gitlab-rails/auth.log' => {
@@ -336,6 +358,20 @@ module GreenHat
             %r{nginx/current}
           ]
         },
+        'nginx-ingress.log' => {
+          format: :kube_nginx,
+          log: true,
+          pattern: [
+            /nginx-ingress.log/
+          ]
+        },
+        'nginx/access.log' => {
+          format: :nginx,
+          log: true,
+          pattern: [
+            %r{nginx/access.log}
+          ]
+        },
         'nginx/gitlab_pages_access.log' => {
           format: :nginx,
           log: true,
@@ -422,7 +458,7 @@ module GreenHat
           ]
         },
         'registry/current' => {
-          format: :time_shellwords,
+          format: :time_registry,
           log: true,
           pattern: [
             %r{registry/current},
@@ -444,16 +480,18 @@ module GreenHat
         },
         'puma/puma_stderr.log' => {
           format: :raw,
-          log: true,
+          log: false,
           pattern: [
-            %r{puma/puma_stderr.log}
+            %r{puma/puma_stderr.log},
+            %r{webservice.log/puma.stderr.log}
           ]
         },
         'puma/puma_stdout.log' => {
           format: :json,
           log: true,
           pattern: [
-            %r{puma/puma_stdout.log}
+            %r{puma/puma_stdout.log},
+            %r{webservice.log/puma.stdout.log}
           ]
         },
         'gitlab-pages/current' => {
@@ -529,6 +567,20 @@ module GreenHat
             %r{unicorn/current}
           ]
         },
+        'unicorn_stats' => {
+          format: :time_space,
+          log: true,
+          pattern: [
+            /unicorn_stats/
+          ]
+        },
+        'unicorn/unicorn_stdout.log' => {
+          format: :raw,
+          log: true,
+          pattern: [
+            %r{unicorn/unicorn_stdout.log}
+          ]
+        },
         'running_swappiness' => {
           format: :raw,
           pattern: [
@@ -566,6 +618,7 @@ module GreenHat
           log: true,
           pattern: [
             %r{sidekiq/current},
+            /sidekiq_current/,
             %r{sidekiq/@.*}
           ]
         },
@@ -629,6 +682,12 @@ module GreenHat
             /uname/
           ]
         },
+        'gitlab_socket_stats' => {
+          format: :gitlab_socket_stats,
+          pattern: [
+            /gitlab_socket_stats/
+          ]
+        },
         'uptime' => {
           format: :raw,
           pattern: [
@@ -745,13 +804,16 @@ module GreenHat
         'gitlab-rails/application.log' => {
           format: :raw,
           pattern: [
-            %r{gitlab-rails/application.log}
+            %r{gitlab-rails/application.log},
+            %r{webservice.log/application.log}
           ]
         },
         'gitlab-rails/production.log' => {
           format: :raw,
           pattern: [
-            %r{gitlab-rails/production.log}
+            /production.log/,
+            %r{gitlab-rails/production.log},
+            %r{webservice.log/production.log}
           ]
         },
         'gitlab/version-manifest.txt' => {
@@ -765,6 +827,72 @@ module GreenHat
           pattern: [
             %r{sidekiq/perf.data}
           ]
+        },
+
+        # ======================================================================
+        # KubeSoS TODO Section
+        # Things I am going to shortcut and set to raw for now
+        # ======================================================================
+        # Attempted Parsing
+        'kubesos_json' => {
+          log: true,
+          format: :kube_json,
+          pattern: [
+            /gitaly.log/
+          ]
+        },
+
+        'kube_webservice' => {
+          log: false,
+          format: :kube_webservice,
+          pattern: [
+            /^webservice\.log$/
+          ]
+        },
+
+        'kubesos' => {
+          format: :raw,
+          pattern: [
+            /all_values.yaml/,
+            %r{webservice.log/sidekiq_client.log},
+            /chart-version/,
+            /configmaps/,
+            /describe_deployments/,
+            /describe_ingress/,
+            /describe_nodes/,
+            /describe_pods/,
+            /describe_pv/,
+            /describe_pvc/,
+            /events/,
+            /get_deployments/,
+            /get_endpoints/,
+            /get_jobs/,
+            /get_pods/,
+            /get_pv/,
+            /get_pvc/,
+            /get_services/,
+            /gitaly.log/,
+            /gitlab-exporter.log/,
+            /gitlab-pages.log/,
+            /gitlab-shell.log/,
+            /grafana.log/,
+            /helm-version/,
+            /kubectl-check/,
+            /migrations.log/,
+            /minio.log/,
+            /nfs-client-provisioner.log/,
+            /operator.log/,
+            /postgresql.log/,
+            /prometheus.log/,
+            /redis.log/,
+            /registry.log/,
+            /secrets/,
+            /sidekiq.log/,
+            /task-runner.log/,
+            /top_nodes/,
+            /top_pods/,
+            /user_supplied_values.yaml/
+          ]
         }
       }
     end

data/lib/greenhat/thing/formatters/json.rb

@@ -43,6 +43,10 @@ module GreenHat
     # Check for Common Fields
     def format_json_time(result)
       result.time = format_time_parse(result.time) if result.key? :time
+
+      # If timestamp exists but time doesn't, add time
+      result.time = format_time_parse(result.timestamp) if result.key?(:timestamp) && !result.key?(:time)
+
       result.created_at = format_time_parse(result.created_at) if result.key? :created_at
       result.enqueued_at = format_time_parse(result.enqueued_at) if result.key? :enqueued_at
     rescue StandardError => e

data/lib/greenhat/thing/formatters/kube_json.rb

@@ -0,0 +1,36 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # K8s logs seem to have a lot of non-json entries
+    # Extract all json, ignore everything else
+    # ==========================================================================
+    def format_kube_json
+      self.result = raw.map do |row|
+        # Skip all non-json
+        next unless row.first == '{'
+
+        result = begin
+          Oj.load row
+        rescue EncodingError
+          puts
+          next
+        end
+
+        # Parsing Time
+        format_json_traverse result
+
+        result.sort.to_h
+      rescue StandardError => e
+        # TODO: Background Logger?
+        e.message
+        LogBot.warn('JSON Parse', e.message)
+        next
+      end.compact
+
+      :ok
+    end
+    # =========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/kube_nginx.rb

@@ -0,0 +1,48 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Formatters
+    # ==========================================================================
+    def format_kube_nginx
+      self.result = raw.map do |row|
+        ip, _sym, remote_user, rest = row.split(' ', 4)
+
+        time, rest = rest.split(']', 2)
+        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+
+        verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
+
+        method, path, http_version = verb.split
+
+        {
+          remote_addr: ip,
+          remote_user: remote_user,
+          method: method,
+          path: path,
+          status: status,
+          bytes: bytes,
+          http_version: http_version,
+          http_referer: http_referer,
+          http_user_agent: http_user_agent,
+          gzip_ratio: gzip_ratio,
+          time: time
+        }
+
+        # Fall back for malformed logs
+      rescue StandardError
+        { message: row }
+      end
+
+      :ok
+    end
+
+    # rubocop:disable Layout/LineLength
+    # NGINX Conf: /var/opt/gitlab/nginx/conf/nginx.conf
+    # log_format gitlab_access '$remote_addr - $remote_user [$time_local] "$request_method $filtered_request_uri $server_protocol" $status $body_bytes_sent "$filtered_http_referer" "$http_user_agent" $gzip_ratio';
+    # rubocop:enable Layout/LineLength
+
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/kube_webservice.rb

@@ -0,0 +1,51 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Gitlab Tail Formatter
+    # ==========================================================================
+    def format_kube_webservice
+      # Revert to raw for cats
+      self.kind = :raw
+
+      output = {}
+      current_log = nil
+
+      raw.each do |line|
+        next if line.blank?
+
+        if line.include? '*** '
+          current_log = /\*\*\* (.+?) \*\*\*/.match(line).captures.first
+        else
+          output[current_log] ||= [] unless current_log.nil?
+          output[current_log].push line unless current_log.nil?
+        end
+      end
+
+      # Remove Empty Entries
+      output.reject { |_k, v| v.empty? }
+
+      # Root Dir
+      root_dir = "#{$TMP}/#{name}"
+      Dir.mkdir(root_dir)
+
+      # Write Files / Create Things
+      output.each do |k, v|
+        file_name = k.gsub('/var/log/gitlab/', '')
+
+        dir = "#{root_dir}/gitlab-rails"
+        Dir.mkdir(dir) unless File.exist?(dir)
+
+        File.write("#{root_dir}/#{file_name}", v.join("\n"))
+
+        # Thing Setup
+        archive.things_create(file: "#{root_dir}/#{file_name}").setup
+      end
+
+      # Link
+      self.result = raw
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/nginx.rb

@@ -7,10 +7,11 @@ module GreenHat
     # ==========================================================================
     def format_nginx
       self.result = raw.map do |row|
-        ip, _sym, remote_user, rest = row.split(' ', 4)
+        conn_data, rest = row.split(' [', 2)
+        ip, _sym, remote_user = conn_data.split(' ', 3)
 
         time, rest = rest.split(']', 2)
-        time = Time.strptime(time, '[%d/%b/%Y:%H:%M:%S %z')
+        time = Time.strptime(time, '%d/%b/%Y:%H:%M:%S %z')
 
         verb, status, bytes, http_referer, http_user_agent, gzip_ratio = Shellwords.split(rest)
 
@@ -29,6 +30,9 @@ module GreenHat
           gzip_ratio: gzip_ratio,
           time: time
         }
+      rescue StandardError => e
+        LogBot.warn('NGINX Parse', e.message)
+        { msg: row }
       end
 
       :ok

data/lib/greenhat/thing/formatters/registry.rb

@@ -0,0 +1,47 @@
+# Top
+module GreenHat
+  # Log
+  module Formatters
+    # ==========================================================================
+    # Registry Split
+    # ==========================================================================
+    # https://docs.docker.com/registry/configuration/#log
+    # Formatters text, json
+    # registry['log_formatter'] = "json"
+    # TODO: Logstash (Not working in 14.3)
+
+    def format_time_registry
+      self.result = raw.map do |row|
+        result = row[0] == '{' ? registry_json(row) : registry_shell_words(row)
+
+        # Timestamp Parsing
+        result.ts = Time.parse result.ts if result.key? 'ts'
+        result.time = Time.parse(result.time)
+
+        result
+      end
+    end
+
+    def registry_shell_words(row)
+      time, msg = row.split(' ', 2)
+
+      output = Shellwords.split(msg).each_with_object({}) do |x, h|
+        key, value = x.split('=')
+        next if value.nil?
+
+        h[key.to_sym] = value.numeric? ? value.to_f : value
+      end
+
+      output[:time] = time
+
+      output
+    end
+
+    def registry_json(msg)
+      Oj.load(msg)
+    rescue EncodingError
+      { message: msg }
+    end
+    # ==========================================================================
+  end
+end

data/lib/greenhat/thing/formatters/time_space.rb

@@ -18,19 +18,3 @@ module GreenHat
     # ==========================================================================
   end
 end
-
-# def time_space
-#   IO.foreach(file) do |row|
-#     time, msg = row.split(' ', 2)
-#     result = {
-#       time: parse_datetime(time), msg: msg, host: thing.host,
-#       source: path_format
-#     }
-
-#     result[:time] ||= datetime
-#     post result
-#   rescue StandardError => e
-#     puts "Unable to Parse, #{name}:#{e.message}"
-#     post(time: datetime, msg: row, source: path_format, host: thing.host)
-#   end
-# end

data/lib/greenhat/thing/helpers.rb

@@ -69,3 +69,15 @@ class FalseClass
     0
   end
 end
+
+# https://stackoverflow.com/a/449322/1678507
+class Time
+  # Time#round already exists with different meaning in Ruby 1.9
+  def round_off(seconds = 60)
+    Time.at((to_f / seconds).round * seconds).utc
+  end
+
+  def floor(seconds = 60)
+    Time.at((to_f / seconds).floor * seconds).utc
+  end
+end

data/lib/greenhat/thing/kind.rb

@@ -73,6 +73,11 @@ module GreenHat
         end
       end
 
+      if Settings.assume_raw?
+        self.type = 'raw'
+        return true
+      end
+
       puts "Unable to determine file type for #{name.pastel(:yellow)}"
       puts "Use '#{'json'.pastel(:cyan)}' or '#{'raw'.pastel(:cyan)}' if there are no matches (see file_types.rb)"
 

data/lib/greenhat/thing.rb

@@ -140,4 +140,15 @@ class Thing < Teron
 
     []
   end
+
+  def query_save(results, name)
+    self.archive = Archive.first
+    self.result = results
+    self.name = name
+    self.log = true
+    self.parsed = true
+    self.type = 'json'
+    self.result_fields = field_processing
+    save!
+  end
 end

data/lib/greenhat/version.rb

@@ -1,3 +1,3 @@
 module GreenHat
-  VERSION = '0.3.3'.freeze
+  VERSION = '0.4.0'.freeze
 end

data/lib/greenhat/views/api.slim

@@ -0,0 +1,55 @@
+== slim :chart_template
+  .row
+    .chart-menu
+      ul.section.table-of-contents
+        li
+          a href="#top"  Top
+        li
+          a href="#path"  Path
+        li
+          a href="#time"  Time
+
+    h3.white-text.center API
+    
+    h4.col.s12.white-text.center id="top" Top
+    .col.s12.card.padding.z-depth-4
+      == bubble_chart(faststats_top('gitlab-rails/api_json.log', :project_stats),  xtitle:'Duration', ytitle:'% Count', label: "Total", title: 'by Project')
+      .col.s12.padding
+        .legend X: Duration
+        .legend.red Y: % of total calls
+        .legend.blue Size: Total Calls
+
+
+    .col.s12.card.padding.z-depth-4
+      == bubble_chart(faststats_top('gitlab-rails/api_json.log', :user_stats),  xtitle:'Duration', ytitle:'% Count', label: "Total", title: 'by User')
+      .col.s12.padding
+        .legend X: Duration
+        .legend.red Y: % of total calls
+        .legend.blue Size: Total Calls
+
+    .col.s12.card.padding.z-depth-4
+      == pie_chart(api_duration_pie, title: 'Duration Total')
+
+    h4.col.s12.white-text.center id="path" Path
+    .col.s12.card.padding.z-depth-4
+      == pie_chart(faststats_field('gitlab-rails/api_json.log', :count), title: 'Total by Path')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(faststats_field('gitlab-rails/api_json.log',:p95), title: 'p95', ytitle: 'ms')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(faststats_field('gitlab-rails/api_json.log',:median), title: 'Median', ytitle: 'ms')
+
+    h4.col.s12.white-text.center id="time" Over Time
+    
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_avg_path_duration_series, title: 'Path Duration', ytitle: 'seconds')
+
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_avg_duration_series, title: 'Avg Duration', ytitle: 'ms')
+
+    .col.s12.card.padding.z-depth-4
+      == column_chart(api_duration_series_stacked, stacked: true, title: 'Duration Breakdown', ytitle: 'seconds')
+
+    .col.s12.card.padding.z-depth-4
+      == area_chart(api_duration_series_stacked, stacked: true, title: 'Duration Breakdown', ytitle: 'seconds')

data/lib/greenhat/views/chart.slim

@@ -0,0 +1,42 @@
+doctype html
+html
+  head
+    == slim :headers
+  body.grey.darken-4
+    == slim :css
+
+    nav.blue-grey.darken-4
+      .nav-wrapper
+        a.brand-logo.center href="/"  Charts Home
+        ul.left.white-text 
+          li
+            a href ='/'
+              .btn-flat.white-text
+                i.fa.fa-home.left.line-height
+                | Home
+    
+          li
+            a href ='/chart'
+              .btn-flat.white-text
+                i.fa.fa-chart-pie.left.line-height
+                | Charts
+          li
+            a href ='/chart/time'
+              .btn-flat.white-text
+                i.fa.fa-chart-line.left.line-height
+                | Time Query
+
+    .container.padding
+      .row        
+        h5.white-text.center Dashboards
+      .row
+        .col.s4.padding
+          a.blue.btn-large.chart-btn href="/charts/gitaly" Gitaly
+        .col.s4.padding
+          a.green.btn-large.chart-btn href="/charts/sidekiq" Sidekiq
+        .col.s4.padding
+          a.red.btn-large.chart-btn href="/charts/api" Api
+        .col.s4.padding
+          a.purple.btn-large.chart-btn href="/charts/production" Production
+        .col.s4.padding
+          a.teal.btn-large.chart-btn href="/charts/workhorse" Workhorse

data/lib/greenhat/views/chart_template.slim

@@ -0,0 +1,31 @@
+doctype html
+html
+  head
+    == slim :headers
+
+  body.grey.darken-4
+    == slim :css
+
+    nav.blue-grey.darken-4
+      .nav-wrapper
+        a.brand-logo.center href="/"  Charts Home
+        ul.left.white-text 
+          li
+            a href ='/'
+              .btn-flat.white-text
+                i.fa.fa-home.left.line-height
+                | Home
+    
+          li
+            a href ='/chart'
+              .btn-flat.white-text
+                i.fa.fa-chart-pie.left.line-height
+                | Charts
+          li
+            a href ='/chart/time'
+              .btn-flat.white-text
+                i.fa.fa-chart-line.left.line-height
+                | Time Query
+
+    .container.padding
+      == yield