greenhat 0.3.3 → 0.4.0

data/lib/greenhat/web/gitaly.rb

@@ -0,0 +1,65 @@
+# General Web Helper
+module WebHelpers
+  # ======================================================
+  # == [ Gitaly ]
+  # ======================================================
+  def gitaly_avg_method_series
+    query = [
+      'gitaly/current',
+      '--slice=time,grpc.time_ms,grpc.method',
+      '--exists=grpc.time_ms',
+      '--exists=time',
+      '--grpc.time_ms>=1'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    method_key = :'grpc.method'
+    duration_key = :'grpc.time_ms'
+
+    # Collect Total by Duration
+    top = results.each_with_object({}) do |v, obj|
+      obj[v[method_key]] ||= []
+      obj[v[method_key]] << v[duration_key]
+
+      obj
+    end
+
+    # Average / Round
+    top.transform_values! { |x| (x.sum / x.size.to_f).round(1) }
+
+    # Only top by duration
+    method_calls = top.sort_by(&:last)[-10..].map(&:first)
+    results.select! { |x| method_calls.any? x[method_key] }
+
+    build_group_time_avg(results, method_key, duration_key) # 5 Minutes
+  end
+
+  def gitaly_errors_series
+    query = [
+      'gitaly/current',
+      '--slice=time,grpc.method,level',
+      '--exists=time,level,time',
+      '--level=error'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_group_time_total(results, :'grpc.method')
+  end
+
+  def gitaly_duration
+    query = [
+      'gitaly/current',
+      '--slice=time,grpc.time_ms',
+      '--exists=grpc.time_ms',
+      '--exists=time',
+      '--grpc.time_ms!=0',
+      '--exact'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_percentile_list(results, :'grpc.time_ms')
+  end
+end

data/lib/greenhat/web/helpers.rb

@@ -0,0 +1,198 @@
+# General Web Helper
+module WebHelpers
+  def arg_parse
+    @files, @flags, @args = GreenHat::Args.parse Shellwords.split(params[:query])
+  end
+
+  def percent(value, total)
+    ((value.to_i / total.to_f) * 100).round(2)
+  end
+
+  def build_time_index(results, interval = 5.minutes)
+    index = {}
+    start = results.min_by(&:time).time.floor(interval)
+    finish = results.max_by(&:time).time.floor(interval)
+
+    loop do
+      index[start] = 0
+      start += interval
+      break if start > finish
+    end
+
+    index
+  rescue StandardError => e
+    LogBot.fatal('Index Error', e.message)
+  ensure
+    {}
+  end
+
+  # Without Grouping Just total count
+  def build_time_total(results, interval = 5.minutes)
+    index = build_time_index(results, interval)
+
+    results.each do |r|
+      index[r.time.floor(interval)] += 1
+    end
+
+    index
+  end
+
+  # Without Grouping Avg
+  def build_time_avg(results, sum, interval = 5.minutes)
+    index = build_time_index(results, interval)
+
+    index.transform_values! { |_y| [] }
+
+    results.each do |r|
+      index[r.time.floor(interval)] << r[sum]
+    end
+
+    index.transform_values! do |l|
+      (l.sum / l.size.to_f)
+    end
+
+    index
+  end
+
+  # by Occurance/Count / Key to String
+  def build_group_time_total(results, group, interval = 5.minutes)
+    default_index = build_time_index(results, interval)
+    list = {}
+
+    results.map { |x| x[group] }.uniq.each do |key|
+      key = 'None' if key.nil?
+      list[key.to_s] = {
+        name: key.to_s,
+        data: default_index.clone
+      }
+    end
+
+    results.each do |r|
+      key = r[group].nil? ? 'None' : r[group]
+      list[key.to_s].data[r.time.floor(interval)] += 1
+    end
+
+    list.values
+  end
+
+  # by Sum
+  def build_group_time_sum(results, group, sum, interval = 5.minutes)
+    default_index = build_time_index(results, interval)
+    list = {}
+
+    results.map { |x| x[group] }.uniq.each do |key|
+      key = 'None' if key.nil?
+      list[key] = {
+        name: key,
+        data: default_index.clone.transform_values { |_y| [] }
+      }
+    end
+
+    results.each do |r|
+      key = r[group].nil? ? 'None' : r[group]
+      list[key].data[r.time.floor(interval)] << r[sum]
+    end
+
+    # Transform / Calculate
+    list.each do |_k, v|
+      v.data.transform_values! do |l|
+        l.empty? ? 0 : l.sum
+      end
+    end
+
+    list.values
+  end
+
+  # by Avg
+  def build_group_time_avg(results, group, sum, interval = 5.minutes)
+    default_index = build_time_index(results, interval)
+    list = {}
+
+    results.map { |x| x[group] }.uniq.each do |key|
+      key = 'None' if key.nil?
+      list[key] = {
+        name: key,
+        data: default_index.clone.transform_values { |_y| [] }
+      }
+    end
+
+    results.each do |r|
+      key = r[group].nil? ? 'None' : r[group]
+      list[key].data[r.time.floor(interval)] << r[sum]
+    end
+
+    # Transform / Calculate
+    list.each do |_k, v|
+      v.data.transform_values! do |l|
+        l.empty? ? 0 : (l.sum / l.size.to_f) # .round(1)
+      end
+    end
+
+    list.values
+  end
+
+  # Stack AVG Helper
+  def build_stack_time_avg(results, stacks, interval = 5.minutes)
+    default_index = build_time_index(results, interval)
+    list = {}
+
+    # MS are Hard
+    # results.each do |r|
+    #   stacks.each do |stack|
+    #     next unless r[stack]
+
+    #     r[stack] = r[stack] * 1000
+    #   end
+    # end
+
+    # Build List
+    stacks.each do |stack|
+      data = default_index.clone
+      data.transform_values! { |_y| [] } # Ensure Uniq
+      list[stack] = {
+        name: stack, data: data
+      }
+    end
+
+    # Collect Stack Data
+    results.each do |r|
+      stacks.each do |stack|
+        next unless r.key?(stack)
+
+        list[stack].data[r.time.floor(interval)] << r[stack]
+      end
+    end
+
+    # Transform / Calculate
+    list.each do |_k, v|
+      v.data.transform_values! do |l|
+        l.empty? ? 0 : (l.sum / l.size.to_f)
+      end
+    end
+
+    list.values
+  end
+
+  # Helper to get the bottom Ten
+  def top_method_calls(list)
+    sorted = list.sort_by(&:last)
+
+    all = if sorted.length < 10
+            sorted
+          else
+            sorted[-10..]
+          end
+
+    all.map(&:first)
+  end
+
+  def build_count_by_occurance(results, key, limit = 10)
+    output = results.each_with_object(Hash.new(0)) do |result, counts|
+      counts[result[key]] += 1
+    end
+
+    return output if limit.zero?
+
+    output.sort_by(&:last).reverse[0..(limit - 1)].to_h
+  end
+end

data/lib/greenhat/web/production.rb

@@ -0,0 +1,104 @@
+# General Web Helper
+module WebHelpers
+  # ======================================================
+  # == [ Production Log ]
+  # gitlab-rails/production_json.log
+  # ======================================================
+  # def prod_avg_path_duration_series
+  #   query = [
+  #     'gitlab-rails/production_json.log',
+  #     '--slice=time,path,duration_s'
+  #   ].join(' ')
+
+  #   results = GreenHat::ShellHelper.filter_internal(query)
+
+  #   method_key = :path
+  #   duration_key = :duration_s
+
+  #   # Collect Total by Duration
+  #   top = results.each_with_object({}) do |v, obj|
+  #     obj[v[method_key]] ||= []
+  #     obj[v[method_key]] << v[duration_key]
+
+  #     obj
+  #   end
+
+  #   # Average / Round
+  #   top.transform_values! { |x| (x.sum / x.size.to_f).round(1) }
+
+  #   # Only top by duration
+  #   method_calls = top_method_calls(top)
+  #   results.select! { |x| method_calls.any? x[method_key] }
+
+  #   build_group_time_avg(results, method_key, duration_key) # 5 Minutes
+  # end
+
+  def prod_status_series
+    query = [
+      'gitlab-rails/production_json.log',
+      '--slice=time,status',
+      '--exists=status'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_group_time_total(results, :status)
+  end
+
+  def prod_duration_series
+    query = [
+      'gitlab-rails/production_json.log',
+      '--slice=time,path,duration_s',
+      '--exists=duration_s'
+    ].join(' ')
+
+    # MS is hard
+    results = GreenHat::ShellHelper.filter_internal(query)
+    results.each do |r|
+      r[:duration_s] = r[:duration_s] * 1000
+    end
+
+    # build_time_avg(results, :duration_s).transform_values { |x| x.round(1) }
+    build_percentile_list(results, :duration_s)
+  end
+
+  def prod_duration_series_stacked
+    stacks = %i[
+      redis_duration_s
+      view_duration_s
+      gitaly_duration_s
+      redis_cache_duration_s
+      duration_s
+      db_duration_s
+      redis_shared_state_duration_s
+      queue_duration_s
+      db_duration_s
+    ]
+
+    query = [
+      'gitlab-rails/production_json.log',
+      "--slice=time,#{stacks.join(',')}"
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    output = build_stack_time_avg(results, stacks)
+
+    # MS are hard
+    output.each do |dataset|
+      dataset[:data].transform_values! { |x| (x * 1000).round(1) }
+    end
+
+    output
+  end
+
+  def prod_duration_pie
+    all = faststats_run('gitlab-rails/production_json.log', 'top')[:totals]
+
+    # Grab Specifics
+    totals = all.except(:count, :fails).transform_values(&:round)
+
+    # Organize
+    totals.sort_by(&:last).reverse.to_h
+  end
+end

data/lib/greenhat/web/sidekiq.rb

@@ -0,0 +1,73 @@
+# General Web Helper
+module WebHelpers
+  # ======================================================
+  # == [ Sidekiq ]
+  # ======================================================
+  def sidekiq_avg_duration_series
+    query = [
+      'sidekiq/current',
+      '--slice=time,class,duration_s',
+      '--duration_s>=1'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    method_key = :class
+    duration_key = :duration_s
+
+    # Collect Total by Duration
+    top = results.each_with_object({}) do |v, obj|
+      obj[v[method_key]] ||= []
+      obj[v[method_key]] << v[duration_key]
+
+      obj
+    end
+
+    # Average / Round
+    top.transform_values! { |x| (x.sum / x.size.to_f).round(1) }
+
+    # Only top by duration
+    method_calls = top_method_calls(top)
+    results.select! { |x| method_calls.any? x[method_key] }
+
+    build_group_time_avg(results, method_key, duration_key) # 5 Minutes
+  end
+
+  def sidekiq_duration
+    query = [
+      'sidekiq/current',
+      '--slice=time,duration_s',
+      '--exists=duration_s',
+      '--duration_s!=0',
+      '--exact'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_percentile_list(results, :duration_s)
+  end
+
+  def sidekiq_errors_series
+    query = [
+      'sidekiq/current',
+      '--slice=time,class',
+      '--exists=errors'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_group_time_total(results, :class)
+  end
+
+  def sidekiq_job_latency
+    query = [
+      'sidekiq/current',
+      '--slice=time,class,scheduling_latency_s',
+      '--exists=scheduling_latency_s'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_percentile_list(results, :scheduling_latency_s)
+  end
+end

data/lib/greenhat/web/stats_helpers.rb

@@ -0,0 +1,74 @@
+# General Web Helper
+module WebHelpers
+  # Time / Percentile without Grouping
+  def build_time_percentile(results, sum, percentile = 0.90, interval = 5.minutes)
+    index = build_time_index(results, interval)
+
+    index.transform_values! { |_y| [] }
+
+    results.each do |r|
+      index[r.time.floor(interval)] << r[sum]
+    end
+
+    index.transform_values! do |l|
+      l.percentile(percentile)
+    end
+
+    index
+  end
+
+  # Time / Min/Max without Grouping
+  def build_time_method(results, sum, method = :max, interval = 5.minutes)
+    index = build_time_index(results, interval)
+
+    index.transform_values! { |_y| [] }
+
+    results.each do |r|
+      index[r.time.floor(interval)] << r[sum]
+    end
+
+    index.transform_values! do |l|
+      next if l.empty?
+
+      l.send(method)
+    end
+
+    index
+  end
+
+  # Helper to make rounding Values Easier
+  def round_values(results, round_value = 1)
+    results.transform_values { |x| x.round(round_value) }
+  end
+
+  # Helper to grab the most common desired results
+  def build_percentile_list(results, key, round_value = 1)
+    p99 = build_time_percentile(results, key, 0.99)
+    p95 = build_time_percentile(results, key, 0.95)
+    max = build_time_method(results, key, :max)
+    mean = build_time_method(results, key, :mean)
+    [
+      { name: 'p99', data: round_values(p99, round_value) },
+      { name: 'p95', data: round_values(p95, round_value) },
+      { name: 'mean', data: round_values(mean, round_value) },
+      { name: 'max', data: round_values(max, round_value) }
+    ]
+  end
+end
+
+# May be worth adding other stats at some point
+# p99 = build_time_percentile(results, :duration_ms, 0.99)
+# p95 = build_time_percentile(results, :duration_ms, 0.95)
+# min = build_time_method(results, :duration_ms, :min)
+# max = build_time_method(results, :duration_ms, :max)
+# median = build_time_method(results, :duration_ms, :median)
+# mean = build_time_method(results, :duration_ms, :mean)
+
+# [
+#   { name: 'p99', data: round_values(p99) },
+#   { name: 'p95', data: round_values(p95) },
+#   { name: 'median', data: median },
+#   { name: 'mean', data: round_values(mean) },
+#   { name: 'min', data: min },
+#   { name: 'max', data: max }
+# ]

data/lib/greenhat/web/time.rb

@@ -0,0 +1,36 @@
+# Time Series Helpers
+module TimeHelpers
+  def time_series
+    results = GreenHat::ShellHelper.filter_internal(params[:query])
+
+    # Prevent exceptions on empty results
+    return [] if results.empty?
+
+    # Ensure Time
+    results.select! { |x| x.key?(:time) && !x[:time].nil? }
+
+    if !params[:avg].blank?
+      build_group_time_avg(results, group_sym, avg_sym, param_interval)
+    elsif !params[:group].blank?
+      build_group_time_total(results, group_sym, param_interval)
+    else
+      build_time_total(results, param_interval)
+    end
+  end
+
+  def param_interval
+    if params[:interval]
+      params[:interval].to_i.minutes
+    else
+      5.minutes
+    end
+  end
+
+  def group_sym
+    params[:group].strip.to_sym
+  end
+
+  def avg_sym
+    params[:avg].strip.to_sym
+  end
+end

data/lib/greenhat/web/workhorse.rb

@@ -0,0 +1,43 @@
+# General Web Helper
+module WebHelpers
+  # ======================================================
+  # == [ API ]
+  # ======================================================
+  def workhorse_request_total
+    query = [
+      'gitlab-workhorse/current',
+      '--slice=time,status',
+      '--exists=status'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_group_time_total(results, :status)
+  end
+
+  def workhorse_client_total
+    query = [
+      'gitlab-workhorse/current',
+      '--slice=user_agent',
+      '--exists=user_agent'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_count_by_occurance(results, :user_agent, 5)
+  end
+
+  def workhorse_duration
+    query = [
+      'gitlab-workhorse/current',
+      '--slice=time,duration_ms',
+      '--exists=duration_ms',
+      '--duration_ms!=0',
+      '--exact'
+    ].join(' ')
+
+    results = GreenHat::ShellHelper.filter_internal(query)
+
+    build_percentile_list(results, :duration_ms)
+  end
+end

data/lib/greenhat/web.rb

@@ -1,46 +1,90 @@
 require 'sinatra/base'
 
+# Chart Shims
+# https://github.com/ankane/chartkick/blob/master/lib/chartkick.rb
+require 'chartkick/enumerable'
+require 'chartkick/helper'
+require 'chartkick/version'
+require 'chartkick/sinatra'
+
+# Load Web Helpers
+require_all "#{File.dirname(__FILE__)}/web"
+
 # Helper
 module GreenHat
   # Web Helper
   module Web
     # Template Helper
     class MyApp < Sinatra::Base
-      set :server, :puma
-      set :server_settings, log_requests: false, debug: false, stdout_redirect: '/dev/null', quiet: true,
-                            Port: 15_123
-      set :logging, false
+      helpers Chartkick::Helper
+      helpers ::WebHelpers
+      helpers ::TimeHelpers
+
+      configure do
+        set :server, :puma
+        set :port, 4567
+        set :quiet, true
+        set :server_settings, Silent: true
+      end
 
       get '/' do
-        'Hello world!'
+        slim :index
       end
-    end
 
-    def self.start
-      @thread = Thread.new do
-        # original_stderr = $stderr.clone
-        # original_stdout = $stdout.clone
-        # $stderr.reopen(File.new('/dev/null', 'w'))
-        # $stdout.reopen(File.new('/dev/null', 'w'))
+      get '/chart' do
+        slim :chart
+      end
 
-        app = MyApp.run!
-        # $stdout.reopen(original_stdout)
-        # $stderr.reopen(original_stderr)
+      get '/charts/:chart_type' do
+        # TODO: Better validation?
+        slim params[:chart_type].to_sym
+      end
 
-        # $stdout.reopen($STDOUT_CLONE)
+      get '/chart/time' do
+        @index = time_series unless params[:query].empty?
+        slim :time
+      end
 
-        app
+      get '/chart/chartkick.js' do
+        content_type 'application/javascript'
+        File.read("#{File.dirname(__FILE__)}/views/chartkick.js")
       end
+    end
 
-      sleep 0.2
+    def self.setup
+      Archive.all.map(&:host_create).map(&:save!)
+    end
+
+    def self.toggle
+      if alive?
+        LogBot.info('Stopping Web')
+        stop
+      else
+        start
+      end
+    end
+
+    def self.start
+      setup if Host.count.zero?
+
+      return true if alive? # Prevent Dupes
+
+      LogBot.info('Starting Web')
+
+      @thread = Thread.new { MyApp.run! }
+      # MyApp.run! # Debugging
     end
 
     def self.thread
       @thread
     end
 
+    def self.alive?
+      !thread.nil? && thread.alive?
+    end
+
     def self.stop
-      thread.kill if thread.alive?
+      thread.kill if alive?
     end
   end
 end