request_response_stats 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 53fa4042d8fbb53e6cce2300c4b4da6005f63a62
-  data.tar.gz: 1719a05129a6c9e664961f9318af5639a6305fd1
+  metadata.gz: 6a32b1fbcf4ef6f4b8c97147dc7fa0a8acf08aaa
+  data.tar.gz: d2c29cbd26516dd5200a897d8d0e6669cb289b96
 SHA512:
-  metadata.gz: 4b71eb4ad1cdb2a5714e906e21fd3cc94ac644cc1091d26dd738ae049c76943af819d828a069d65ef747f993cf7c1eb7bc18a4e7c1be34e0da8bf2bfb9bdc7dd
-  data.tar.gz: b6b5b5ff0dd49d1e8e4311b5d5910a03882a5c7a78fbc51849eaa77cbc3c7ef825bef753860478cbc71999bd1261da02873b13ffc022a4df54167f882b936296
+  metadata.gz: a0c8c978edb8ee2c48fbe8e1fe42d798488faea0ee17fc9623dbc278191994fe04eb951d6cb8323b69c6a2cbfa9d4a458d9de7b320242285bc0deed9ba675559
+  data.tar.gz: 06505e148f4c4d8b68217dcf2e8d892c71f66574030a51439e93fdd6adcfe53c5f8674ad27369a3bb1aeec44146d903c88c2b395cbe5fe4469ca8c1fe0537b97

data/.gitignore

@@ -13,4 +13,6 @@
 
 **/.DS_Store
 **/*.swp
+**/*.swo
 .byebug_history
+tags

data/.travis.yml

@@ -3,3 +3,9 @@ language: ruby
 rvm:
   - 2.4.1
 before_install: gem install bundler -v 1.15.4
+services:
+  - redis-server
+  - mongodb
+before_script:
+  - sleep 15
+  - mongo mydb_test --eval 'db.createUser({user:"travis",pwd:"test",roles:["readWrite"]});'

data/Gemfile

@@ -4,3 +4,5 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in request_response_stats.gemspec
 gemspec
+
+gem 'codecov', :require => false, :group => :test

data/README.md

@@ -20,6 +20,12 @@
 
 # RequestResponseStats
 
+[![Gem Version](https://badge.fury.io/rb/request_response_stats.svg)](https://badge.fury.io/rb/request_response_stats)
+[![Build Status](https://travis-ci.org/goyalmunish/request_response_stats.svg?branch=master)](https://travis-ci.org/goyalmunish/request_response_stats)
+[![codecov](https://codecov.io/gh/goyalmunish/request_response_stats/branch/master/graph/badge.svg)](https://codecov.io/gh/goyalmunish/request_response_stats)
+[![Maintainability](https://api.codeclimate.com/v1/badges/0c231c47679470213426/maintainability)](https://codeclimate.com/github/goyalmunish/request_response_stats/maintainability)
+[![Inline docs](http://inch-ci.org/github/goyalmunish/request_response_stats.svg?branch=master)](http://inch-ci.org/github/goyalmunish/request_response_stats)
+
 ## Prerequisites
 
 The gem uses [Redis](https://github.com/redis/redis-rb) as a temporary storage to store the captured stats data. For permanent storage of this data, [MongoDB](https://github.com/mongodb/mongoid) is being used.
@@ -67,9 +73,9 @@ Configure `config/initializers/request_response_stat_config.rb` as per your requ
 
 ### Documentation References
 
-Refer: http://www.rubydoc.info/gems/request_response_stats/
+Reference to Ruby docs: http://www.rubydoc.info/gems/request_response_stats/
 
-But, you can get better documentation by running tests :wink:.
+But, you can get better documentation by running tests :wink:. Refer: https://travis-ci.org/goyalmunish/request_response_stats.
 
 ### Checking current data in redis
 

data/lib/generators/templates/req_res_stat_controller.rb

@@ -1,35 +1,36 @@
 # File: app/controllers/req_res_stat_controller.rb
 
-# TODO: This controller is yet to be tested
+# TODO: This controller is not fully tested yet
 class ReqResStatController < ApplicationController
+  # Note: Refer https://www.chartkick.com/ and gems: chartkick for easily creating charts using Ruby
 
   # params[:start_time] format: "2009-06-24 12:39:54 +09:00"
   # params[:end_time] format: "2009-06-24 12:39:54 +09:00"
   # params[:stat_key] popular choices: "request_count", "error_count", "min_time", "max_time", "avg_time"
   def get_stats
-    stat_key = params[:stat_key].to_sym
-    start_time = parse_date_time_zone params[:start_time]
-    end_time = parse_date_time_zone params[:end_time]
-    granularity_in_hours = params[:granularity_in_hours].to_i.hours if params[:granularity_in_hours].present?
+    # query conditions
+    start_time = params[:start_time].present? ? parase_data_time_zone(params[:start_time]) : Time.now - 7.days
+    end_time = params[:end_time].present? ? parase_data_time_zone(params[:end_time]) : Time.now
+    granularity_in_hours = params[:granularity_in_hours].present? ? params[:granularity_in_hours].to_i.hours : 1.hour
 
-    min_values = ReqResStat.get_min(stat_key, start_time, end_time, granularity_in_hours)
-    max_values = ReqResStat.get_max(stat_key, start_time, end_time, granularity_in_hours)
-    avg_values = ReqResStat.get_avg(stat_key, start_time, end_time, granularity_in_hours)
+    # firing the base query only once
+    base_req_res_stats = ReqResStat.get_within(start_time, end_time)
 
-    return_value = {
-      start_time: start_time,
-      end_time: end_time,
-      granularity_in_hours: granularity_in_hours,
-      min_values: min_values,
-      max_values: max_values,
-      avg_values: avg_values,
-    }
-
-    render json: return_value
+    # gathering data
+    @request_count_stats = fetch_stats_for(:request_count, start_time, end_time, granularity_in_hours, base_req_res_stats)
+    @max_time_stats = fetch_stats_for(:max_time, start_time, end_time, granularity_in_hours, base_req_res_stats)
+    @avg_time_stats = fetch_stats_for(:avg_time, start_time, end_time, granularity_in_hours, base_req_res_stats)
+    @min_time_stats = fetch_stats_for(:min_time, start_time, end_time, granularity_in_hours, base_req_res_stats)
 
-    rescue Exception => ex
-      error_message = [ex.message, ex.backtrace.join("\n")].join("\n")
-      render json: {error_message: error_message}
+    render json: {
+      request_count_stats: @request_count_stats,
+      max_time_stats: @max_time_stats,
+      avg_time_stats: @avg_time_stats,
+      min_time_stats: @min_time_stats
+    }
+  rescue Exception => ex
+    error_message = [ex.message, ex.backtrace.join("\n")].join("\n")
+    render json: {error_message: error_message}
   end
 
   def get_details
@@ -45,14 +46,62 @@ class ReqResStatController < ApplicationController
     }
 
     render json: return_value
-
-    rescue Exception => ex
-      error_message = [ex.message, ex.backtrace.join("\n")].join("\n")
-      render json: {error_message: error_message}
+  rescue Exception => ex
+    error_message = [ex.message, ex.backtrace.join("\n")].join("\n")
+    render json: {error_message: error_message}
   end
 
   private
 
+  # if `base_records` are passed, then the base_records won't be created using `start_time` and `end_time`,
+  # so the values for `start_time` and `end_time` are ignored if `base_records` is present
+  def fetch_stats_for(stat_key, start_time, end_time, granularity_in_hours, base_records=nil)
+    if base_records
+      # use the passed basic dataset
+      base_req_res_stats = base_records
+    else
+      # create the basic dataset
+      base_req_res_stats = ReqResStat.get_within(start_time, end_time)
+    end
+
+    min_values = line_chart_data base_req_res_stats.get_min(stat_key, start_time, end_time, granularity_in_hours)
+    max_values = line_chart_data base_req_res_stats.get_max(stat_key, start_time, end_time, granularity_in_hours)
+    avg_values = line_chart_data base_req_res_stats.get_avg(stat_key, start_time, end_time, granularity_in_hours)
+
+    stats_data = [
+      {
+        name: "Min. Values",
+        data: min_values
+      },
+      {
+        name: "Max. Values",
+        data: max_values
+      },
+      {
+        name: "Avg. Values",
+        data: avg_values
+      }
+    ]
+
+    data = {
+      start_time: start_time,
+      end_time: end_time,
+      granularity_in_hours: granularity_in_hours,
+      stats_data: stats_data
+    }
+
+    data
+  end
+
+  def line_chart_data(values)
+    values_data = {}
+    values.each do |elem|
+      values_data[elem[:start_time]] = elem[:data] || 0
+    end
+
+    values_data
+  end
+
   # time format: "2009-06-24 12:39:54 +09:00"
   def parse_date_time_zone(date_time_zone)
     date, time, zone = date_time_zone.split(" ")

data/lib/request_response_stats/controller_concern.rb

@@ -17,7 +17,11 @@ module RequestResponseStats
             begin
               rrs = RequestResponse.new(request, response, {redis_connection: $redis, gather_stats: true, mongoid_doc_model: ReqResStat})
               rrs.capture_request_response_cycle_start_info
-              yield
+              if block_given?
+                yield
+              else
+                raise StandardError, "No block received. Investigate!"
+              end
               rrs.capture_request_response_cycle_end_info
             rescue Exception => ex
               rrs.try(:capture_request_response_cycle_error_info)

data/lib/request_response_stats/redis_record.rb

@@ -45,27 +45,33 @@ module RequestResponseStats
     }.stringify_keys
 
     class << self
-      # it returns the redis connection
+
+      # returns the redis connection
       # this method must be redefined for `RedisRecord` to be useable
       def redis
         raise StandardError, "UNDEFINED #{__method__}"
       end
 
       # get value from redis
+      # wrapper from redis' `get` method
       def get(key)
         redis.get(key)
       end
 
       # set value to redis
+      # wrapper from redis' `set` method
       def set(key, value, options={})
         redis.set(key, value, options)
       end
 
       # delete value from redis
+      # wrapper from redis' `del` method
       def del(key)
         redis.del(key)
       end
 
+      # returns all request_response_stats relevant redis keys
+      # by default only PUBLIC keys are returned
       def all_keys(opts={})
         support = opts[:support] || false
         if support
@@ -91,6 +97,7 @@ module RequestResponseStats
         data
       end
 
+      # returns collection of all relevant PUBLIC request_response_stats data from redis
       def hashify_all_data(opts={})
         support = opts[:support] || false
         req_res_stat = ActiveSupport::HashWithIndifferentAccess.new
@@ -101,10 +108,12 @@ module RequestResponseStats
         req_res_stat
       end
 
+      # flushes all request_response_stats data from redis
       def flush_all_keys
         redis.del(*all_keys(support: true)) if all_keys.present?
       end
 
+      # it has to be overridden
       def group_stats_by_time_duration
         raise StandardError, "UNDEFINED #{__method__}"
       end
@@ -148,6 +157,9 @@ module RequestResponseStats
         alerts_data
       end
 
+      # returns all PUBLIC request_response_stats related freezed keys from redis
+      # freezed key: redis key which will no longer be updated
+      # only freezed keys are eligible to be moved to mongo
       def freezed_keys
         all_keys.map{|k| self.new(k)}.select{|k| k.is_key_freezed?}.map{|rr| rr.key}
       end

data/lib/request_response_stats/req_res_stat.rb

@@ -5,10 +5,10 @@ require 'mongoid'
 
 class RequestResponseStats::ReqResStat
   include Mongoid::Document
-  # include Mongoid::Timestamps
 
   store_in collection: "statsReqRes"
 
+  # defining fields
   field :key_name, type: String
   field :server_name, type: String
   field :api_name, type: String
@@ -32,9 +32,21 @@ class RequestResponseStats::ReqResStat
   field :min_gc_stat_diff, type: Hash
   field :max_gc_stat_diff, type: Hash
 
+  # defining indexes
+  index({key_name: 1}, {unique: true, background: true})  # note that this is unique index
+  index({start_time: -1, end_time: -1}, {unique: false, background: true})
+  index({api_name: 1}, {unique: false, background: true})
+  index({server_name: 1}, {unique: false, background: true})
+  index({request_count: 1}, {unique: false, background: true})
+  index({error_count: 1}, {unique: false, background: true})
+  index({max_time: 1}, {unique: false, background: true})
+  index({avg_time: 1}, {unique: false, background: true})
+  index({min_time: 1}, {unique: false, background: true})
+
   DEFAULT_STATS_GRANULARITY = 1.hour
   PERCISION = 2
 
+  # returns a string identifying server_name, api_name, and api_verb
   def server_plus_api
     [server_name, api_name, api_verb].join("_")
   end
@@ -43,26 +55,36 @@ class RequestResponseStats::ReqResStat
     # Note:
     # `start_time` and `end_time` are Time objects
     # `start_time` in inclusive but `end_time` is not
+    # Use `get_within` with `nil` values for `start_time` and `end_time` to minimize database hits
+    # for same kind of queries on same date-range of data
     def get_within(start_time, end_time)
-      where(:start_time.gte => start_time, :end_time.lt => end_time)
+      if start_time || end_time
+        where(:start_time.gte => start_time, :end_time.lt => end_time)
+      else
+        all
+      end
     end
 
     # wrapper around `get_stat` for :sum stat
+    # for more info, check documentation for `get_stat`
     def get_sum(key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
       get_stat("sum", key, start_time, end_time, granularity)
     end
 
     # wrapper around `get_stat` for :min stat
+    # for more info, check documentation for `get_stat`
     def get_min(key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
       get_stat("min", key, start_time, end_time, granularity)
     end
 
     # wrapper around `get_stat` for :max stat
+    # for more info, check documentation for `get_stat`
     def get_max(key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
       get_stat("max", key, start_time, end_time, granularity)
     end
 
     # wrapper around `get_stat` for :avg stat
+    # for more info, check documentation for `get_stat`
     def get_avg(key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
       data = get_stat("sum", key, start_time, end_time, granularity)
       data.each do |e|
@@ -77,8 +99,13 @@ class RequestResponseStats::ReqResStat
       data
     end
 
+    # instead of aggregated values (such as in `get_stat`), it returns grouped values for given key, given granularity level,
+    # and given start_time and end_time
     # set `stat_type` as `nil` to return grouped but uncompacted data
     # otherwise, you can set `stat_type` as :sum, :max, :min, :avg to get grouped data
+    # TODO: Ignore `start_time` and `end_time` if a time-based collection is passed
+    # TODO: Optimize `get_time_ranges` to not to calculate time_ranges again and again for same `start_time` and `end_time` (that is,
+    # for same time-based collection
     def get_details(key, start_time, end_time, stat_type = nil, granularity = DEFAULT_STATS_GRANULARITY)
       # get ungrouped data
       stat_type = stat_type.to_s.to_sym if stat_type
@@ -149,6 +176,7 @@ class RequestResponseStats::ReqResStat
       time_ranges
     end
 
+    # it returns aggreated values for given key for given granularity in between given start_time and end_time
     # stat: ["sum", "min", "max"]
     # Note that [].sum is 0, whereas, [].min and [].max is nil
     def get_stat(stat_type, key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)

data/lib/request_response_stats/request_response.rb

@@ -37,6 +37,7 @@ module RequestResponseStats
     end
 
     # captures request info that will be used at the end of request-response cycle
+    # note that the captured infomation is saved only temporarily
     def capture_request_response_cycle_start_info
       return gather_stats unless gather_stats
 
@@ -156,28 +157,50 @@ module RequestResponseStats
     end
 
     # captures error info
+    # it is called if an exception is raised, and it in turns calls capture_request_response_cycle_end_info with capture_error: true
     def capture_request_response_cycle_error_info
       capture_request_response_cycle_end_info(capture_error: true)
     end
 
     # moves data from redis to mongo
-    def move_data_from_redis_to_mongo
-      moved_keys = redis_record.freezed_keys.select do |redis_key|
-        value = redis_record.formatted_parsed_get_for_mongo(redis_key)
-        mongo_doc = mongoid_doc_model.create(value)
-        redis_record.del redis_key if mongo_doc
-        mongo_doc
-      end
+    # only freezed and PUBLIC keys are moved
+    def move_data_from_redis_to_mongo(at_once=true)
+      if at_once
+        # multiple records will be inserted to mongodb at once
+        # this is to minimize the index creation time
+        values = []
+        redis_keys = []
+        redis_record.freezed_keys.each do |redis_key|
+          values << redis_record.formatted_parsed_get_for_mongo(redis_key)
+          redis_keys << redis_key
+        end
+        mongoid_doc_model.create(values)
+        redis_record.del(*redis_keys) if redis_keys.size > 0
+
+        redis_keys.size
+      else
+        # records will be inserted to mongo one at a time
+        # corresponding key from redis will be deleted only after successful creation of mongodb record
+        moved_keys = redis_record.freezed_keys.select do |redis_key|
+          value = redis_record.formatted_parsed_get_for_mongo(redis_key)
+          mongo_doc = mongoid_doc_model.create(value)
+          redis_record.del redis_key if mongo_doc
+          mongo_doc
+        end
 
-      moved_keys.size
+        moved_keys.size
+      end
     end
 
     private
 
+    # returns current time
     def get_system_current_time
       Time.now.to_f.round(SECONDS_PRECISION)
     end
 
+    # returns current system memory
+    # it uses `free` command to capture system memory info
     def get_system_memory_info_mb
       key_name = redis_record.support_key(get_server_hostname, [get_server_hostname, "memory"].join("_"))
       value = ActiveSupport::HashWithIndifferentAccess.new(redis_record.parsed_get key_name)
@@ -195,6 +218,7 @@ module RequestResponseStats
       return_value
     end
 
+    # returns the difference (new - old) in gc_stat
     def get_gc_stat_diff(old_gc_stat, new_gc_stat)
       stat_diff = {}
       gc_keys = new_gc_stat.keys.map{ |k| k.to_s.to_sym }
@@ -236,16 +260,22 @@ module RequestResponseStats
       stat
     end
 
+    # returns system used memory
+    # uses `get_system_memory_info` to get the info
     def get_system_used_memory_mb
       # (`free -ml | grep 'Mem:' | awk -F' ' '{ print $3 }'`.strip.to_i rescue 0).round(MEMORY_PRECISION)
       get_system_memory_info_mb[:used_memory]
     end
 
+    # returns used swap memory
+    # uses `get_system_memory_info` to get the info
     def get_system_used_swap_memory_mb
       # (`free -ml | grep 'Swap:' | awk -F' ' '{ print $3 }'`.strip.to_i rescue 0).round(MEMORY_PRECISION)
       get_system_memory_info_mb[:used_swap_memory]
     end
 
+    # returns system hostname
+    # uses linux `hostname` command to get the info
     def get_server_hostname
       (`hostname`).strip
     end

data/lib/request_response_stats/version.rb

@@ -1,5 +1,5 @@
 # File: lib/request_response_stats/version.rb
 
 module RequestResponseStats
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: request_response_stats
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Munish Goyal
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-09 00:00:00.000000000 Z
+date: 2017-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport