request_response_stats 0.1.1

data/lib/request_response_stats/req_res_stat.rb

@@ -0,0 +1,170 @@
+# File: app/models/req_res_stat.rb
+# File: lib/request_response_stats/req_res_stat.rb
+
+require 'mongoid'
+
+class RequestResponseStats::ReqResStat
+  include Mongoid::Document
+  # include Mongoid::Timestamps
+
+  store_in collection: "statsReqRes"
+
+  field :key_name, type: String
+  field :server_name, type: String
+  field :api_name, type: String
+  field :api_verb, type: String
+  field :api_controller, type: String
+  field :api_action, type: String
+  field :request_count, type: Integer
+  field :min_time, type: Float
+  field :max_time, type: Float
+  field :avg_time, type: Float
+  field :start_time, type: DateTime
+  field :end_time, type: DateTime
+  field :error_count, type: Integer
+  field :min_used_memory_MB, type: Integer
+  field :max_used_memory_MB, type: Integer
+  field :avg_used_memory_MB, type: Integer
+  field :min_swap_memory_MB, type: Integer
+  field :max_swap_memory_MB, type: Integer
+  field :avg_swap_memory_MB, type: Integer
+  field :avg_gc_stat_diff, type: Hash
+  field :min_gc_stat_diff, type: Hash
+  field :max_gc_stat_diff, type: Hash
+
+  DEFAULT_STATS_GRANULARITY = 1.hour
+  PERCISION = 2
+
+  def server_plus_api
+    [server_name, api_name, api_verb].join("_")
+  end
+
+  class << self
+    # Note:
+    # `start_time` and `end_time` are Time objects
+    # `start_time` in inclusive but `end_time` is not
+    def get_within(start_time, end_time)
+      where(:start_time.gte => start_time, :end_time.lt => end_time)
+    end
+
+    # wrapper around `get_stat` for :sum stat
+    def get_sum(key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
+      get_stat("sum", key, start_time, end_time, granularity)
+    end
+
+    # wrapper around `get_stat` for :min stat
+    def get_min(key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
+      get_stat("min", key, start_time, end_time, granularity)
+    end
+
+    # wrapper around `get_stat` for :max stat
+    def get_max(key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
+      get_stat("max", key, start_time, end_time, granularity)
+    end
+
+    # wrapper around `get_stat` for :avg stat
+    def get_avg(key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
+      data = get_stat("sum", key, start_time, end_time, granularity)
+      data.each do |e|
+        e[:stat_type] = "avg"
+        if e[:count] != 0
+          e[:data] = (e[:data] * 1.0 / e[:count]).try(:round, PERCISION)
+        else
+          e[:data] = 0
+        end
+
+      end
+      data
+    end
+
+    # set `stat_type` as `nil` to return grouped but uncompacted data
+    # otherwise, you can set `stat_type` as :sum, :max, :min, :avg to get grouped data
+    def get_details(key, start_time, end_time, stat_type = nil, granularity = DEFAULT_STATS_GRANULARITY)
+      # get ungrouped data
+      stat_type = stat_type.to_s.to_sym if stat_type
+      key = key.to_s.to_sym
+      relevant_records = get_within(start_time, end_time)
+      time_ranges = get_time_ranges(start_time, end_time, granularity)
+      stats = time_ranges.map do |time_range|
+        data_for_time_range = relevant_records.get_within(*time_range.values).map{ |r|
+          {server_plus_api: r.server_plus_api, data: r[key], key_name: r.key_name}
+        }
+        {data: data_for_time_range, **time_range}
+      end
+
+      # grouping data by :server_plus_api
+      stats.each do |r|
+        data = r[:data]
+        data = data.map{ |e| {server_plus_api: e[:server_plus_api], data: e[:data]} }
+        data = data.group_by { |e| e[:server_plus_api] }
+        r[:data] = data
+      end
+
+      # calculating grouped value based on stat_type
+      if stat_type
+        if [:sum, :min, :max].include? stat_type
+
+          # calculate grouped value
+          stats.each do |r|
+            data = r[:data]
+            data = data.map do |k, v|
+              # {server_plus_api: k, data: v.map{|e| e[:data]}}
+              element_data = v.map{|e| e[:data]}
+              {server_plus_api: k, count: element_data.size, data: element_data.compact.public_send(stat_type).try(:round, PERCISION)}
+            end
+            r[:data] = data
+          end
+
+          stats
+        elsif stat_type == :avg
+          data = get_details(key, start_time, end_time, stat_type = :sum, granularity)
+          data.each do |r|
+            r[:data].each do |e|
+              e[:data] = (e[:data] * 1.0 / e[:count]).try(:round, PERCISION)
+            end
+          end
+
+          data
+        else
+          "This :stat_type is not supported"
+        end
+      else
+        stats
+      end
+    end
+
+    private
+
+    def get_time_ranges(start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
+      slots = (((end_time - start_time) / granularity).ceil) rescue 0
+      current_start_time = start_time
+      time_ranges = (1..slots).map do |slot|
+        value = {start_time: current_start_time, end_time: current_start_time + granularity}
+        current_start_time += granularity
+
+        value
+      end
+      time_ranges[-1][:end_time] = end_time if time_ranges[-1] && (time_ranges[-1][:end_time] > end_time)
+
+      time_ranges
+    end
+
+    # stat: ["sum", "min", "max"]
+    # Note that [].sum is 0, whereas, [].min and [].max is nil
+    def get_stat(stat_type, key, start_time, end_time, granularity = DEFAULT_STATS_GRANULARITY)
+      stat_type = stat_type.to_s.to_sym
+      key = key.to_s.to_sym
+      relevant_records = get_within(start_time, end_time)
+      time_ranges = get_time_ranges(start_time, end_time, granularity)
+      stats = time_ranges.map do |time_range|
+        time_range_data = relevant_records.get_within(*time_range.values).pluck(key)
+        data = time_range_data.compact.public_send(stat_type).try(:round, PERCISION)
+        {key: key, stat_type: stat_type, data: data, count: time_range_data.size, **time_range}
+      end
+
+      stats
+    end
+
+  end
+
+end

data/lib/request_response_stats/request_response.rb

@@ -0,0 +1,259 @@
+# File: lib/request_response_stats/request_response.rb
+
+require_relative 'redis_record'
+require_relative 'req_res_stat'
+
+module RequestResponseStats
+  class RequestResponse
+    attr_accessor :request, :response
+    attr_accessor :redis_record
+    attr_accessor :redis, :mongoid_doc_model, :gather_stats
+
+    LONGEST_REQ_RES_CYCLE = 2.hours
+    SECONDS_PRECISION = 3
+    MEMORY_PRECISION = 0
+    SYS_CALL_FREQ = 60.seconds
+
+    # Set `GROUP_STATS_BY_TIME_DURATION` to `false` if no time based grouping is required, otherwise you can set it to value such as `1.minute` (but within a day)
+    GROUP_STATS_BY_TIME_DURATION = 1.minute
+
+    # Here:
+    # `redis_connection` is connection to redis db
+    # `mongoid_doc_model` is Mongoid::Document model which specifies document schema compatible to data structure in redis
+    # if `gather_stats` is `false`, they new data won't be added to the redis db
+    def initialize(req=nil, res=nil, opts={redis_connection: $redis, mongoid_doc_model: ReqResStat, gather_stats: true})
+      @request = req
+      @response = res
+      @redis = opts[:redis_connection]
+      @mongoid_doc_model = opts[:mongoid_doc_model]
+      @gather_stats = opts[:gather_stats]
+
+      @redis_record = RedisRecord
+
+      # adding behavior to dependents
+      temp_redis = @redis  # TODO: check why using @redis directly is not working. Do instance variable have specifal meaning inside defin_singleton_method block?
+      @redis_record.define_singleton_method(:redis) { temp_redis }
+      @redis_record.define_singleton_method(:group_stats_by_time_duration) { GROUP_STATS_BY_TIME_DURATION }
+    end
+
+    # captures request info that will be used at the end of request-response cycle
+    def capture_request_response_cycle_start_info
+      return gather_stats unless gather_stats
+
+      # get system info
+      current_time = get_system_current_time
+
+      # temporarily save request info
+      req_info = {
+        req_object_id: request.object_id,
+        res_object_id: response.object_id,
+        server_name: (request.env["SERVER_NAME"] rescue "some_server_name"),
+        req_path: (request.path rescue "some_path"),
+        req_http_verb: (request.method rescue "some_method"),
+        req_time: current_time,
+        req_url: (request.url rescue "some_url"),
+        req_format: (request.parameters["format"] rescue "some_format"),
+        req_controller: (request.parameters["controller"] rescue "some_controller"),
+        req_action: (request.parameters["action"] rescue "some_action"),
+        remote_ip: (request.remote_ip rescue "some_ip"),
+        gc_stat: get_gc_stat,
+      }
+      redis_req_key_name = redis_record.req_key(get_server_hostname, req_info[:req_object_id])
+      redis_record.jsonified_set(redis_req_key_name, req_info, {ex: LONGEST_REQ_RES_CYCLE}, {strict_key_check: false})
+
+      # return key_name
+      redis_req_key_name
+    end
+
+    # captures respose info and makes use of already captured request info
+    # to save info about current request-response cycle to redis
+    def capture_request_response_cycle_end_info(capture_error: false)
+      return gather_stats unless gather_stats
+
+      # get system info
+      current_time = get_system_current_time
+      current_used_memory = get_system_used_memory_mb
+      current_swap_memory = get_system_used_swap_memory_mb
+      current_hostname = get_server_hostname
+      current_gc_stat = get_gc_stat
+
+      res_info = {
+        req_object_id: request.object_id,
+        res_object_id: response.object_id,
+        res_time: current_time,
+      }
+
+      # fetching temporary request info
+      # return false if temporary request info cannot be found
+      redis_req_key_name = redis_record.req_key(get_server_hostname, res_info[:req_object_id])
+      req_info = ActiveSupport::HashWithIndifferentAccess.new(redis_record.parsed_get(redis_req_key_name))
+      return false if req_info == {}
+      redis_record.del redis_req_key_name
+
+      # generating request-response-cycle info
+      req_res_info = {
+        key_name: nil,
+        # server_name: req_info[:server_name],
+        server_name: current_hostname,
+        api_name: req_info[:req_path],
+        api_verb: req_info[:req_http_verb],
+        api_controller: req_info[:req_controller],
+        api_action: req_info[:req_action],
+        request_count: 0,
+        min_time: nil,
+        max_time: nil,
+        avg_time: 0,
+        start_time: nil,  # slot starting time
+        end_time: nil,  # slot ending time
+        error_count: 0,
+        min_used_memory_MB: nil,
+        max_used_memory_MB: nil,
+        avg_used_memory_MB: 0,
+        min_swap_memory_MB: nil,
+        max_swap_memory_MB: nil,
+        avg_swap_memory_MB: 0,
+        avg_gc_stat_diff: Hash.new(0),
+        min_gc_stat_diff: {},
+        max_gc_stat_diff: {},
+      }
+      redis_req_res_key_name = redis_record.req_res_key(req_res_info[:server_name], req_res_info[:api_name], req_res_info[:api_verb])
+      req_res_info[:key_name] = redis_req_res_key_name
+      req_res_info[:start_time], req_res_info[:end_time] = redis_record.get_slot_range_for_key(redis_req_res_key_name).map(&:to_s)
+      req_res_info_parsed = redis_record.parsed_get(redis_req_res_key_name)
+      req_res_info = if req_res_info_parsed.present?
+        # making use of existing value from db
+        ActiveSupport::HashWithIndifferentAccess.new(req_res_info_parsed)
+      else
+        # using default value
+        ActiveSupport::HashWithIndifferentAccess.new(req_res_info)
+      end
+      current_cycle_time = (res_info[:res_time] - req_info[:req_time]).round(SECONDS_PRECISION)
+      current_gc_stat_diff = get_gc_stat_diff(req_info[:gc_stat], current_gc_stat)
+      req_res_info[:min_time] = [req_res_info[:min_time], current_cycle_time].compact.min
+      req_res_info[:max_time] = [req_res_info[:max_time], current_cycle_time].compact.max
+      req_res_info[:avg_time] = ((req_res_info[:avg_time] * req_res_info[:request_count] + current_cycle_time)/(req_res_info[:request_count] + 1)).round(SECONDS_PRECISION)
+      req_res_info[:min_used_memory_MB] = [req_res_info[:min_used_memory_MB], current_used_memory].compact.min
+      req_res_info[:max_used_memory_MB] = [req_res_info[:max_used_memory_MB], current_used_memory].compact.max
+      req_res_info[:avg_used_memory_MB] = ((req_res_info[:avg_used_memory_MB] * req_res_info[:request_count] + current_used_memory)/(req_res_info[:request_count] + 1)).round(MEMORY_PRECISION)
+      req_res_info[:min_swap_memory_MB] = [req_res_info[:min_swap_memory_MB], current_swap_memory].compact.min
+      req_res_info[:max_swap_memory_MB] = [req_res_info[:max_swap_memory_MB], current_swap_memory].compact.max
+      req_res_info[:avg_swap_memory_MB] = (req_res_info[:avg_swap_memory_MB] * req_res_info[:request_count] + current_swap_memory)/(req_res_info[:request_count] + 1)
+      req_res_info[:min_gc_stat_diff] = get_min_max_sum_gc_stat_diff(:min, req_res_info[:min_gc_stat_diff], current_gc_stat_diff)
+      req_res_info[:max_gc_stat_diff] = get_min_max_sum_gc_stat_diff(:max, req_res_info[:min_gc_stat_diff], current_gc_stat_diff)
+      req_res_info[:avg_gc_stat_diff] = get_avg_gc_stat_diff(req_res_info[:request_count], req_res_info[:min_gc_stat_diff], current_gc_stat_diff)
+      req_res_info[:request_count] += 1  # Note: updation of `request_count` should be the last
+
+      # if error is raised
+      if capture_error
+        req_res_info[:error_count] += 1
+      end
+
+      # saving request-respose-cycle info to redis db
+      redis_record.jsonified_set(redis_req_res_key_name, req_res_info)
+
+      # return request-response-cycle info key
+      redis_req_res_key_name
+    end
+
+    # captures error info
+    def capture_request_response_cycle_error_info
+      capture_request_response_cycle_end_info(capture_error: true)
+    end
+
+    # moves data from redis to mongo
+    def move_data_from_redis_to_mongo
+      moved_keys = redis_record.freezed_keys.select do |redis_key|
+        value = redis_record.formatted_parsed_get_for_mongo(redis_key)
+        mongo_doc = mongoid_doc_model.create(value)
+        redis_record.del redis_key if mongo_doc
+        mongo_doc
+      end
+
+      moved_keys.size
+    end
+
+    private
+
+    def get_system_current_time
+      Time.now.to_f.round(SECONDS_PRECISION)
+    end
+
+    def get_system_memory_info_mb
+      key_name = redis_record.support_key(get_server_hostname, [get_server_hostname, "memory"].join("_"))
+      value = ActiveSupport::HashWithIndifferentAccess.new(redis_record.parsed_get key_name)
+      return_value = if value == {}
+        mem_info = (`free -ml`).split(" ") rescue []
+        used_memory = mem_info[8].strip.to_i rescue 0
+        used_swap_memory = mem_info[27].strip.to_i rescue 0
+        data = {used_memory: used_memory, used_swap_memory: used_swap_memory}
+        redis_record.set(key_name, data.to_json, {ex: SYS_CALL_FREQ})
+        data
+      else
+        value
+      end
+
+      return_value
+    end
+
+    def get_gc_stat_diff(old_gc_stat, new_gc_stat)
+      stat_diff = {}
+      gc_keys = new_gc_stat.keys.map{ |k| k.to_s.to_sym }
+      gc_keys.each do |key|
+        if old_gc_stat[key] && new_gc_stat[key]
+          stat_diff[key] = new_gc_stat[key] - old_gc_stat[key]
+        else
+          stat_diff[key] = 0
+        end
+      end
+
+      stat_diff
+    end
+
+    # stat_type can be :min, :max, :sum
+    def get_min_max_sum_gc_stat_diff(stat_type, old_gsd, new_gsd)
+      stat_type = stat_type.to_s.to_sym
+      stat = {}
+      stat_keys = new_gsd.keys.map{ |k| k.to_s.to_sym }
+      stat_keys.each do |key|
+        if [:min, :max, :sum].include?(stat_type)
+          stat[key] = [new_gsd[key], old_gsd[key]].compact.public_send(stat_type)
+        else
+          "Invalid :stat_type"
+        end
+      end
+
+      stat
+    end
+
+    def get_avg_gc_stat_diff(existing_request_count, old_gsd, new_gsd)
+      stat_type = stat_type.to_s
+      stat = {}
+      stat_keys = new_gsd.keys.map{ |k| k.to_s.to_sym }
+      stat_keys.each do |key|
+        stat[key] = (new_gsd[key] * existing_request_count +  old_gsd[key])/(existing_request_count + 1)
+      end
+
+      stat
+    end
+
+    def get_system_used_memory_mb
+      # (`free -ml | grep 'Mem:' | awk -F' ' '{ print $3 }'`.strip.to_i rescue 0).round(MEMORY_PRECISION)
+      get_system_memory_info_mb[:used_memory]
+    end
+
+    def get_system_used_swap_memory_mb
+      # (`free -ml | grep 'Swap:' | awk -F' ' '{ print $3 }'`.strip.to_i rescue 0).round(MEMORY_PRECISION)
+      get_system_memory_info_mb[:used_swap_memory]
+    end
+
+    def get_server_hostname
+      (`hostname`).strip
+    end
+
+    def get_gc_stat
+      GC.stat
+    end
+  end
+end
+
+

data/lib/request_response_stats/version.rb

@@ -0,0 +1,5 @@
+# File: lib/request_response_stats/version.rb
+
+module RequestResponseStats
+  VERSION = "0.1.1"
+end

data/lib/request_response_stats.rb

@@ -0,0 +1,62 @@
+# File: lib/request_response_stats.rb
+
+# library files
+require "request_response_stats/version"
+require_relative 'request_response_stats/request_response'
+require_relative 'request_response_stats/custom_client'
+require_relative 'request_response_stats/controller_concern'
+require_relative 'request_response_stats/req_res_stat'
+
+module RequestResponseStats
+  # override to set it to false if you want to capture inbound requests
+  RR_INBOUND_STATS = true unless defined? RR_INBOUND_STATS
+  
+  # override to set it to true if you want to capture inbound requests
+  RR_OUTBOUND_STATS = true unless defined? RR_OUTBOUND_STATS
+  
+  if self.method_defined? :custom_alert_code
+    # override to define the code that should be run on encountring alert conditions
+    def self.custom_alert_code(data)
+      raise StandardError, "Undefined custom alter code"
+    end
+  end
+end
+
+# TODO: The following files should not be required like this, instead they should be extracted into correct
+# place in Rails project using `rake` command
+# require_relative 'req_res_stat_controller'
+# require_relative 'request_response_stats_config'
+
+##### Examples: #####
+
+## Checking current redis data:
+=begin
+ # require 'request_response_stats'
+ include RequestResponseStats
+ rrs = RequestResponse.new(nil, nil)
+ ap rrs.redis_record.hashify_all_data
+ ap rrs.redis_record.hashify_all_data.size
+=end
+
+## Manually moving data from Redis to Mongo:
+# ap rrs.move_data_from_redis_to_mongo
+
+## Deleting data from Redis and Mongo:
+# rrs.redis_record.all_keys.each{|k| rrs.redis_record.del k}
+# ReqResStat.all.delete_all
+
+## Getting stats from Mongo:
+=begin
+ ap ReqResStat.all.size
+ ap ReqResStat.all.first
+ t = Time.now
+ ReqResStat.get_max(:max_time, t - 2.day, t, 6.hours).map{|r| r[:data]}
+ ReqResStat.get_avg(:avg_time, t - 2.day, t, 6.hours).map{|r| r[:data]}
+ ReqResStat.get_max(:min_time, t - 2.day, t, 6.hours).map{|r| r[:data]}
+ ap ReqResStat.get_details(:max_time, t - 2.day, t, nil, 6.hours)
+ ap ReqResStat.get_details(:max_time, t - 2.day, t, :max, 6.hours)
+ ap ReqResStat.get_details(:max_time, t - 2.day, t, :min, 6.hours)
+ ap ReqResStat.get_details(:max_time, t - 2.day, t, :sum, 6.hours)
+ ap ReqResStat.get_details(:max_time, t - 2.day, t, :avg, 6.hours)
+=end
+