request_response_stats 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1d93ef2e0bb7c1e31551811923a3b6393721c46f
+  data.tar.gz: a411d96d3df46bbd8832332d23a8dc469371d65b
+SHA512:
+  metadata.gz: d407ec5d98944bdc216d91bbb34e1bdf8af27eb56ead08a11f1e45253bb59415bc1b08f3e943c8ca5fff04eead3671dfa9ea968c81ba3bc60aa191508648cde4
+  data.tar.gz: 6723271a4816804f5d4d660a5492b57f3f1bde43e53835f15ef33623adc79af9f9b0292002836591dbdadb85b2176545bdcbfd22f47d0eb946851bca56edb109

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+**/.DS_Store
+**/*.swp
+.byebug_history

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.15.4

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at munishapc@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in request_response_stats.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Munish Goyal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# RequestResponseStats
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/request_response_stats`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'request_response_stats'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install request_response_stats
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/request_response_stats. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the RequestResponseStats project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/request_response_stats/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "request_response_stats"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/req_res_stat_controller.rb

@@ -0,0 +1,65 @@
+# File: app/controllers/req_res_stat_controller.rb
+
+# TODO: This controller is yet to be tested
+class ReqResStatController < ApplicationController
+
+  # params[:start_time] format: "2009-06-24 12:39:54 +09:00"
+  # params[:end_time] format: "2009-06-24 12:39:54 +09:00"
+  # params[:stat_key] popular choices: "request_count", "error_count", "min_time", "max_time", "avg_time"
+  def get_stats
+    stat_key = params[:stat_key].to_sym
+    start_time = parse_date_time_zone params[:start_time]
+    end_time = parse_date_time_zone params[:end_time]
+    granularity_in_hours = params[:granularity_in_hours].to_i.hours if params[:granularity_in_hours].present?
+
+    min_values = ReqResStat.get_min(stat_key, start_time, end_time, granularity_in_hours)
+    max_values = ReqResStat.get_max(stat_key, start_time, end_time, granularity_in_hours)
+    avg_values = ReqResStat.get_avg(stat_key, start_time, end_time, granularity_in_hours)
+
+    return_value = {
+      start_time: start_time,
+      end_time: end_time,
+      granularity_in_hours: granularity_in_hours,
+      min_values: min_values,
+      max_values: max_values,
+      avg_values: avg_values,
+    }
+
+    render json: return_value
+
+    rescue Exception => ex
+      error_message = [ex.message, ex.backtrace.join("\n")].join("\n")
+      render json: {error_message: error_message}
+  end
+
+  def get_details
+    stat_key = params[:stat_key].to_sym
+    start_time = parse_date_time_zone params[:start_time]
+    end_time = parse_date_time_zone params[:end_time]
+    granularity_in_hours = params[:granularity_in_hours].to_i.hours if params[:granularity_in_hours].present?
+
+    details = ReqResStat.get_details(stat_key, start_time, end_time, nil, granularity_in_hours)
+
+    return_value = {
+      details: details
+    }
+
+    render json: return_value
+
+    rescue Exception => ex
+      error_message = [ex.message, ex.backtrace.join("\n")].join("\n")
+      render json: {error_message: error_message}
+  end
+
+  private
+
+  # time format: "2009-06-24 12:39:54 +09:00"
+  def parse_date_time_zone(date_time_zone)
+    date, time, zone = date_time_zone.split(" ")
+    date = date.split("-")
+    time = time.split(":")
+
+    Time.new(*date, *time, zone)
+  end
+end
+

data/lib/request_response_stats/controller_concern.rb

@@ -0,0 +1,35 @@
+# File: lib/request_response_stats/controller_concern.rb
+
+require_relative 'request_response'
+
+module RequestResponseStats
+  module ControllerConcern
+    def self.included(base)
+      include RequestResponseStats
+
+      base.class_eval do
+        around_action :log_request_response_stats
+        # before_action :log_request_response_stats_before
+        # after_action :log_request_response_stats_end
+
+        def log_request_response_stats
+          if defined?(RR_INBOUND_STATS) && RR_INBOUND_STATS
+            begin
+              rrs = RequestResponse.new(request, response, {redis_connection: $redis, gather_stats: true, mongoid_doc_model: ReqResStat})
+              rrs.capture_request_response_cycle_start_info
+              yield
+              rrs.capture_request_response_cycle_end_info
+            rescue Exception => ex
+              rrs.try(:capture_request_response_cycle_error_info)
+              raise ex
+            end
+          else
+            yield
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/request_response_stats/custom_client.rb

@@ -0,0 +1,76 @@
+# File: lib/request_response_stats/custom_client.rb
+
+require_relative 'request_response'
+require_relative 'dummy_request'
+require_relative 'dummy_response'
+
+module RequestResponseStats
+  module CustomClient
+    RENAME_NAMESPACE = "original_"
+    RENAMED_METHODS = [:get, :post, :patch, :put, :delete, :head, :client]
+    MAX_URL_LENGTH = 250  # Note that value of -1 will include whole of the url
+
+    # By default args is assumed to be an array with first element as uri
+    # If args is an hash contained within one-element array, then you can specify the key
+    # that is to be used to fetch the uri
+    def custom_uri_key
+      false
+    end
+
+    def method_missing(name, *args, &block)
+      if RENAMED_METHODS.include?(name)
+        resp = log_request_response_stats(name, args) { self.public_send("#{RENAME_NAMESPACE}#{name.to_s}".to_sym, *args, &block) }
+        return resp
+      else
+        super
+      end
+    end
+    # module_function :method_missing
+
+    def log_request_response_stats(name, args, &original_call)
+      if defined?(RR_OUTBOUND_STATS) && RR_OUTBOUND_STATS
+        begin
+          uri = args.is_a?(Array) ? args.first : args
+          uri = uri[custom_uri_key] if (uri.is_a?(Hash) && custom_uri_key && uri[custom_uri_key])
+          sanitized_uri = uri.to_s[0..(MAX_URL_LENGTH - 1)]
+          rrs = RequestResponse.new(
+            DummyRequest.new({method: name, path: sanitized_uri}),
+            DummyResponse.new,
+            {redis_connection: $redis, gather_stats: true, mongoid_doc_model: ReqResStat}
+          )
+          rrs.capture_request_response_cycle_start_info
+          resp = original_call.call
+          rrs.capture_request_response_cycle_end_info
+          return resp
+        rescue Exception => ex
+          # Rails.logger.info "Following exception is raised:"
+          # Rails.logger.info ex
+          rrs.try(:capture_request_response_cycle_error_info)
+          raise ex
+        end
+      else
+        resp = original_call.call
+      end
+    end
+    # module_function :log_request_response_stats
+
+    def self.included(base)
+      class << base
+        RENAMED_METHODS.each do |key|
+          if method_defined?(key)
+            alias_method("#{RENAME_NAMESPACE}#{key.to_s}".to_sym, key)
+            remove_method key
+          end
+        end
+      end
+      base.class_eval do
+        const_set("RENAME_NAMESPACE", RENAME_NAMESPACE)
+        const_set("RENAMED_METHODS", RENAMED_METHODS)
+
+        module_function :method_missing
+        module_function :log_request_response_stats
+        module_function :custom_uri_key
+      end
+    end
+  end
+end

data/lib/request_response_stats/dummy_request.rb

@@ -0,0 +1,6 @@
+# File: lib/request_response_stats/dummy_request.rb
+
+module RequestResponseStats
+  class DummyResponse
+  end
+end

data/lib/request_response_stats/dummy_response.rb

@@ -0,0 +1,15 @@
+# File: lib/request_response_stats/dummy_response.rb
+
+module RequestResponseStats
+  class DummyRequest
+    attr_accessor :path, :method, :parameters
+
+    def initialize(options)
+      @path = options[:path]
+      @method = options[:method]
+      @controller = options[:parameters].try(:[], "controller") || "external_controller"
+      @action = options[:parameters].try(:[], "action") || "external_action"
+      @parameters = {"controller" => @controller, "action" => @action}
+    end
+  end
+end

data/lib/request_response_stats/redis_record.rb

@@ -0,0 +1,226 @@
+# File: lib/request_response_stats/redis_record.rb
+
+require 'active_support/time'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'json'
+
+module RequestResponseStats
+  class RedisRecord
+    attr_accessor :key, :value
+
+    REDIS_RR_KEY_NAMESPACE = "api_req_res"
+
+    PERMITTED_KEYS = [
+      :key_name,
+      :server_name,
+      :api_name,
+      :api_verb,
+      :api_controller,
+      :api_action,
+      :request_count,
+      :min_time,
+      :max_time,
+      :avg_time,
+      :start_time,
+      :end_time,
+      :error_count,
+      :min_used_memory_MB,
+      :max_used_memory_MB,
+      :avg_used_memory_MB,
+      :min_swap_memory_MB,
+      :max_swap_memory_MB,
+      :avg_swap_memory_MB,
+      :avg_gc_stat_diff,
+      :min_gc_stat_diff,
+      :max_gc_stat_diff,
+    ].map(&:to_s)
+
+    AT_MAX_TIME = nil unless defined? AT_MAX_TIME
+    AT_ERROR_COUNT = nil unless defined? AT_ERROR_COUNT
+    AT_MAX_SWAP_MEMORY_MB = nil unless defined? AT_MAX_SWAP_MEMORY_MB
+    ALERT_THRESHOLD = {
+      max_time: AT_MAX_TIME || 30,
+      error_count: AT_ERROR_COUNT || 2,
+      max_swap_memory_MB: AT_MAX_SWAP_MEMORY_MB || 200,
+    }.stringify_keys
+
+    class << self
+      # it returns the redis connection
+      # this method must be redefined for `RedisRecord` to be useable
+      def redis
+        raise StandardError, "UNDEFINED #{__method__}"
+      end
+
+      # get value from redis
+      def get(key)
+        redis.get(key)
+      end
+
+      # set value to redis
+      def set(key, value, options={})
+        redis.set(key, value, options)
+      end
+
+      # delete value from redis
+      def del(key)
+        redis.del(key)
+      end
+
+      def all_keys(opts={})
+        support = opts[:support] || false
+        if support
+          regex = /^#{REDIS_RR_KEY_NAMESPACE}/
+        else
+          regex = /^#{REDIS_RR_KEY_NAMESPACE}_PUBLIC/
+        end
+
+        redis.keys.select{|k| k =~ regex}.sort.reverse
+      end
+
+      # return parsed value from redis
+      def parsed_get(key)
+        JSON.parse(redis.get(key) || "{}")
+      end
+
+      # it returns parsed result into the format required for Mongo dump
+      def formatted_parsed_get_for_mongo(key)
+        data = parsed_get(key)
+        data["start_time"] = date_time_str_to_obj(data["start_time"])
+        data["end_time"] = date_time_str_to_obj(data["end_time"])
+
+        data
+      end
+
+      def hashify_all_data(opts={})
+        support = opts[:support] || false
+        req_res_stat = ActiveSupport::HashWithIndifferentAccess.new
+        all_keys(support: support).each do |key|
+          req_res_stat[key] = ActiveSupport::HashWithIndifferentAccess.new(parsed_get key)
+        end
+
+        req_res_stat
+      end
+
+      def flush_all_keys
+        redis.del(*all_keys(support: true)) if all_keys.present?
+      end
+
+      def group_stats_by_time_duration
+        raise StandardError, "UNDEFINED #{__method__}"
+      end
+
+      def support_key(server_name, key_name="default")
+        [REDIS_RR_KEY_NAMESPACE, "SUPPORT", server_name.to_s, key_name].join("_")
+      end
+
+      def req_key(server_name, req_object_id)
+        support_key(server_name, ["REQ_OBJ", req_object_id].join("_"))
+      end
+
+      def req_res_key(server_name, api_name, api_http_verb)
+        ["#{REDIS_RR_KEY_NAMESPACE}_PUBLIC_#{server_name}_#{api_name}_#{api_http_verb}", get_time_slot_name].compact.join("_")
+      end
+
+      def get_slot_range_for_key(redis_key)
+        date_slot_string = redis_key.split("_")[-1]
+
+        get_slot_range_for_date_slot_string(date_slot_string)
+      end
+
+      # set jsonified value to redis and raise alerts
+      def jsonified_set(key, value, options={}, custom_options={strict_key_check: true})
+        value.select!{|k,v| PERMITTED_KEYS.include? k.to_s} if custom_options[:strict_key_check]
+
+        # set jsonified value to redis
+        redis.set(key, value.to_json, options)
+
+        # get alerts collection
+        alerts = ALERT_THRESHOLD.select{ |k, v| value[k] >= v if value[k] }.map{|k,v| {
+          redis_key: key,
+          alarm_key: k,
+          alarm_value: v,
+          actual_value: value[k]
+        }}
+        alerts_data = {data: alerts}.to_json
+        raise_alert(alerts_data) if alerts.present?
+
+        # return alerts
+        alerts_data
+      end
+
+      def freezed_keys
+        all_keys.map{|k| self.new(k)}.select{|k| k.is_key_freezed?}.map{|rr| rr.key}
+      end
+
+      # lets you fetch records for given conditions from redis
+      def query(params={})
+        # to implement
+      end
+
+      private
+
+      # Example: for a value of `1.hour` for `group_stats_by_time_duration`, the data for each endpoint is
+      # divided into `1 day / 1.hour = 24` time slots within a day
+      def get_time_slot_name(current_time = Time.now)
+        current_time_utc = current_time.utc
+        seconds_since_beginning_of_today = (current_time_utc - Time.utc(current_time_utc.year, current_time_utc.month, current_time_utc.day)).to_i
+        # seconds_in_a_day = 24 * 60 * 60
+        num_of_slots_in_a_day = 24.hours / group_stats_by_time_duration
+        seconds_in_a_time_slot = group_stats_by_time_duration.seconds.to_i
+        current_slot = (seconds_since_beginning_of_today / seconds_in_a_time_slot).to_i
+        slot_str_length = num_of_slots_in_a_day.to_s.size
+        current_slot_name = "%0#{slot_str_length}d" % current_slot
+        current_slot_full_name = [
+          current_time_utc.year,
+          "%02d" % current_time_utc.month,  # current_time_utc.month
+          "%02d" % current_time_utc.day,  # current_time_utc.day
+          current_slot_name
+        ].join("-")
+
+        current_slot_full_name
+      end
+
+      # returns time range of given date_slot_stringe
+      # which is [start_time_of_slot, end_time_of_slot]
+      def get_slot_range_for_date_slot_string(date_slot_string)
+        year_num, month_num, date_num, slot_num = date_slot_string.split("-").map(&:to_i)
+        time = Time.utc(year_num, month_num, date_num)
+
+        # num_of_slots_in_a_day = 24.hours / group_stats_by_time_duration
+        seconds_in_a_time_slot = group_stats_by_time_duration.seconds.to_i
+        seconds_already_passed = slot_num * seconds_in_a_time_slot
+        starting_time = time + seconds_already_passed
+        ending_time = starting_time + seconds_in_a_time_slot
+
+        [starting_time, ending_time]
+      end
+
+      # Input example: "2017-11-06 09:01:00 UTC"
+      def date_time_str_to_obj(date_time_str)
+        date, time, zone = date_time_str.split(" ")
+        date = date.split("-").map(&:to_i)
+        time = time.split(":").map(&:to_i)
+        zone = "+00:00" if zone == ["UTC"]
+        DateTime.new(*date, *time, zone)
+      end
+
+      def raise_alert(data)
+        name.split("::")[0].constantize.custom_alert_code(data)
+      end
+
+    end
+
+    def initialize(key, value=nil)
+      @key = key
+      @value = value
+    end
+
+    # if a key is freezed then no more data will be written to it
+    def is_key_freezed?
+      return nil unless self.class.group_stats_by_time_duration
+
+      self.class.get_slot_range_for_key(key)[1] < Time.now
+    end
+
+  end
+end