ezmetrics 2.0.3 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8285f47dd4bba3f689deb9547be625a3f1b1bd6c1a5834b3d7a51604999e2d9d
-  data.tar.gz: d344781510df94d4394edd207723cf82037a34850077d341f57c7b9b54b351f3
+  metadata.gz: 1c984dfd8093ab6d30506e366dd849c7885cc2085134df96770412fba9f11680
+  data.tar.gz: bc6d30ebcf66184deb53c0deb68c41bf74e2d3e88d9ac3a405b0bff8e421735a
 SHA512:
-  metadata.gz: 9e7d50a7d5a39290aa6dcd488d804caec87f8259e3bbde8a6e8bd9f817c16cf6919769cc95ca64366b076b53dac6abb48e33f9c9550d0123e7d681ba652b25f3
-  data.tar.gz: 39e1b548e6f89fda6a92f0c02425b7b5ef81d1cbf525494cfecef98854f23ecb0922ef16b309b0ac718cca8937302005bd1d04cd6a91d87cef7f45891a65619b
+  metadata.gz: 1763e0aaf730c99b89e070d2f36ab99bb38d247cc8e8d0575f5d9c0baa03c93d7fd7aef7d4846fc4c4d346186d2519d9185d1b99e9b5b2fa5ab3a337a9d6e2e7
+  data.tar.gz: 8835762f7d3c666424b9d811182ff009d42d6a86b33ee2e37bf5f91b366e3d39d474028685a5466e4526ce6e55df4a6c4e14bf4b9502a90516cd7b47614d5023

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+patreon: nyku

data/.gitignore

@@ -0,0 +1,3 @@
+dump.rdb
+*.gem
+.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+# Changelog
+
+
+## [3.0.0] - 2020-02-01
+
+### Added
+
+- Dashboard (mountable rails engine)
+
+### Changed
+
+- Internal structure.
+  In order to migrate from older version - replace `EZmetrics.new` with `Ezmetrics::Storage.new`

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+gemspec
+
+gem "redis", "~> 4.0"
+gem "hiredis"
+gem "oj"

data/Gemfile.lock

@@ -0,0 +1,41 @@
+PATH
+  remote: .
+  specs:
+    ezmetrics (3.0.0)
+      hiredis (~> 0.6.3)
+      oj (~> 3.10)
+      redis (~> 4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.3)
+    hiredis (0.6.3)
+    oj (3.10.0)
+    redis (4.1.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.0)
+      rspec-support (~> 3.9.0)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  ezmetrics!
+  hiredis
+  oj
+  redis (~> 4.0)
+  rspec (~> 3.5)
+
+BUNDLED WITH
+   1.17.3

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Nicolae Rotaru
+Copyright (c) 2019-2020 Nicolae Rotaru
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -22,68 +22,17 @@ gem 'ezmetrics'
 
 ## Usage
 
-### Getting started
-
-This tool captures and aggregates Rails application metrics such as
-
-- `duration`
-- `views`
-- `db`
-- `queries`
-- `status`
-
-and stores them for the timeframe you specified, 60 seconds by default.
-
-You can change the timeframe according to your needs and save the metrics by calling `log` method:
-
-```ruby
-# Store the metrics for 60 seconds (default behaviour)
-EZmetrics.new.log(
-  duration: 100.5,
-  views:    40.7,
-  db:       59.8,
-  queries:  4,
-  status:   200
-)
-```
-
-```ruby
-# Store the metrics for 10 minutes
-EZmetrics.new(10.minutes).log(
-  duration: 100.5,
-  views:    40.7,
-  db:       59.8,
-  queries:  4,
-  status:   200
-)
-```
-
----
-
-For displaying metrics you need to call `show` method:
-
-```ruby
-# Aggregate and show metrics for last 60 seconds (default behaviour)
-EZmetrics.new.show
-```
-
-```ruby
-# Aggregate and show metrics for last 10 minutes
-EZmetrics.new(10.minutes).show
-```
-
-You can combine these timeframes, for example - store for 10 minutes, display for 5 minutes.
 
 ### Capture metrics
 
-Just add an initializer to your application:
+Add an initializer to your application:
 
 ```ruby
 # config/initializers/ezmetrics.rb
 
 ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
   event = ActiveSupport::Notifications::Event.new(*args)
-  unless event.payload[:name] == "SCHEMA"
+  unless event.payload[:name] == "SCHEMA" || event.payload[:sql] =~ /\ABEGIN|COMMIT|ROLLBACK\z/
     Thread.current[:queries] ||= 0
     Thread.current[:queries] += 1
   end
@@ -91,22 +40,35 @@ end
 
 ActiveSupport::Notifications.subscribe("process_action.action_controller") do |*args|
   event = ActiveSupport::Notifications::Event.new(*args)
-  EZmetrics.new.log(
+  Ezmetrics::Storage.new(24.hours).log(
     duration: event.duration.to_f,
     views:    event.payload[:view_runtime].to_f,
     db:       event.payload[:db_runtime].to_f,
     status:   event.payload[:status].to_i || 500,
     queries:  Thread.current[:queries].to_i,
   )
+  Thread.current[:queries] = 0
 end
 ```
 
 ### Display metrics
 
+#### 1. Dashboard
+
+```ruby
+# add the following line to 'config/routes.rb'
+
+mount Dashboard::Ezmetrics, at: "/dashboard", as: "dashboard"
+```
+
+![Dashboard](https://user-images.githubusercontent.com/1847948/73551868-e26b2800-444f-11ea-83b9-fc81c8d05c07.png)
+
+#### 2. Directly
+
 As simple as:
 
 ```ruby
-EZmetrics.new.show
+Ezmetrics::Storage.new.show
 ```
 
 This will return a hash with the following structure:
@@ -146,7 +108,7 @@ This will return a hash with the following structure:
 If you prefer a single level object - you can change the default output structure by calling `.flatten` before `.show`
 
 ```ruby
-EZmetrics.new(1.hour).flatten.show(db: :avg, duration: [:avg, :max])
+Ezmetrics::Storage.new(1.hour).flatten.show(db: :avg, duration: [:avg, :max])
 ```
 
 ```ruby
@@ -162,7 +124,7 @@ EZmetrics.new(1.hour).flatten.show(db: :avg, duration: [:avg, :max])
 Same for [partitioned aggregation](#partitioning)
 
 ```ruby
-EZmetrics.new(1.hour).partition_by(:minute).flatten.show(db: :avg, duration: [:avg, :max])
+Ezmetrics::Storage.new(1.hour).partition_by(:minute).flatten.show(db: :avg, duration: [:avg, :max])
 ```
 
 ```ruby
@@ -189,7 +151,7 @@ The aggregation can be easily configured by specifying aggregation options as in
 **1. Single**
 
 ```ruby
-EZmetrics.new.show(duration: :max)
+Ezmetrics::Storage.new.show(duration: :max)
 ```
 
 ```ruby
@@ -205,7 +167,7 @@ EZmetrics.new.show(duration: :max)
 **2. Multiple**
 
 ```ruby
-EZmetrics.new.show(queries: [:max, :avg])
+Ezmetrics::Storage.new.show(queries: [:max, :avg])
 ```
 
 ```ruby
@@ -222,7 +184,7 @@ EZmetrics.new.show(queries: [:max, :avg])
 **3. Requests**
 
 ```ruby
-EZmetrics.new.show(requests: true)
+Ezmetrics::Storage.new.show(requests: true)
 ```
 
 ```ruby
@@ -244,7 +206,7 @@ EZmetrics.new.show(requests: true)
 **4. Combined**
 
 ```ruby
-EZmetrics.new.show(views: :avg, :db: [:avg, :max], requests: true)
+Ezmetrics::Storage.new.show(views: :avg, :db: [:avg, :max], requests: true)
 ```
 
 ```ruby
@@ -279,7 +241,7 @@ By default percentile aggregation is turned off because it requires to store eac
 To enable this feature - you need to set `store_each_value: true` when saving the metrics:
 
 ```ruby
-EZmetrics.new.log(
+Ezmetrics::Storage.new.log(
   duration:         100.5,
   views:            40.7,
   db:               59.8,
@@ -292,7 +254,7 @@ EZmetrics.new.log(
 The aggregation syntax has the following format `metrics_type: :percentile_{number}` where `number` is any integer in the 1..99 range.
 
 ```ruby
-EZmetrics.new.show(db: [:avg, :percentile_90, :percentile_95], duration: :percentile_99)
+Ezmetrics::Storage.new.show(db: [:avg, :percentile_90, :percentile_95], duration: :percentile_99)
 ```
 
 ```ruby
@@ -313,7 +275,7 @@ EZmetrics.new.show(db: [:avg, :percentile_90, :percentile_95], duration: :percen
 If you want to visualize percentile distribution (from 1% to 99%):
 
 ```ruby
-EZmetrics.new.show(duration: :percentile_distribution)
+Ezmetrics::Storage.new.show(duration: :percentile_distribution)
 ```
 
 ```ruby
@@ -340,7 +302,7 @@ To aggregate metrics, partitioned by a unit of time you need to call `.partition
 
 ```ruby
 # Aggregate metrics for last hour, partition by minute
-EZmetrics.new(1.hour).partition_by(:minute).show(duration: [:avg, :max], db: :avg)
+Ezmetrics::Storage.new(1.hour).partition_by(:minute).show(duration: [:avg, :max], db: :avg)
 ```
 
 This will return an array of objects with the following structure:
@@ -387,7 +349,7 @@ like in the example below:
 
 Available time units for partitioning: `second`, `minute`, `hour`, `day`. Default: `minute`.
 
-### Performance
+## Performance
 
 The aggregation speed relies on the performance of **Redis** (data storage) and **Oj** (json serialization/parsing).
 
@@ -395,16 +357,16 @@ You can check the **aggregation** time (in seconds) by running:
 
 ```ruby
 # 1. Simple
-EZmetrics::Benchmark.new.measure_aggregation
+Ezmetrics::Benchmark.new.measure_aggregation
 
 # 2. Partitioned
-EZmetrics::Benchmark.new.measure_aggregation(:minute)
+Ezmetrics::Benchmark.new.measure_aggregation(:minute)
 
 # 3. Percentile
-EZmetrics::Benchmark.new(true).measure_aggregation
+Ezmetrics::Benchmark.new(true).measure_aggregation
 
-# 4.  Percentile (partitioned)
-EZmetrics::Benchmark.new(true).measure_aggregation(:minute)
+# 4. Percentile (partitioned)
+Ezmetrics::Benchmark.new(true).measure_aggregation(:minute)
 ```
 
 | Interval | Simple aggregation | Partitioned | Percentile | Percentile (partitioned) |
@@ -417,6 +379,8 @@ EZmetrics::Benchmark.new(true).measure_aggregation(:minute)
 
 The benchmarks above were run on a _2017 Macbook Pro 2.9 GHz Intel Core i7 with 16 GB of RAM_
 
-### License
+## [Changelog](CHANGELOG.md)
+
+## License
 
 ezmetrics is released under the [MIT License](https://opensource.org/licenses/MIT).

data/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-tactile

data/ezmetrics.gemspec

@@ -0,0 +1,21 @@
+require_relative "lib/ezmetrics/version"
+
+Gem::Specification.new do |gem|
+  gem.name                  = "ezmetrics"
+  gem.version               = Ezmetrics::VERSION
+  gem.date                  = "2020-02-01"
+  gem.summary               = "Rails metrics aggregation tool."
+  gem.description           = "Simple, lightweight and fast metrics aggregation for Rails."
+  gem.authors               = ["Nicolae Rotaru"]
+  gem.email                 = "nyku.rn@gmail.com"
+  gem.homepage              = "https://github.com/nyku/ezmetrics"
+  gem.license               = "MIT"
+  gem.files                 = `git ls-files | grep -Ev '^(spec)'`.split("\n")
+  gem.require_paths         = ["lib"]
+  gem.required_ruby_version = ">= 2.4.0"
+  gem.add_dependency "redis", ["~> 4.0"]
+  gem.add_dependency "hiredis", ["~> 0.6.3"]
+  gem.add_dependency "oj", ["~> 3.10"]
+
+  gem.add_development_dependency "rspec", "~> 3.5"
+end

data/lib/ezmetrics.rb

@@ -2,294 +2,12 @@ require "redis"
 require "redis/connection/hiredis"
 require "oj"
 
-class EZmetrics
-  METRICS               = [:duration, :views, :db, :queries].freeze
-  AGGREGATION_FUNCTIONS = [:max, :avg].freeze
-  PARTITION_UNITS       = [:second, :minute, :hour, :day].freeze
+module Ezmetrics
+  require_relative "ezmetrics/version"
+  require_relative "ezmetrics/storage"
+  require_relative "ezmetrics/benchmark"
 
-  def initialize(interval_seconds=60)
-    @interval_seconds = interval_seconds.to_i
-    @redis            = Redis.new(driver: :hiredis)
-    @schema           = redis_schema
+  if defined?(Rails)
+    require_relative "ezmetrics/dashboard/lib/dashboard"
   end
-
-  def log(payload={duration: 0.0, views: 0.0, db: 0.0, queries: 0, status: 200, store_each_value: false})
-    @safe_payload = {
-      duration:         payload[:duration].to_f,
-      views:            payload[:views].to_f,
-      db:               payload[:db].to_f,
-      queries:          payload[:queries].to_i,
-      status:           payload[:status].to_i,
-      store_each_value: payload[:store_each_value].to_s == "true"
-    }
-
-    this_second          = Time.now.to_i
-    status_group         = "#{payload[:status].to_s[0]}xx"
-    @this_second_metrics = redis.get(this_second)
-
-    if this_second_metrics
-      @this_second_metrics = Oj.load(this_second_metrics)
-
-      METRICS.each do |metrics_type|
-        update_sum(metrics_type)
-        update_max(metrics_type)
-        store_value(metrics_type) if safe_payload[:store_each_value]
-      end
-
-      this_second_metrics[schema["all"]]        += 1
-      this_second_metrics[schema[status_group]] += 1
-    else
-      @this_second_metrics = {
-        "second"       => this_second,
-        "duration_sum" => safe_payload[:duration],
-        "duration_max" => safe_payload[:duration],
-        "views_sum"    => safe_payload[:views],
-        "views_max"    => safe_payload[:views],
-        "db_sum"       => safe_payload[:db],
-        "db_max"       => safe_payload[:db],
-        "queries_sum"  => safe_payload[:queries],
-        "queries_max"  => safe_payload[:queries],
-        "2xx"          => 0,
-        "3xx"          => 0,
-        "4xx"          => 0,
-        "5xx"          => 0,
-        "all"          => 1
-      }
-
-      if safe_payload[:store_each_value]
-        this_second_metrics.merge!(
-          "duration_values" => [safe_payload[:duration]],
-          "views_values"    => [safe_payload[:views]],
-          "db_values"       => [safe_payload[:db]],
-          "queries_values"  => [safe_payload[:queries]]
-        )
-      end
-
-      this_second_metrics[status_group] = 1
-
-      @this_second_metrics = this_second_metrics.values
-    end
-
-    redis.setex(this_second, interval_seconds, Oj.dump(this_second_metrics))
-    true
-  rescue => error
-    formatted_error(error)
-  end
-
-  def show(options=nil)
-    @options = options || default_options
-    partitioned_metrics ? aggregate_partitioned_data : aggregate_data
-  end
-
-  def flatten
-    @flat = true
-    self
-  end
-
-  def partition_by(time_unit=:minute)
-    time_unit = PARTITION_UNITS.include?(time_unit) ? time_unit : :minute
-    @partitioned_metrics = interval_metrics.group_by { |array| second_to_partition_unit(time_unit, array[schema["second"]]) }
-    self
-  end
-
-  private
-
-  attr_reader :redis, :interval_seconds, :interval_metrics, :requests, :flat, :schema,
-    :storage_key, :safe_payload, :this_second_metrics, :partitioned_metrics, :options
-
-  def aggregate_data
-    return {} unless interval_metrics.any?
-    @requests = interval_metrics.sum { |array| array[schema["all"]] }
-    build_result
-  rescue
-    {}
-  end
-
-  def aggregate_partitioned_data
-    partitioned_metrics.map do |partition, metrics|
-      @interval_metrics = metrics
-      @requests = interval_metrics.sum { |array| array[schema["all"]] }
-      METRICS.each { |metrics_type| instance_variable_set("@sorted_#{metrics_type}_values", nil) }
-      flat ? { timestamp: partition, **build_result } : { timestamp: partition, data: build_result }
-    end
-  rescue
-    self
-  end
-
-  def build_result
-    result = {}
-
-    if options[:requests]
-      append_requests_to_result(result, { all: requests, grouped: count_all_status_groups })
-    end
-
-    options.each do |metrics, aggregation_functions|
-      next unless METRICS.include?(metrics)
-      aggregation_functions = [aggregation_functions] unless aggregation_functions.is_a?(Array)
-      next unless aggregation_functions.any?
-
-      aggregation_functions.each do |aggregation_function|
-        aggregated_metrics = aggregate(metrics, aggregation_function)
-        append_metrics_to_result(result, metrics, aggregation_function, aggregated_metrics)
-      end
-    end
-    result
-  ensure
-    result
-  end
-
-  def append_requests_to_result(result, aggregated_requests)
-    return result[:requests] = aggregated_requests unless flat
-
-    result[:requests_all] = aggregated_requests[:all]
-    aggregated_requests[:grouped].each do |group, counter|
-      result[:"requests_#{group}"] = counter
-    end
-  end
-
-  def append_metrics_to_result(result, metrics, aggregation_function, aggregated_metrics)
-    return result[:"#{metrics}_#{aggregation_function}"] = aggregated_metrics if flat
-
-    result[metrics] ||= {}
-    result[metrics][aggregation_function] = aggregated_metrics
-  end
-
-  def second_to_partition_unit(time_unit, second)
-    return second if time_unit == :second
-    time = Time.at(second)
-    return (time - time.sec - time.min * 60 - time.hour * 3600).to_i if time_unit == :day
-    return (time - time.sec - time.min * 60).to_i                    if time_unit == :hour
-    (time - time.sec).to_i
-  end
-
-  def interval_metrics
-    @interval_metrics ||= begin
-      interval_start    = Time.now.to_i - interval_seconds
-      interval_keys     = (interval_start..Time.now.to_i).to_a
-      redis.mget(interval_keys).compact.map { |array| Oj.load(array) }
-    end
-  end
-
-  def aggregate(metrics, aggregation_function)
-    return avg("#{metrics}_sum") if aggregation_function == :avg
-    return max("#{metrics}_max") if aggregation_function == :max
-
-    if aggregation_function == :percentile_distribution
-      sorted_values = send("sorted_#{metrics}_values")
-      return (1..99).to_a.inject({}) do |result, number|
-        result[number] = percentile(sorted_values, number)
-        result
-      end
-    end
-
-    percentile = aggregation_function.match(/percentile_(?<value>\d+)/)
-
-    if percentile && percentile["value"].to_i.between?(1, 99)
-      sorted_values = send("sorted_#{metrics}_values")
-      percentile(sorted_values, percentile["value"].to_i)
-    end
-  end
-
-  METRICS.each do |metrics|
-    define_method "sorted_#{metrics}_values" do
-      instance_variable_get("@sorted_#{metrics}_values") || instance_variable_set(
-        "@sorted_#{metrics}_values", interval_metrics.map { |array| array[schema["#{metrics}_values"]] }.flatten.compact.sort
-      )
-    end
-  end
-
-  def redis_schema
-    [
-      "second",
-      "duration_sum",
-      "duration_max",
-      "views_sum",
-      "views_max",
-      "db_sum",
-      "db_max",
-      "queries_sum",
-      "queries_max",
-      "2xx",
-      "3xx",
-      "4xx",
-      "5xx",
-      "all",
-      "duration_values",
-      "views_values",
-      "db_values",
-      "queries_values"
-    ].each_with_index.inject({}){ |result, pair| result[pair[0]] = pair[1] ; result }
-  end
-
-  def update_sum(metrics)
-    this_second_metrics[schema["#{metrics}_sum"]] += safe_payload[metrics]
-  end
-
-  def store_value(metrics)
-    this_second_metrics[schema["#{metrics}_values"]] << safe_payload[metrics]
-  end
-
-  def update_max(metrics)
-    max_value = [safe_payload[metrics], this_second_metrics[schema["#{metrics}_max"]]].max
-    this_second_metrics[schema["#{metrics}_max"]] = max_value
-  end
-
-  def avg(metrics)
-    (interval_metrics.sum { |array| array[schema[metrics]] }.to_f / requests).round
-  end
-
-  def max(metrics)
-    interval_metrics.max { |array| array[schema[metrics]] }[schema[metrics]].round
-  end
-
-  def percentile(sorted_array, pcnt)
-    array_length = sorted_array.length
-
-    return "not enough data (requests: #{array_length}, required: #{pcnt})" if array_length < pcnt
-
-    rank  = (pcnt.to_f / 100) * (array_length + 1)
-    whole = rank.truncate
-
-    # if has fractional part
-    if whole != rank
-      s0 = sorted_array[whole - 1]
-      s1 = sorted_array[whole]
-
-      f = (rank - rank.truncate).abs
-
-      return ((f * (s1 - s0)) + s0)&.round
-    else
-      return (sorted_array[whole - 1])&.round
-    end
-  end
-
-  def count_all_status_groups
-    interval_metrics.inject({ "2xx" => 0, "3xx" => 0, "4xx" => 0, "5xx" => 0 }) do |result, array|
-      result["2xx"] += array[schema["2xx"]]
-      result["3xx"] += array[schema["3xx"]]
-      result["4xx"] += array[schema["4xx"]]
-      result["5xx"] += array[schema["5xx"]]
-      result
-    end
-  end
-
-  def default_options
-    {
-      duration: AGGREGATION_FUNCTIONS,
-      views:    AGGREGATION_FUNCTIONS,
-      db:       AGGREGATION_FUNCTIONS,
-      queries:  AGGREGATION_FUNCTIONS,
-      requests: true
-    }
-  end
-
-  def formatted_error(error)
-    {
-      error:     error.class.name,
-      message:   error.message,
-      backtrace: error.backtrace.reject { |line| line.match(/ruby|gems/) }
-    }
-  end
-end
-
-require "ezmetrics/benchmark"
+end