ezmetrics 1.0.6 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: dc9e49c32e99f5d45f45ca6d3d467e53c09b8502
-  data.tar.gz: ce9f340d9f70ddfd7aebbac487e2d1b4b094527f
+SHA256:
+  metadata.gz: 408332cfc86d05b75d8b6004098d736ad60a624189f46022af822fdf6c24960d
+  data.tar.gz: 6d4ef89fa6a80e28cb443e16f1528b351a93bf7bf50e2a8fc756dc943a25f11f
 SHA512:
-  metadata.gz: 295ae4b987d80ec5feb6cbb21e0f5dee0ea6523d140e50840b5637b57c3263f712b5c798eed3c69794fb8618a5236e7f6005a1fee34f4e2f6e8812f97ab636e7
-  data.tar.gz: 8c9523d1385036e83a1eaea18e275c91f2865c391ad60f50a2af76b865843338e1d4408e8f08f037c2af14ba552181f7a82e85be7ea5b7242755f2c4070fb418
+  metadata.gz: 99a4ac48560305d2967cdcc8b248978ee3f0748ab1fb7da2830a3804d732d29664f25bfdc4bd2bfa50c449994913ecd455362f57af1f39f0f45e904e36876c12
+  data.tar.gz: c53617fbc324765b9b388b1280b6d363a3707b002a2718cd0467a16ff24708a8adbf81a812aca111a643924de4834a155efdef252911195570837fdb8f6a9dce

data/README.md

@@ -2,8 +2,7 @@
 
 [![Gem Version](https://badge.fury.io/rb/ezmetrics.svg)](https://badge.fury.io/rb/ezmetrics)
 
-A simple tool for capturing and displaying Rails metrics.
-
+Simple, lightweight and fast metrics aggregation for Rails.
 
 ## Installation
 
@@ -16,6 +15,7 @@ gem 'ezmetrics'
 ### Getting started
 
 This tool captures and aggregates Rails application metrics such as
+
 - `duration`
 - `views`
 - `db`
@@ -27,32 +27,42 @@ and stores them for the timeframe you specified, 60 seconds by default.
 You can change the timeframe according to your needs and save the metrics by calling `log` method:
 
 ```ruby
-  # Store the metrics for 60 seconds (default behaviour)
-  EZmetrics.new.log(duration: 100.5, views: 40.7, db: 59.8, queries: 4, status: 200)
+# Store the metrics for 60 seconds (default behaviour)
+EZmetrics.new.log(
+  duration: 100.5,
+  views:    40.7,
+  db:       59.8,
+  queries:  4,
+  status:   200
+)
 ```
-or
 
 ```ruby
-  # Store the metrics for 10 minutes
-  EZmetrics.new(10.minutes).log(duration: 100.5, views: 40.7, db: 59.8, queries: 4, status: 200)
+# Store the metrics for 10 minutes
+EZmetrics.new(10.minutes).log(
+  duration: 100.5,
+  views:    40.7,
+  db:       59.8,
+  queries:  4,
+  status:   200
+)
 ```
 
-For displaying metrics you need call `show` method:
+---
+
+For displaying metrics you need to call `show` method:
 
 ```ruby
-  # Aggregate and show metrics for last 60 seconds (default behaviour)
-  EZmetrics.new.show
+# Aggregate and show metrics for last 60 seconds (default behaviour)
+EZmetrics.new.show
 ```
 
-or
-
 ```ruby
-  # Aggregate and show metrics for last 10 minutes
-  EZmetrics.new(10.minutes).show
+# Aggregate and show metrics for last 10 minutes
+EZmetrics.new(10.minutes).show
 ```
 
-> Please note that you can combine these timeframes, for example - store for 10 minutes, display for 5 minutes.
-
+You can combine these timeframes, for example - store for 10 minutes, display for 5 minutes.
 
 ### Capture metrics
 
@@ -121,15 +131,191 @@ This will return a hash with the following structure:
 }
 ```
 
-### Performance
+---
+
+If you prefer a single level object - you can change the default output structure by calling `.flatten` before `.show`
+
+```ruby
+EZmetrics.new(1.hour).flatten.show(db: :avg, duration: [:avg, :max])
+```
+
+```ruby
+{
+  db_avg:       182,
+  duration_avg: 205,
+  duration_max: 5171
+}
+```
+
+---
+
+Same for [partitioned aggregation](#partitioning)
+
+```ruby
+EZmetrics.new(1.hour).partition_by(:minute).flatten.show(db: :avg, duration: [:avg, :max])
+```
+
+```ruby
+[
+  {
+    timestamp:    1575242880,
+    db_avg:       387,
+    duration_avg: 477,
+    duration_max: 8566
+  },
+  {
+    timestamp:    1575242940,
+    db_avg:       123,
+    duration_avg: 234,
+    duration_max: 3675
+  }
+]
+```
+
+### Aggregation
+
+The aggregation can be easily configured by specifying aggregation options as in the following examples:
+
+**1. Single**
+
+```ruby
+EZmetrics.new.show(duration: :max)
+```
+
+```ruby
+{
+  duration: {
+    max: 9675
+  }
+}
+```
+
+---
+
+**2. Multiple**
+
+```ruby
+EZmetrics.new.show(queries: [:max, :avg])
+```
+
+```ruby
+{
+  queries: {
+    max: 76,
+    avg: 26
+  }
+}
+```
+
+---
+
+**3. Requests**
+
+```ruby
+EZmetrics.new.show(requests: true)
+```
+
+```ruby
+{
+  requests: {
+    all: 2000,
+    grouped: {
+      "2xx" => 1900,
+      "3xx" => 15,
+      "4xx" => 80,
+      "5xx" => 5
+    }
+  }
+}
+```
+
+---
+
+**4. Combined**
+
+```ruby
+EZmetrics.new.show(views: :avg, :db: [:avg, :max], requests: true)
+```
+
+```ruby
+{
+  views: {
+    avg: 12
+  },
+  db: {
+    avg: 155,
+    max: 4382
+  },
+  requests: {
+    all: 2000,
+    grouped: {
+      "2xx" => 1900,
+      "3xx" => 15,
+      "4xx" => 80,
+      "5xx" => 5
+    }
+  }
+}
+```
+
+### Partitioning
+
+If you want to visualize your metrics by using a **line chart**, you will need to use partitioning.
+
+To aggregate metrics, partitioned by a unit of time you need to call `.partition_by({time_unit})` before calling `.show`
+
+```ruby
+# Aggregate metrics for last hour, partition by minute
+EZmetrics.new(1.hour).partition_by(:minute).show(duration: [:avg, :max], db: :avg)
+```
 
-The implementation is based on **Redis** commands such as:
+This will return an array of objects with the following structure:
 
-- [`get`](https://redis.io/commands/get)
-- [`mget`](https://redis.io/commands/mget)
-- [`setex`](https://redis.io/commands/setex)
+```ruby
+[
+  {
+    timestamp: # UNIX timestamp
+    data:      # a hash with aggregated metrics
+  }
+]
+```
 
-which are extremely fast.
+like in the example below:
+
+```ruby
+[
+  {
+    timestamp: 1575242880,
+    data: {
+      duration: {
+        avg: 477,
+        max: 8566
+      },
+      db: {
+        avg: 387
+      }
+    }
+  },
+  {
+    timestamp: 1575242940,
+    data: {
+      duration: {
+        avg: 234,
+        max: 3675
+      },
+      db: {
+        avg: 123
+      }
+    }
+  }
+]
+```
+
+Available time units for partitioning: `second`, `minute`, `hour`, `day`. Default: `minute`.
+
+### Performance
+
+The aggregation speed relies on the performance of **Redis** (data storage) and **Oj** (json serialization/parsing).
 
 You can check the **aggregation** time by running:
 
@@ -137,11 +323,28 @@ You can check the **aggregation** time by running:
 EZmetrics::Benchmark.new.measure_aggregation
 ```
 
-The result of running this benchmark on a *2017 Macbook Pro 2.9 GHz Intel Core i7 with 16 GB of RAM*:
+| Interval | Duration (seconds) |
+| :------: | :----------------: |
+| 1 minute |        0.0         |
+|  1 hour  |        0.04        |
+| 12 hours |        0.49        |
+| 24 hours |        1.51        |
+| 48 hours |        3.48        |
+
+---
+
+To check the **partitioned aggregation** time you need to run:
+
+```ruby
+EZmetrics::Benchmark.new.measure_aggregation(:minute)
+```
 
 | Interval | Duration (seconds) |
-|:--------:|:------------------:|
+| :------: | :----------------: |
 | 1 minute |        0.0         |
-|  1 hour  |        0.11        |
-| 12 hours |        1.6         |
-| 24 hours |        3.5         |
+|  1 hour  |        0.04        |
+| 12 hours |        0.53        |
+| 24 hours |        1.59        |
+| 48 hours |        3.51        |
+
+The benchmarks above were run on a _2017 Macbook Pro 2.9 GHz Intel Core i7 with 16 GB of RAM_

data/lib/ezmetrics.rb

@@ -1,11 +1,15 @@
-require "redis" unless defined?(Redis)
-require "json"  unless defined?(JSON)
+require "redis"
+require "redis/connection/hiredis"
+require "oj"
 
 class EZmetrics
+  METRICS               = [:duration, :views, :db, :queries].freeze
+  AGGREGATION_FUNCTIONS = [:max, :avg].freeze
+  PARTITION_UNITS       = [:second, :minute, :hour, :day].freeze
+
   def initialize(interval_seconds=60)
     @interval_seconds = interval_seconds.to_i
     @redis            = Redis.new
-    @storage_key      = "ez-metrics"
   end
 
   def log(payload={duration: 0.0, views: 0.0, db: 0.0, queries: 0, status: 200})
@@ -19,12 +23,12 @@ class EZmetrics
 
     this_second          = Time.now.to_i
     status_group         = "#{payload[:status].to_s[0]}xx"
-    @this_second_metrics = redis.get("#{storage_key}:#{this_second}")
+    @this_second_metrics = redis.get(this_second)
 
     if this_second_metrics
-      @this_second_metrics = JSON.parse(this_second_metrics)
+      @this_second_metrics = Oj.load(this_second_metrics)
 
-      [:duration, :views, :db, :queries].each do |metrics_type|
+      METRICS.each do |metrics_type|
         update_sum(metrics_type)
         update_max(metrics_type)
       end
@@ -33,6 +37,7 @@ class EZmetrics
       this_second_metrics["statuses"][status_group] += 1
     else
       @this_second_metrics = {
+        "second"       => this_second,
         "duration_sum" => safe_payload[:duration],
         "duration_max" => safe_payload[:duration],
         "views_sum"    => safe_payload[:views],
@@ -47,112 +52,155 @@ class EZmetrics
       this_second_metrics["statuses"][status_group] = 1
     end
 
-    redis.setex("#{storage_key}:#{this_second}", interval_seconds, JSON.generate(this_second_metrics))
-
+    redis.setex(this_second, interval_seconds, Oj.dump(this_second_metrics))
     true
   rescue => error
     formatted_error(error)
   end
 
-  def show
-    interval_start    = Time.now.to_i - interval_seconds
-    interval_keys     = (interval_start..Time.now.to_i).to_a.map { |second| "#{storage_key}:#{second}" }
-    @interval_metrics = redis.mget(interval_keys).compact.map { |hash| JSON.parse(hash) }
+  def show(options=nil)
+    @options = options || default_options
+    partitioned_metrics ? aggregate_partitioned_data : aggregate_data
+  end
+
+  def flatten
+    @flat = true
+    self
+  end
+
+  def partition_by(time_unit=:minute)
+    time_unit = PARTITION_UNITS.include?(time_unit) ? time_unit : :minute
+    @partitioned_metrics = interval_metrics.group_by { |h| second_to_partition_unit(time_unit, h["second"]) }
+    self
+  end
 
-    return empty_metrics_object unless interval_metrics.any?
+  private
 
-    @requests = interval_metrics.map { |hash| hash["statuses"]["all"] }.compact.sum
+  attr_reader :redis, :interval_seconds, :interval_metrics, :requests, :flat,
+    :storage_key, :safe_payload, :this_second_metrics, :partitioned_metrics, :options
 
-    metrics_object
+  def aggregate_data
+    return {} unless interval_metrics.any?
+    @requests = interval_metrics.sum { |hash| hash["statuses"]["all"] }
+    build_result
   rescue
-    empty_metrics_object
+    {}
   end
 
-  private
+  def aggregate_partitioned_data
+    partitioned_metrics.map do |partition, metrics|
+      @interval_metrics = metrics
+      @requests = interval_metrics.sum { |hash| hash["statuses"]["all"] }
+      flat ? { timestamp: partition, **build_result } : { timestamp: partition, data: build_result }
+    end
+  rescue
+    new(options)
+  end
+
+  def build_result
+    result = {}
+
+    if options[:requests]
+      append_requests_to_result(result, { all: requests, grouped: count_all_status_groups })
+    end
+
+    options.each do |metrics, aggregation_functions|
+      next unless METRICS.include?(metrics)
+      aggregation_functions = [aggregation_functions] unless aggregation_functions.is_a?(Array)
+      next unless aggregation_functions.any?
+
+      aggregation_functions.each do |aggregation_function|
+        aggregated_metrics = aggregate(metrics, aggregation_function)
+        append_metrics_to_result(result, metrics, aggregation_function, aggregated_metrics)
+      end
+    end
+    result
+  ensure
+    result
+  end
+
+  def append_requests_to_result(result, aggregated_requests)
+    return result[:requests] = aggregated_requests unless flat
+
+    result[:requests_all] = aggregated_requests[:all]
+    aggregated_requests[:grouped].each do |group, counter|
+      result[:"requests_#{group}"] = counter
+    end
+  end
 
-  attr_reader :redis, :interval_seconds, :interval_metrics, :requests, :storage_key,
-    :safe_payload, :this_second_metrics
+  def append_metrics_to_result(result, metrics, aggregation_function, aggregated_metrics)
+    return result[:"#{metrics}_#{aggregation_function}"] = aggregated_metrics if flat
+
+    result[metrics] ||= {}
+    result[metrics][aggregation_function] = aggregated_metrics
+  end
+
+  def second_to_partition_unit(time_unit, second)
+    return second if time_unit == :second
+    time = Time.at(second)
+    return (time - time.sec - time.min * 60 - time.hour * 3600).to_i if time_unit == :day
+    return (time - time.sec - time.min * 60).to_i                    if time_unit == :hour
+    (time - time.sec).to_i
+  end
+
+  def interval_metrics
+    @interval_metrics ||= begin
+      interval_start    = Time.now.to_i - interval_seconds
+      interval_keys     = (interval_start..Time.now.to_i).to_a
+      redis.mget(interval_keys).compact.map { |hash| Oj.load(hash) }
+    end
+  end
+
+  def aggregate(metrics, aggregation_function)
+    return unless AGGREGATION_FUNCTIONS.include?(aggregation_function)
+    return avg("#{metrics}_sum") if aggregation_function == :avg
+    return max("#{metrics}_max") if aggregation_function == :max
+  end
 
   def update_sum(metrics)
-    this_second_metrics["#{metrics}_sum"] += safe_payload[metrics.to_sym]
+    this_second_metrics["#{metrics}_sum"] += safe_payload[metrics]
   end
 
   def update_max(metrics)
-    max_value = [safe_payload[metrics.to_sym], this_second_metrics["#{metrics}_max"]].max
+    max_value = [safe_payload[metrics], this_second_metrics["#{metrics}_max"]].max
     this_second_metrics["#{metrics}_max"] = max_value
   end
 
   def avg(metrics)
-    (interval_metrics.map { |h| h[metrics.to_s] }.sum.to_f / requests).round
+    (interval_metrics.sum { |h| h[metrics] }.to_f / requests).round
   end
 
   def max(metrics)
-    interval_metrics.map { |h| h[metrics.to_s] }.max.round
+    interval_metrics.max { |h| h[metrics] }[metrics].round
   end
 
-  def count(group)
-    interval_metrics.map { |h| h["statuses"][group.to_s] }.sum
-  end
-
-  def formatted_error(error)
-    {
-      error:     error.class.name,
-      message:   error.message,
-      backtrace: error.backtrace.reject { |line| line.match(/ruby|gems/) }
-    }
+  def count_all_status_groups
+    interval_metrics.inject({ "2xx" => 0, "3xx" => 0, "4xx" => 0, "5xx" => 0 }) do |result, h|
+      result["2xx"] += h["statuses"]["2xx"]
+      result["3xx"] += h["statuses"]["3xx"]
+      result["4xx"] += h["statuses"]["4xx"]
+      result["5xx"] += h["statuses"]["5xx"]
+      result
+    end
   end
 
-  def metrics_object
+  def default_options
     {
-      duration: {
-        avg: avg(:duration_sum),
-        max: max(:duration_max)
-      },
-      views: {
-        avg: avg(:views_sum),
-        max: max(:views_max)
-      },
-      db: {
-        avg: avg(:db_sum),
-        max: max(:db_max)
-      },
-      queries: {
-        avg: avg(:queries_sum),
-        max: max(:queries_max)
-      },
-      requests: {
-        all:     requests,
-        grouped: {
-          "2xx" => count("2xx"),
-          "3xx" => count("3xx"),
-          "4xx" => count("4xx"),
-          "5xx" => count("5xx")
-        }
-      }
+      duration: AGGREGATION_FUNCTIONS,
+      views:    AGGREGATION_FUNCTIONS,
+      db:       AGGREGATION_FUNCTIONS,
+      queries:  AGGREGATION_FUNCTIONS,
+      requests: true
     }
   end
 
-  def empty_metrics_object
+  def formatted_error(error)
     {
-      duration: {
-        avg: 0,
-        max: 0
-      },
-      views: {
-        avg: 0,
-        max: 0
-      },
-      db: {
-        avg: 0,
-        max: 0
-      },
-      queries: {
-        avg: 0,
-        max: 0
-      },
-      requests: {}
+      error:     error.class.name,
+      message:   error.message,
+      backtrace: error.backtrace.reject { |line| line.match(/ruby|gems/) }
     }
   end
 end
 
-require "ezmetrics/benchmark"
+require "ezmetrics/benchmark"

data/lib/ezmetrics/benchmark.rb

@@ -11,15 +11,16 @@ class EZmetrics::Benchmark
       "1.minute" => 60,
       "1.hour  " => 3600,
       "12.hours" => 43200,
-      "24.hours" => 86400
+      "24.hours" => 86400,
+      "48.hours" => 172800
     }
   end
 
-  def measure_aggregation
+  def measure_aggregation(partition_by=nil)
     write_metrics
     print_header
     intervals.each do |interval, seconds|
-      result = measure_aggregation_time(interval, seconds)
+      result = measure_aggregation_time(interval, seconds, partition_by)
       print_row(result)
     end
     cleanup_metrics
@@ -35,36 +36,43 @@ class EZmetrics::Benchmark
     seconds.times do |i|
       second = start - i
       payload = {
-        "duration_sum" => rand(10000),
-        "duration_max" => rand(10000),
-        "views_sum"    => rand(1000),
-        "views_max"    => rand(1000),
-        "db_sum"       => rand(8000),
-        "db_max"       => rand(8000),
-        "queries_sum"  => rand(100),
-        "queries_max"  => rand(100),
-        "statuses"     => {
-          "2xx" => rand(10),
-          "3xx" => rand(10),
-          "4xx" => rand(10),
-          "5xx" => rand(10),
-          "all" => rand(40)
+        "second"          => second,
+        "duration_sum"    => rand(10000),
+        "duration_max"    => rand(10000),
+        "views_sum"       => rand(1000),
+        "views_max"       => rand(1000),
+        "db_sum"          => rand(8000),
+        "db_max"          => rand(8000),
+        "queries_sum"     => rand(100),
+        "queries_max"     => rand(100),
+        "statuses"        => {
+          "2xx" => rand(1..10),
+          "3xx" => rand(1..10),
+          "4xx" => rand(1..10),
+          "5xx" => rand(1..10),
+          "all" => rand(1..40)
         }
       }
-      redis.setex("ez-metrics:#{second}", seconds, JSON.generate(payload))
+      redis.setex(second, seconds, Oj.dump(payload))
     end
     nil
   end
 
   def cleanup_metrics
     interval_start = Time.now.to_i - intervals.values.max - 100
-    interval_keys  = (interval_start..Time.now.to_i).to_a.map { |second| "ez-metrics:#{second}" }
+    interval_keys  = (interval_start..Time.now.to_i).to_a
     redis.del(interval_keys)
   end
 
-  def measure_aggregation_time(interval, seconds)
+  def measure_aggregation_time(interval, seconds, partition_by)
     iterations.times do
-      durations << ::Benchmark.measure { EZmetrics.new(seconds).show }.real
+      durations << ::Benchmark.measure do
+        if partition_by
+          EZmetrics.new(seconds).partition_by(partition_by).show
+        else
+          EZmetrics.new(seconds).show
+        end
+      end.real
     end
 
     return {
@@ -84,4 +92,4 @@ class EZmetrics::Benchmark
   def print_footer
     print "#{'─'*31}\n"
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ezmetrics
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.2.2
 platform: ruby
 authors:
 - Nicolae Rotaru
@@ -24,6 +24,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: hiredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.3
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +66,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.5'
-description: A simple tool for capturing and displaying Rails metrics.
+description: Simple, lightweight and fast metrics aggregation for Rails.
 email: nyku.rn@gmail.com
 executables: []
 extensions: []
@@ -67,8 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Rails metrics aggregation tool.