librato-metrics 0.6.0.pre2 → 0.6.0.pre3

data/README.md

@@ -40,9 +40,9 @@ Make sure you have [an account for Metrics](https://metrics.librato.com/) and th
 
     Librato::Metrics.authenticate 'email', 'api_key'
 
-## Sending Metrics
+## Sending Measurements
 
-If you are sending very many metrics or sending them very often, it will be much higher performance to bundle them up together to reduce your request volume. Use `Queue` for this.
+If you are sending very many measurements or sending them very often, it will be much higher performance to bundle them up together to reduce your request volume. Use `Queue` for this.
 
 Queue up a simple gauge metric named `temperature`:
 
@@ -65,15 +65,46 @@ Save all queued metrics:
 
     queue.submit
 
+## Aggregate Measurements
+
+If you are measuring something very frequently e.g. per-request in a web application (order mS)  you may not want to send each individual measurement, but rather periodically send a [single aggregate measurement](http://dev.librato.com/v1/post/metrics#gauge_specific), spanning multiple seconds or even minutes. Use an `Aggregator` for this.
+
+Aggregate a simple gauge metric named `response_latency`:
+
+    aggregator = Librato::Metrics::Aggregator.new
+    aggregator.add :response_latency => 85.0
+    aggregator.add :response_latency => 100.5
+    aggregator.add :response_latency => 150.2
+    aggregator.add :response_latency => 90.1
+    aggregator.add :response_latency => 92.0
+
+Which would result in a gauge measurement like:
+
+    {:name => "response_latency", :count => 5, :sum => 517.8, :min => 85.0, :max => 150.2}
+
+You can specify a source during aggregate construction:
+
+    aggregator = Librato::Metrics::Aggregator.new(:source => 'foobar')
+
+You can aggregate multiple metrics at once:
+
+    aggregator.add :app_latency => 35.2, :db_latency => 120.7
+
+Send the current aggregated metrics to Metrics:
+
+    aggregator.submit
+
 ## Benchmarking
 
-If you have operations in your application you want to record execution time for, you can use the `#time` method:
+If you have operations in your application you want to record execution time for, both `Queue` and `Aggregator` support the `#time` method:
 
-    queue.time :my_measurement do
+    aggregator.time :my_measurement do
       # do work...
     end
 
-If you need extra attributes for the measurement, simply add them on:
+The difference between the two is that `Queue` submits each timing measurement individually, while `Aggregator` submits a single timing measurement spanning all executions.
+
+If you need extra attributes for a `Queue` timing measurement, simply add them on:
 
     queue.time :my_measurement, :source => 'app1' do
       # do work...
@@ -162,4 +193,4 @@ These are features we expect to add in future versions, roughly in the order of
 
 ## Copyright
 
-Copyright (c) 2011-2012 [Librato Inc.](http://librato.com) See LICENSE for details.
+Copyright (c) 2011-2012 [Librato Inc.](http://librato.com) See LICENSE for details.

data/lib/librato/metrics.rb

@@ -4,6 +4,7 @@ $:.unshift(File.dirname(__FILE__)) unless
 require 'base64'
 require 'multi_json'
 
+require 'metrics/aggregator'
 require 'metrics/client'
 require 'metrics/collection'
 require 'metrics/connection'

data/lib/librato/metrics/aggregator.rb

@@ -0,0 +1,68 @@
+require 'aggregate'
+require 'metrics/processor'
+
+module Librato
+  module Metrics
+
+    class Aggregator
+      include Processor
+      
+      attr_reader :source
+
+      def initialize(options={})
+        @aggregated = {}
+        @client = options[:client] || Librato::Metrics.client
+        @source = options[:source]
+      end
+
+      # Add a metric entry to the metric set:
+      #
+      # @param Hash metrics metrics to add
+      def add(args)
+        args.each do |k, v|
+          value = v.respond_to?(:each) ? v[:value] : v
+
+          @aggregated[k] ||= Aggregate.new
+          @aggregated[k] << value
+        end
+      end
+      
+      # Returns true if aggregate contains no measurements
+      #
+      # @return Boolean
+      def empty?
+        @aggregated.empty?
+      end
+
+      # Remove all queued metrics
+      #
+      def flush
+        @aggregated = {}
+      end
+      alias :clear :flush
+
+      def queued
+        gauges = []
+
+        @aggregated.each do |k,v|
+          gauges << {
+            :name => k.to_s,
+            :count => v.count,
+            :sum => v.sum,
+
+            # TODO: make float/non-float consistent in the gem
+            :min => v.min.to_f,
+            :max => v.max.to_f
+            # TODO: expose v.sum2 and include
+          }
+        end
+
+        req = { :gauges => gauges }
+        req[:source] = @source if @source
+
+        req
+      end
+
+    end
+  end
+end

data/lib/librato/metrics/client.rb

@@ -103,7 +103,7 @@ module Librato
         # expects 200
         url = connection.build_url("metrics/#{metric}", query)
         response = connection.get(url)
-        parsed = MultiJson.decode(response.body)
+        parsed = MultiJson.load(response.body)
         # TODO: pagination support
         query.empty? ? parsed : parsed["measurements"]
       end

data/lib/librato/metrics/collection.rb

@@ -14,7 +14,7 @@ module Librato
         # expects 200
         url = connection.build_url(path, query)
         response = connection.get(url)
-        parsed = MultiJson.decode(response.body)
+        parsed = MultiJson.load(response.body)
         results = parsed["metrics"]
         return results if parsed["query"]["found"] <= MAX_RESULTS
         query[:offset] = MAX_RESULTS
@@ -22,7 +22,7 @@ module Librato
           # expects 200
           url = connection.build_url(path, query)
           response = connection.get(url)
-          parsed = MultiJson.decode(response.body)
+          parsed = MultiJson.load(response.body)
           results.push(*parsed["metrics"])
           query[:offset] += MAX_RESULTS
         end while query[:offset] < parsed["query"]["found"]

data/lib/librato/metrics/persistence/direct.rb

@@ -17,7 +17,7 @@ module Librato
             requests = [queued]
           end
           requests.each do |request|
-            payload = MultiJson.encode(request)
+            payload = MultiJson.dump(request)
             # expects 200
             client.connection.post('metrics', payload)
           end

data/lib/librato/metrics/processor.rb

@@ -0,0 +1,78 @@
+module Librato
+  module Metrics
+    
+    module Processor
+      MEASUREMENTS_PER_REQUEST = 500
+      
+      attr_reader :per_request
+      
+      # The current Client instance this queue is using to authenticate
+      # and connect to Librato Metrics. This will default to the primary
+      # client used by the Librato::Metrics module unless it has been
+      # set to something else.
+      #
+      # @return [Librato::Metrics::Client]
+      def client
+        @client ||= Librato::Metrics.client
+      end
+
+      # The object this MetricSet will use to persist
+      #
+      def persister
+        @persister ||= create_persister
+      end
+
+      # Persist currently queued metrics
+      #
+      # @return Boolean
+      def submit
+        raise NoMetricsQueued if self.queued.empty?
+        options = {:per_request => @per_request}
+        if persister.persist(self.client, self.queued, options)
+          flush and return true
+        end
+        false
+      end
+      
+      # Capture execution time for a block and queue
+      # it as the value for a metric. Times are recorded
+      # in milliseconds.
+      #
+      # Options are the same as for {#add}.
+      #
+      # @example Queue API request response time
+      #   queue.time :api_request_time do
+      #     # API request..
+      #   end
+      #
+      # @example Queue API request response time w/ source
+      #   queue.time :api_request_time, :source => 'app1' do
+      #     # API request..
+      #   end
+      #
+      # @param [Symbol|String] name Metric name
+      # @param [Hash] options Metric options
+      def time(name, options={})
+        start = Time.now
+        yield
+        duration = (Time.now - start) * 1000.0 # milliseconds
+        metric = {name => options.merge({:value => duration})}
+        add metric
+      end
+      alias :benchmark :time
+      
+    private
+
+      def create_persister
+        type = self.client.persistence.to_s.capitalize
+        Librato::Metrics::Persistence.const_get(type).new
+      end
+
+      def epoch_time
+        Time.now.to_i
+      end
+      
+    end
+    
+  end
+end

data/lib/librato/metrics/queue.rb

@@ -1,16 +1,18 @@
+require 'metrics/processor'
+
+
 module Librato
   module Metrics
     class Queue
-      MEASUREMENTS_PER_REQUEST = 500
+      include Processor
 
-      attr_reader :per_request
       attr_accessor :skip_measurement_times
 
       def initialize(options={})
         @queued = {}
+        @client = options[:client] || Librato::Metrics.client
         @per_request = options[:per_request] || MEASUREMENTS_PER_REQUEST
         @skip_measurement_times = options[:skip_measurement_times]
-        @client = options[:client] || Librato::Metrics.client
       end
 
       # Add a metric entry to the metric set:
@@ -37,16 +39,6 @@ module Librato
         queued
       end
 
-      # The current Client instance this queue is using to authenticate
-      # and connect to Librato Metrics. This will default to the primary
-      # client used by the Librato::Metrics module unless it has been
-      # set to something else.
-      #
-      # @return [Librato::Metrics::Client]
-      def client
-        @client ||= Librato::Metrics.client
-      end
-
       # Currently queued counters
       #
       # @return [Array]
@@ -69,12 +61,6 @@ module Librato
       alias :clear :flush
       alias :flush_queued :flush
 
-      # The object this MetricSet will use to persist
-      #
-      def persister
-        @persister ||= create_persister
-      end
-
       # Currently queued gauges
       #
       # @return Array
@@ -97,56 +83,6 @@ module Librato
       end
       alias :length :size
 
-      # Persist currently queued metrics
-      #
-      # @return Boolean
-      def submit
-        raise NoMetricsQueued if self.queued.empty?
-        options = {:per_request => @per_request}
-        if persister.persist(self.client, self.queued, options)
-          flush and return true
-        end
-        false
-      end
-
-      # Capture execution time for a block and queue
-      # it as the value for a metric. Times are recorded
-      # in milliseconds.
-      #
-      # Options are the same as for {#add}.
-      #
-      # @example Queue API request response time
-      #   queue.time :api_request_time do
-      #     # API request..
-      #   end
-      #
-      # @example Queue API request response time w/ source
-      #   queue.time :api_request_time, :source => 'app1' do
-      #     # API request..
-      #   end
-      #
-      # @param [Symbol|String] name Metric name
-      # @param [Hash] options Metric options
-      def time(name, options={})
-        start = Time.now
-        yield
-        duration = (Time.now - start) * 1000.0 # milliseconds
-        metric = {name => options.merge({:value => duration})}
-        add metric
-      end
-      alias :benchmark :time
-
-    private
-
-      def create_persister
-        type = self.client.persistence.to_s.capitalize
-        Librato::Metrics::Persistence.const_get(type).new
-      end
-
-      def epoch_time
-        Time.now.to_i
-      end
-
     end
   end
 end

data/lib/librato/metrics/version.rb

@@ -1,5 +1,5 @@
 module Librato
   module Metrics
-    VERSION = "0.6.0.pre2"
+    VERSION = "0.6.0.pre3"
   end
 end

data/librato-metrics.gemspec

@@ -25,7 +25,8 @@ Gem::Specification.new do |s|
 
   ## runtime dependencies
   s.add_dependency 'faraday', '~>0.7.6'
-  s.add_dependency 'multi_json'
+  s.add_dependency 'multi_json', '~>1.3.1'
+  s.add_dependency 'aggregate', '~>0.2.2'
 
   ## development dependencies
   s.add_development_dependency 'rake'

data/spec/unit/metrics/aggregator_spec.rb

@@ -0,0 +1,173 @@
+require "spec_helper.rb"
+module Librato
+  module Metrics
+
+    describe Aggregator do
+
+      before(:all) do
+        @time = Time.now.to_i
+        Aggregator.stub(:epoch_time).and_return(@time)
+      end
+
+      describe "initialization" do
+        context "with specified client" do
+          it "should set to client" do
+            barney = Client.new
+            a = Aggregator.new(:client => barney)
+            a.client.should be barney
+          end
+        end
+
+        context "without specified client" do
+          it "should use Librato::Metrics client" do
+            a = Aggregator.new
+            a.client.should be Librato::Metrics.client
+          end
+        end
+
+        context "with specified source" do
+          it "should set to source" do
+            a = Aggregator.new(:source => 'rubble')
+            a.source.should == 'rubble'
+          end
+        end
+
+        context "without specified source" do
+          it "should not have a source" do
+            a = Aggregator.new
+            a.source.should be_nil
+          end
+        end
+      end
+
+      describe "#add" do
+        context "with single hash argument" do
+          it "should record a single aggregate" do
+            subject.add :foo => 3000
+            expected = { #:measure_time => @time, TODO: support specific time
+                :gauges => [
+                { :name => 'foo',
+                  :count => 1,
+                  :sum => 3000.0,
+                  :min => 3000.0,
+                  :max => 3000.0}
+                ]
+            }
+            subject.queued.should equal_unordered(expected)
+          end
+
+          it "should aggregate multiple measurements" do
+            subject.add :foo => 1
+            subject.add :foo => 2
+            subject.add :foo => 3
+            subject.add :foo => 4
+            subject.add :foo => 5
+            expected = { :gauges => [
+                { :name => 'foo',
+                  :count => 5,
+                  :sum => 15.0,
+                  :min => 1.0,
+                  :max => 5.0}
+                ]
+            }
+            subject.queued.should equal_unordered(expected)
+          end
+        end
+
+        context "with multiple hash arguments" do
+          it "should record a single aggregate" do
+            subject.add :foo => 3000
+            subject.add :bar => 30
+            expected = { 
+              #:measure_time => @time, TODO: support specific time
+              :gauges => [
+                { :name => 'foo',
+                  :count => 1,
+                  :sum => 3000.0,
+                  :min => 3000.0,
+                  :max => 3000.0},
+                { :name => 'bar',
+                  :count => 1,
+                  :sum => 30.0,
+                  :min => 30.0,
+                  :max => 30.0},
+                ]
+            }
+            subject.queued.should equal_unordered(expected)
+          end
+
+          it "should aggregate multiple measurements" do
+            subject.add :foo => 1
+            subject.add :foo => 2
+            subject.add :foo => 3
+            subject.add :foo => 4
+            subject.add :foo => 5
+
+            subject.add :bar => 6
+            subject.add :bar => 7
+            subject.add :bar => 8
+            subject.add :bar => 9
+            subject.add :bar => 10
+            expected = { :gauges => [
+                { :name => 'foo',
+                  :count => 5,
+                  :sum => 15.0,
+                  :min => 1.0,
+                  :max => 5.0},
+                { :name => 'bar',
+                  :count => 5,
+                  :sum => 40.0,
+                  :min => 6.0,
+                  :max => 10.0}
+                ]
+            }
+            subject.queued.should equal_unordered(expected)
+          end
+        end
+      end
+
+      describe "#submit" do
+        before(:all) do
+          Librato::Metrics.authenticate 'me@librato.com', 'foo'
+          Librato::Metrics.persistence = :test
+        end
+
+        context "when successful" do
+          it "should flush queued metrics and return true" do
+            subject.add :steps => 2042, :distance => 1234
+            subject.submit.should be_true
+            subject.empty?.should be_true
+          end
+        end
+
+        context "when failed" do
+          it "should preserve queue and return false" do
+            subject.add :steps => 2042, :distance => 1234
+            subject.persister.return_value(false)
+            subject.submit.should be_false
+            subject.empty?.should be_false
+          end
+        end
+      end
+
+      describe "#time" do
+        context "with metric name only" do
+          it "should queue metric with timed value" do
+            1.upto(5) do
+              subject.time :sleeping do
+                sleep 0.1
+              end
+            end
+            queued = subject.queued[:gauges][0]
+            queued[:name].should == 'sleeping'
+            queued[:count].should be 5
+            queued[:sum].should be > 500
+            queued[:sum].should be_within(150).of(500)
+          end
+        end
+      end
+
+    end
+
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: librato-metrics
 version: !ruby/object:Gem::Version
-  version: 0.6.0.pre2
+  version: 0.6.0.pre3
   prerelease: 6
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-09 00:00:00.000000000Z
+date: 2012-04-26 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
-  requirement: &70288570431740 !ruby/object:Gem::Requirement
+  requirement: &70105435038240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,21 +21,32 @@ dependencies:
         version: 0.7.6
   type: :runtime
   prerelease: false
-  version_requirements: *70288570431740
+  version_requirements: *70105435038240
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70288570431320 !ruby/object:Gem::Requirement
+  requirement: &70105435037740 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *70105435037740
+- !ruby/object:Gem::Dependency
+  name: aggregate
+  requirement: &70105435037280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.2
   type: :runtime
   prerelease: false
-  version_requirements: *70288570431320
+  version_requirements: *70105435037280
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70288570430860 !ruby/object:Gem::Requirement
+  requirement: &70105435036900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70288570430860
+  version_requirements: *70105435036900
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70288570430360 !ruby/object:Gem::Requirement
+  requirement: &70105435068860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +65,10 @@ dependencies:
         version: 2.6.0
   type: :development
   prerelease: false
-  version_requirements: *70288570430360
+  version_requirements: *70105435068860
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &70288570429940 !ruby/object:Gem::Requirement
+  requirement: &70105435068440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70288570429940
+  version_requirements: *70105435068440
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &70288570429480 !ruby/object:Gem::Requirement
+  requirement: &70105435067980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70288570429480
+  version_requirements: *70105435067980
 - !ruby/object:Gem::Dependency
   name: rdiscount
-  requirement: &70288570429060 !ruby/object:Gem::Requirement
+  requirement: &70105435067560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +98,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70288570429060
+  version_requirements: *70105435067560
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70288570428640 !ruby/object:Gem::Requirement
+  requirement: &70105435067140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +109,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70288570428640
+  version_requirements: *70105435067140
 - !ruby/object:Gem::Dependency
   name: popen4
-  requirement: &70288570428220 !ruby/object:Gem::Requirement
+  requirement: &70105435066720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +120,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70288570428220
+  version_requirements: *70105435066720
 description: An easy to use ruby wrapper for Librato's Metrics API
 email: matt@librato.com
 executables: []
@@ -126,6 +137,7 @@ files:
 - Rakefile
 - benchmarks/array_vs_set.rb
 - lib/librato/metrics.rb
+- lib/librato/metrics/aggregator.rb
 - lib/librato/metrics/client.rb
 - lib/librato/metrics/collection.rb
 - lib/librato/metrics/connection.rb
@@ -137,6 +149,7 @@ files:
 - lib/librato/metrics/persistence.rb
 - lib/librato/metrics/persistence/direct.rb
 - lib/librato/metrics/persistence/test.rb
+- lib/librato/metrics/processor.rb
 - lib/librato/metrics/queue.rb
 - lib/librato/metrics/version.rb
 - librato-metrics.gemspec
@@ -146,6 +159,7 @@ files:
 - spec/integration/metrics_spec.rb
 - spec/rackups/status.ru
 - spec/spec_helper.rb
+- spec/unit/metrics/aggregator_spec.rb
 - spec/unit/metrics/client_spec.rb
 - spec/unit/metrics/connection_spec.rb
 - spec/unit/metrics/queue_spec.rb
@@ -182,6 +196,7 @@ test_files:
 - spec/integration/metrics_spec.rb
 - spec/rackups/status.ru
 - spec/spec_helper.rb
+- spec/unit/metrics/aggregator_spec.rb
 - spec/unit/metrics/client_spec.rb
 - spec/unit/metrics/connection_spec.rb
 - spec/unit/metrics/queue_spec.rb