librato-rails 0.5.2 → 0.6.0

data/README.md

@@ -28,7 +28,9 @@ Create a `config/librato.yml` like the following:
     production:
       user: <your-email>
       token: <your-api-key>
-      
+
+(the file is parsed via ERB in case you need to add some magic in there - useful in some cloud environments)
+
 OR provide `LIBRATO_METRICS_USER` and `LIBRATO_METRICS_TOKEN` environment variables. If both env variables and a config file are present, environment variables will take precendence.
 
 Note that using a configuration file allows you to specify configurations per-environment. Submission will be disabled in any environment without credentials. However, if environment variables are set they will be used in all environments. 
@@ -53,17 +55,30 @@ Use for tracking a running total of something _across_ requests, examples:
     Librato.increment 'sales_completed'
     
     # increment by five
-    Librato.increment 'items_purchased', 5
+    Librato.increment 'items_purchased', :by => 5
+    
+    # increment with a custom source
+    Librato.increment 'user.purchases', :source => user.id
     
 Other things you might track this way: user signups, requests of a certain type or to a certain route, total jobs queued or processed, emails sent or received
 
+###### Sporadic Increment Reporting
+
+Note that `increment` is primarily used for tracking the rate of occurrence of some event. Given this increment metrics are _continuous by default_: after being called on a metric once they will report on every interval, reporting zeros for any interval when increment was not called on the metric.
+
+Especially with custom sources you may want the opposite behavior - reporting a measurement only during intervals where `increment` was called on the metric:
+
+    # report a value for 'user.uploaded_file' only during non-zero intervals
+    Librato.increment 'user.uploaded_file', :source => user.id, :sporadic => true
+
 #### measure
 
 Use when you want to track an average value _per_-request. Examples:
 
     Librato.measure 'user.social_graph.nodes', 212
 
-    Librato.measure 'jobs.queued', 3
+	# report from a custom source
+    Librato.measure 'jobs.queued', 3, :source => 'worker.12'
     
 
 #### timing
@@ -96,6 +111,16 @@ Can also be written as:
 
 Symbols can be used interchangably with strings for metric names.
 
+## Cross-process Aggregation
+
+`librato-rails` submits measurements back to the Librato platform on a _per-process_ basis. By default these measurements are then combined into a single measurement per source (default is your hostname) before persisting the data. 
+
+For example if you have 4 hosts with 8 unicorn instances each (i.e. 32 processes total), on the Metrics site you'll find 4 data streams (1 per host) instead of 32.
+Current pricing applies after aggregation, so in this case you will be charged for 4 streams instead of 32.
+
+If you want to report per-process instead, you can set `source_pids` to `true` in
+your config, which will append the process id to the source name used by each thread. 
+
 ## Contribution
 
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.

data/lib/librato/rails/aggregator.rb

@@ -3,7 +3,7 @@ module Librato
     class Aggregator
       extend Forwardable
 
-      def_delegators :@cache, :empty?
+      def_delegators :@cache, :empty?, :prefix, :prefix=
 
       def initialize(options={})
         @cache = Librato::Metrics::Aggregator.new(:prefix => options[:prefix])
@@ -11,11 +11,19 @@ module Librato
       end
 
       def [](key)
+        fetch(key)
+      end
+      
+      def fetch(key, options={})
         return nil if @cache.empty?
         gauges = nil
+        source = options[:source]
         @lock.synchronize { gauges = @cache.queued[:gauges] }
         gauges.each do |metric|
-          return metric if metric[:name] == key.to_s
+          if metric[:name] == key.to_s
+            return metric if !source && !metric[:source]
+            return metric if source.to_s == metric[:source]
+          end
         end
         nil
       end
@@ -24,8 +32,7 @@ module Librato
         @lock.synchronize { @cache.clear }
       end
 
-      # transfer all measurements to a queue and
-      # reset internal status
+      # transfer all measurements to queue and reset internal status
       def flush_to(queue, options={})
         queued = nil
         @lock.synchronize do
@@ -36,9 +43,26 @@ module Librato
         queue.merge!(queued) if queued
       end
 
+      # @example Simple measurement
+      #   measure 'sources_returned', sources.length
+      #
+      # @example Simple timing in milliseconds
+      #   timing 'twitter.lookup', 2.31
+      #
+      # @example Block-based timing
+      #   timing 'db.query' do
+      #     do_my_query
+      #   end
+      #
+      # @example Custom source
+      #    measure 'user.all_orders', user.order_count, :source => user.id
+      #
       def measure(*args, &block)
+        options = {}
         event = args[0].to_s
         returned = nil
+        
+        # handle block or specified argument
         if block_given?
           start = Time.now
           returned = yield
@@ -48,8 +72,19 @@ module Librato
         else
           raise "no value provided"
         end
+        
+        # detect options hash if present
+        if args.length > 1 and args[-1].respond_to?(:each)
+          options = args[-1]
+        end
+        source = options[:source]
+        
         @lock.synchronize do
-          @cache.add event => value
+          if source
+            @cache.add event => {:source => source, :value => value}
+          else
+            @cache.add event => value
+          end
         end
         returned
       end

data/lib/librato/rails/collector.rb

@@ -0,0 +1,45 @@
+
+# an abstract collector object which can be given measurement values
+# and can periodically report those values back to the Metrics service
+
+module Librato
+  module Rails
+    class Collector
+      extend Forwardable
+    
+      def_delegators :counters, :increment
+      def_delegators :aggregate, :measure, :timing
+    
+      # access to internal aggregator object
+      def aggregate
+        @aggregator_cache ||= Aggregator.new(:prefix => @prefix)
+      end
+      
+      # access to internal counters object
+      def counters
+        @counter_cache ||= CounterCache.new
+      end
+      
+      # remove any accumulated but unsent metrics
+      def delete_all
+        aggregate.delete_all
+        counters.delete_all
+      end
+      
+      def group(prefix)
+        group = Group.new(prefix)
+        yield group
+      end
+      
+      # update prefix
+      def prefix=(new_prefix)
+        @prefix = new_prefix
+        aggregate.prefix = @prefix
+      end
+      
+      def prefix
+        @prefix
+      end
+    end
+  end
+end

data/lib/librato/rails/counter_cache.rb

@@ -2,6 +2,8 @@ module Librato
   module Rails
     
     class CounterCache
+      DEFAULT_SOURCE = '%%'
+      
       extend Forwardable
     
       def_delegators :@cache, :empty?
@@ -9,32 +11,108 @@ module Librato
       def initialize
         @cache = {}
         @lock = Mutex.new
+        @sporadics = {}
       end
     
+      # Retrieve the current value for a given metric. This is a short
+      # form for convenience which only retrieves metrics with no custom
+      # source specified. For more options see #fetch.
+      #
+      # @param [String|Symbol] key metric name
+      # @return [Integer|Float] current value
       def [](key)
-        @lock.synchronize { @cache[key.to_s] }
+        @lock.synchronize do 
+          @cache[key.to_s][DEFAULT_SOURCE] 
+        end
       end
       
+      # removes all tracked metrics. note this removes all measurement
+      # data AND metric names any continuously tracked metrics will not
+      # report until they get another measurement
       def delete_all
         @lock.synchronize { @cache.clear }
       end
       
+      
+      def fetch(key, options={})
+        source = DEFAULT_SOURCE
+        if options[:source]
+          source = options[:source].to_s
+        end
+        @lock.synchronize do 
+          return nil unless @cache[key.to_s]
+          @cache[key.to_s][source]
+        end
+      end
+      
+      # transfer all measurements to queue and reset internal status
       def flush_to(queue)
         counts = nil
         @lock.synchronize do
-          counts = @cache.dup
-          @cache.each_key { |key| @cache[key] = 0 }
+          # work off of a duplicate data set so we block for
+          # as little time as possible
+          counts = Marshal.load(Marshal.dump(@cache))
+          reset_cache
         end
-        counts.each do |key, value| 
-          queue.add key => value
+        counts.each do |key, data|
+          data.each do |source, value|
+            if source == DEFAULT_SOURCE
+              queue.add key => value
+            else
+              queue.add key => {:value => value, :source => source}
+            end
+          end
         end
       end
     
-      def increment(counter, by=1)
+      # Increment a given metric
+      #
+      # @example Increment metric 'foo' by 1
+      #   increment :foo
+      #
+      # @example Increment metric 'bar' by 2
+      #   increment :bar, :by => 2
+      #
+      # @example Increment metric 'foo' by 1 with a custom source
+      #   increment :foo, :source => user.id
+      #
+      def increment(counter, options={})
         counter = counter.to_s
+        if options.is_a?(Fixnum) 
+          # suppport legacy style
+          options = {:by => options}
+        end
+        by = options[:by] || 1
+        source = DEFAULT_SOURCE
+        if options[:source]
+          source = options[:source].to_s
+        end
+        if options[:sporadic]
+          make_sporadic(counter, source)
+        end
         @lock.synchronize do
-          @cache[counter] ||= 0
-          @cache[counter] += by
+          @cache[counter] ||= {}
+          @cache[counter][source] ||= 0
+          @cache[counter][source] += by
+        end
+      end
+      
+      private
+    
+      def make_sporadic(metric, source)
+        @sporadics[metric] ||= Set.new
+        @sporadics[metric] << source
+      end
+    
+      def reset_cache
+        # remove any source/metric pairs that aren't continuous
+        @sporadics.each do |key, sources|
+          sources.each { |source| @cache[key].delete(source) }
+        end
+        @sporadics.clear
+        # reset all continuous source/metric pairs to 0
+        @cache.each_key do |key|
+          @cache[key].each_key { |source| @cache[key][source] = 0 }
         end
       end
     

data/lib/librato/rails/railtie.rb

@@ -8,9 +8,7 @@ module Librato
       initializer 'librato_rails.setup' do |app|
         # don't start in test mode or in the console
         unless ::Rails.env.test? || defined?(::Rails::Console)
-          Librato::Rails.setup
-
-          app.middleware.use Librato::Rack::Middleware
+          Librato::Rails.setup(app)
         end
       end
     end

data/lib/librato/rails/version.rb

@@ -1,5 +1,5 @@
 module Librato
   module Rails
-    VERSION = "0.5.2"
+    VERSION = "0.6.0"
   end
 end

data/lib/librato/rails.rb

@@ -7,6 +7,7 @@ require 'librato/metrics'
 
 require 'librato/rack'
 require 'librato/rails/aggregator'
+require 'librato/rails/collector'
 require 'librato/rails/counter_cache'
 require 'librato/rails/group'
 require 'librato/rails/worker'
@@ -28,33 +29,23 @@ module Librato
     mattr_accessor :user
     mattr_accessor :token
     mattr_accessor :flush_interval
-    mattr_accessor :prefix
     mattr_accessor :source_pids
 
     # config defaults
     self.flush_interval = 60 # seconds
-    self.source_pids = true
+    self.source_pids = false # append process id to the source?
 
-    def_delegators :counters, :increment
-    def_delegators :aggregate, :measure, :timing
+    # a collector instance handles all measurement addition/storage
+    def_delegators :collector, :aggregate, :counters, :delete_all, :group, :increment, 
+                               :measure, :prefix, :prefix=, :timing
 
     class << self
 
-      # access to internal aggregator object
-      def aggregate
-        @aggregator_cache ||= Aggregator.new(:prefix => self.prefix)
-      end
-
       # set custom api endpoint
       def api_endpoint=(endpoint)
         @api_endpoint = endpoint
       end
 
-      # access to client instance
-      def client
-        @client ||= prepare_client
-      end
-
       # detect / update configuration
       def check_config
         if self.config_file && File.exists?(self.config_file)
@@ -80,15 +71,14 @@ module Librato
         end
       end
 
-      # access to internal counters object
-      def counters
-        @counter_cache ||= CounterCache.new
+      # access to client instance
+      def client
+        @client ||= prepare_client
       end
-
-      # remove any accumulated but unsent metrics
-      def delete_all
-        aggregate.delete_all
-        counters.delete_all
+      
+      # collector instance which is tracking all measurement additions
+      def collector
+        @collector ||= Collector.new
       end
 
       # send all current data to Metrics
@@ -101,12 +91,7 @@ module Librato
         logger.debug queue.queued
         queue.submit unless queue.empty?
       rescue Exception => error
-        logger.error "[librato-rails] submission failed permanently, worker exiting: #{error}"
-      end
-
-      def group(prefix)
-        group = Group.new(prefix)
-        yield group
+        logger.error "[librato-rails] submission failed permanently: #{error}"
       end
 
       def logger
@@ -119,16 +104,13 @@ module Librato
       end
 
       # run once during Rails startup sequence
-      def setup
+      def setup(app)
         check_config
-        # return unless self.email && self.api_key
+        return unless credentials_present?
         logger.info "[librato-rails] starting up with #{app_server}..."
         @pid = $$
-        if forking_server?
-          install_worker_check
-        else
-          start_worker # start immediately
-        end
+        app.middleware.use Librato::Rack::Middleware
+        start_worker unless forking_server?
       end
 
       def source
@@ -169,6 +151,10 @@ module Librato
           :other
         end
       end
+      
+      def credentials_present?
+        self.user && self.token
+      end
 
       def forking_server?
         FORKING_SERVERS.include?(app_server)