vinted-resque-metrics 0.0.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 062fadf259abb6e87ffd46519d5195aa2acb7816
+  data.tar.gz: 1064e5f84426606cd0d89a252946d01665d69c3b
+SHA512:
+  metadata.gz: 4724dacbff95e8496ff18cf87e69d9978f660b2297af79248f4635883f97167c9aaf88119308a5f36dbc66ffa6d2e7513023d2512c6bb5e9b40912eb6ff65990
+  data.tar.gz: d87b7c305d024dbc259f503c8415d99412054ae9c44c6eead45a37d399634c5d90a6837e577019a7be4681f4babd800cfca7b742fc52f125f6bd067965fefb3a

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,17 @@
+source "http://rubygems.org"
+
+gem 'resque', '~>1.19'
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  platform :mri_19 do
+    gem "ruby-debug19"
+  end
+  gem 'minitest', '>=2'
+  gem "bundler"
+  gem "jeweler"
+end

data/Gemfile.lock

@@ -0,0 +1,81 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.3.5)
+    archive-tar-minitar (0.5.2)
+    builder (3.2.2)
+    columnize (0.3.4)
+    faraday (0.7.6)
+      addressable (~> 2.2)
+      multipart-post (~> 1.1)
+      rack (~> 1.1)
+    git (1.2.6)
+    github_api (0.4.10)
+      faraday (~> 0.7.6)
+      hashie (~> 1.2.0)
+      multi_json (~> 1.0)
+      oauth2 (~> 0.5.2)
+    hashie (1.2.0)
+    highline (1.6.20)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.1)
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
+    mini_portile (0.5.2)
+    minitest (2.8.0)
+    multi_json (1.0.3)
+    multipart-post (1.2.0)
+    nokogiri (1.6.1)
+      mini_portile (~> 0.5.0)
+    oauth2 (0.5.2)
+      faraday (~> 0.7)
+      multi_json (~> 1.0)
+    rack (1.3.5)
+    rack-protection (1.1.4)
+      rack
+    rake (10.1.1)
+    rdoc (4.1.1)
+      json (~> 1.4)
+    redis (2.2.2)
+    redis-namespace (1.0.3)
+      redis (< 3.0.0)
+    resque (1.19.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.0.2)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+    sinatra (1.3.1)
+      rack (~> 1.3, >= 1.3.4)
+      rack-protection (~> 1.1, >= 1.1.2)
+      tilt (~> 1.3, >= 1.3.3)
+    tilt (1.3.3)
+    vegas (0.1.8)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  jeweler
+  minitest (>= 2)
+  resque (~> 1.19)
+  ruby-debug19

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Aaron Quint
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,165 @@
+= resque-metrics
+
+A simple Resque plugin that times and saves some simple metrics for Resque jobs back into redis. Based on this system
+you could build some simple auto-scaling mechanism based on the speed and ETA of queues. Also includes a hook/callback
+mechanism for recording/sending the metrics to your favorite tool (AKA statsd/graphite).
+
+== Installation
+
+    gem install resque-metrics
+
+== Usage
+
+Given a job, extend the job class with Resque::Metrics.
+
+    class SomeJob
+      extend ::Resque::Metrics
+
+      @queue = :jobs
+
+      def self.perform(x, y)
+        # sleep 10
+      end
+
+    end
+
+By default this will record the total job count, the total count of jobs enqueued, the total time the jobs took, the avg time the jobs took. It will also record the total number of job failures. 
+These metrics are also tracked by queue and job class. So for the job above, it will record values and you will be able to fetch them
+with module methods:
+
+    Resque::Metrics.total_job_count #=> 1
+    Resque::Metrics.total_job_count_by_job(SomeJob) #=> 1
+    Resque::Metrics.total_job_count_by_queue(:jobs) #=> 10000
+    Resque::Metrics.total_job_time #=> 10000
+    Resque::Metrics.total_job_time_by_job(SomeJob) #=> 10000
+    Resque::Metrics.total_job_time_by_queue(:jobs) #=> 10000
+    Resque::Metrics.avg_job_time #=> 1000
+    Resque::Metrics.avg_job_time_by_job(SomeJob) #=> 1000
+    Resque::Metrics.avg_job_time_by_queue(:jobs) #=> 1000
+    Resque::Metrics.failed_job_count #=> 1
+    Resque::Metrics.failed_job_count_by_job(SomeJob) #=> 0
+    Resque::Metrics.failed_job_count_by_queue(:jobs) #=> 0
+
+All values are recorded and returned as integers. For times, values are in milliseconds.
+
+=== Forking Metrics
+
+Resque::Metrics can also record forking metrics but these are not on by default as `before_fork` and `after_fork` are singluar hooks. 
+If you don't need to define your own fork hooks you can simply add a line to an initializer:
+
+    Resque::Metrics.watch_fork
+
+If you do define you're own fork hooks:
+
+    Resque.before_fork do |job|
+      # my own fork code
+      Resque::Metrics.before_fork.call(job)
+    end
+    
+    # Resque::Metrics.(before/after)_fork just returns a lambda so just assign it if you like
+    Resque.after_fork = Resque::Metrics.after_fork
+
+Once enabled this will add `.*_fork_*` methods like `avg_fork_time`, etc. 
+Latest Resque is required for fork recording to work.
+
+=== Queue Depth Metrics
+
+Resque::Metrics can also record queue depth metrics. These are not on by default,
+as they need to run on an interval to be useful. You can record them manually
+by running in a console:
+
+    Resque::Metrics.record_depth
+
+You can imagine placing this in a small script, and using cron to run it. Once you'll have access to:
+
+    Resque::Metrics.failed_depth #=> 1
+    Resque::Metrics.pending_depth #=> 1
+    Resque::Metrics.depth_by_queue(:jobs) #=> 1
+
+=== Metric Backends
+
+By default, Resque::Metrics keeps all it's metrics in Resque's redis instance, but supports plugging in other backends. Resque::Metrics itself supports redis and statsd. Here's how you would enable statsd:
+
+    # list current backends
+    Resque::Metrics.backends
+    # build your statsd instance
+    statsd = Statsd.new 'localhost', 8125
+    # add a Resque::Metrics::Backend
+    Resque::Metrics.backends.append Resque::Metrics::Backends::Statsd.new(statsd)
+
+==== Statsd
+
+If you have already have a statsd object for you application, just pass it to Resque::Metrics::Backends::Statsd. The statsd client already supports namespacing, and in addition, Resque::Metrics all its metrics under 'resque' under that namespace.
+
+Here's a list of metrics emitted:
+
+  resque.job.<job>.complete.count
+  resque.job.<job>.complete.time
+  resque.queue.<queue>.complete.count
+  resque.queue.<queue>.complete.time
+  resque.complete.count
+  resque.complete.time
+
+  resque.job.<job>.enqueue.count
+  resque.job.<job>.enqueue.time
+  resque.queue.<queue>.enqueue.count
+  resque.queue.<queue>.enqueue.time
+  resque.enqueue.count
+  resque.enqueue.time
+
+  resque.job.<job>.fork.count
+  resque.job.<job>.fork.time
+  resque.queue.<queue>.fork.count
+  resque.queue.<queue>.fork.time
+  resque.fork.count
+  resque.fork.time
+
+  resque.job.<job>.failure.count
+  resque.queue.<queue>.failure.count
+  resque.failure.count
+
+  resque.depth.failed
+  resque.depth.pending
+  resque.depth.queue.<queue>
+
+==== Writing your own
+
+To write your own, you create your own class, and then implmement the following that you care about:
+
+* increment_metric(metric, by = 1)
+* set_metric(metric, val)
+* set_avg(metric, num, total)
+* get_metric(metric)
+
+Resque::Metrics will in turn call each of these methods for each of it's backend
+if it responds_to? it. For get_metric, since it returns a value, only will use
+the first backend that responds_to? it.
+
+=== Callbacks/Hooks
+
+Resque::Metrics also has a simple callback/hook system so you can send data to your favorite agent. All hooks are passed the job class,
+the queue, and the time of the metric. 
+
+    # Also `on_job_fork`, `on_job_enqueue`, and `on_job_failure` (`on_job_failure does not include `time`)
+    Resque::Metrics.on_job_complete do |job_class, queue, time|
+      # send to your metrics agent
+      Statsd.timing "resque.#{job_class}.complete_time", time
+      Statsd.increment "resque.#{job_class}.complete"
+      # etc
+    end
+    
+== Contributing to resque-metrics
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Aaron Quint. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "resque-metrics"
+  gem.homepage = "http://github.com/quirkey/resque-metrics"
+  gem.license = "MIT"
+  gem.summary = %Q{A Resque plugin for recording simple metrics for your jobs}
+  gem.description = <<-desc
+A simple Resque plugin that times and saves some simple metrics for Resque jobs back into redis. Based on this system
+you could build some simple auto-scaling mechanism based on the speed and ETA of queues. Also includes a hook/callback
+mechanism for recording/sending the metrics to your favorite tool (AKA statsd/graphite).
+desc
+  gem.email = "aaron@quirkey.com"
+  gem.authors = ["Aaron Quint"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+0.0.6

data/lib/resque-metrics.rb

@@ -0,0 +1 @@
+require 'resque/metrics'

data/lib/resque/metrics.rb

@@ -0,0 +1,360 @@
+require 'resque'
+require 'resque/metrics/backends'
+
+module Resque
+  module Metrics
+
+    def self.extended(klass)
+      klass.extend(Hooks)
+    end
+
+    def self.redis
+      @_redis ||= ::Resque.redis
+    end
+
+    def self.redis=(redis)
+      @_redis = redis
+    end
+
+    def self.use_multi=(multi)
+      @_use_multi = multi
+    end
+
+    def self.use_multi?
+      @_use_multi
+    end
+
+    def self.backends
+      @_backends ||= begin
+                       self.backends = [Resque::Metrics::Backends::Redis.new(redis)]
+                     end
+    end
+
+    def self.backends=(new_backends)
+      @_backends = new_backends
+    end
+
+    def self.run_backends(method, *args)
+      ran_any = false
+
+      backends.each do |backend|
+        if backend.respond_to?(method)
+          ran_any = true
+          backend.send method, *args
+        end
+      end
+
+      raise "No backend responded to #{method}: #{backends.inspect}" unless ran_any
+    end
+
+    def self.run_first_backend(method, *args)
+      backend = backends.detect {|backend| backend.respond_to?(method)}
+      raise "No backend responds to #{method}: #{backends.inspect}" unless backend
+
+      backend.send method, *args
+    end
+
+    def self.watch_fork
+      ::Resque.before_fork = before_fork
+      ::Resque.after_fork = after_fork
+    end
+
+    def self.on_job_fork(&block)
+      set_callback(:on_job_fork, &block)
+    end
+
+    def self.on_job_complete(&block)
+      set_callback(:on_job_complete, &block)
+    end
+
+    def self.on_job_enqueue(&block)
+      set_callback(:on_job_enqueue, &block)
+    end
+
+    def self.on_job_failure(&block)
+      set_callback(:on_job_failure, &block)
+    end
+
+    def self.set_callback(callback_name, &block)
+      @callbacks ||= {}
+      @callbacks[callback_name] ||= []
+      @callbacks[callback_name] << block
+    end
+
+    def self.run_callback(callback_name, *args)
+      if @callbacks && @callbacks[callback_name]
+        @callbacks[callback_name].each {|callback| callback.call(*args) }
+      end
+    end
+
+    def self.before_fork
+      lambda do |job|
+        start = Time.now.to_f * 1000
+        key = "_metrics_:fork_start:#{job.worker.to_s}"
+        ::Resque.redis.set key, start
+        ::Resque.redis.expire key, 60 * 60 * 60
+        true
+      end
+    end
+
+    def self.after_fork
+      lambda do |job|
+        end_time = Time.now.to_f * 1000
+        key = "_metrics_:fork_start:#{job.worker.to_s}"
+        start_time = ::Resque.redis.get key
+        if start_time
+          total = (end_time - start_time.to_f).to_i
+          ::Resque::Metrics.record_job_fork(job, total)
+        end
+        true
+      end
+    end
+
+    def self.record_depth
+      set_metric 'depth:failed', Resque::Failure.count
+      set_metric 'depth:pending', Resque.info[:pending]
+
+      Resque.queues.each do |queue|
+        set_metric "depth:queue:#{queue}", Resque.size(queue)
+      end
+
+      true
+    end
+
+    def self.record_job_fork(job, time)
+      job_class = job.payload_class
+      queue = job.queue
+      multi do
+        increment_metric "fork_time", time
+        increment_metric "fork_time:queue:#{queue}", time
+        increment_metric "fork_time:job:#{job_class}", time
+        increment_metric "fork_count"
+        increment_metric "fork_count:queue:#{queue}"
+        increment_metric "fork_count:job:#{job_class}"
+      end
+      set_avg "avg_fork_time", total_fork_time , total_fork_count
+      set_avg "avg_fork_time:queue:#{queue}", total_fork_time_by_queue(queue) , total_fork_count_by_queue(queue)
+      set_avg "avg_fork_time:job:#{job_class}", total_fork_time_by_job(job_class) , total_fork_count_by_job(job_class)
+      run_callback(:on_job_fork, job_class, queue, time)
+    end
+
+    def self.record_job_enqueue(job_class, *args)
+      queue = Resque.queue_from_class(job_class)
+      increment_metric "enqueue_count"
+      increment_metric "enqueue_count:job:#{job_class}"
+      increment_metric "enqueue_count:queue:#{queue}"
+
+      size = Resque.encode(args).length
+      multi do
+        increment_metric "payload_size", size
+        increment_metric "payload_size:queue:#{queue}", size
+        increment_metric "payload_size:job:#{job_class}", size
+      end
+      set_avg "avg_payload_size", total_payload_size , total_enqueue_count
+      set_avg "avg_payload_size:queue:#{queue}", total_payload_size_by_queue(queue) , total_enqueue_count_by_queue(queue)
+      set_avg "avg_payload_size:job:#{job_class}", total_payload_size_by_job(job_class) , total_enqueue_count_by_job(job_class)
+      run_callback(:on_job_enqueue, job_class, queue, size)
+      true
+    end
+
+    def self.record_job_completion(job_class, time)
+      queue = Resque.queue_from_class(job_class)
+      multi do
+        increment_metric "job_time", time
+        increment_metric "job_time:queue:#{queue}", time
+        increment_metric "job_time:job:#{job_class}", time
+        increment_metric "job_count"
+        increment_metric "job_count:queue:#{queue}"
+        increment_metric "job_count:job:#{job_class}"
+      end
+      set_avg "avg_job_time", total_job_time, total_job_count
+      set_avg "avg_job_time:queue:#{queue}", total_job_time_by_queue(queue) , total_job_count_by_queue(queue)
+      set_avg "avg_job_time:job:#{job_class}", total_job_time_by_job(job_class) , total_job_count_by_job(job_class)
+      run_callback(:on_job_complete, job_class, queue, time)
+    end
+
+    def self.record_job_failure(job_class, e)
+      queue = Resque.queue_from_class(job_class)
+      
+      multi do
+        increment_metric "failed_job_count"
+        increment_metric "failed_job_count:queue:#{queue}"
+        increment_metric "failed_job_count:job:#{job_class}"
+      end
+      
+      run_callback(:on_job_failure, job_class, queue)
+    end
+
+    def self.multi(&block)
+      use_multi? ? redis.multi(&block) : yield
+    end
+
+    def self.increment_metric(metric, by = 1)
+      run_backends(:increment_metric, metric, by)
+    end
+
+    def self.set_metric(metric, val)
+      run_backends(:set_metric, metric, val)
+    end
+
+    def self.set_avg(metric, num, total)
+      run_backends(:set_avg, metric, num, total)
+    end
+
+    def self.get_metric(metric)
+      run_first_backend(:get_metric, metric)
+    end
+
+    def self.total_enqueue_count
+      get_metric "enqueue_count"
+    end
+
+    def self.total_enqueue_count_by_job(job)
+      get_metric "enqueue_count:job:#{job}"
+    end
+
+    def self.total_enqueue_count_by_queue(queue)
+      get_metric "enqueue_count:queue:#{queue}"
+    end
+
+    def self.avg_job_time
+      get_metric "avg_job_time"
+    end
+
+    def self.avg_job_time_by_queue(queue)
+      get_metric "avg_job_time:queue:#{queue}"
+    end
+
+    def self.avg_job_time_by_job(job)
+      get_metric "avg_job_time:job:#{job}"
+    end
+
+    def self.total_job_time
+      get_metric "job_time"
+    end
+
+    def self.total_job_time_by_queue(queue)
+      get_metric "job_time:queue:#{queue}"
+    end
+
+    def self.total_job_time_by_job(job)
+      get_metric "job_time:job:#{job}"
+    end
+
+    def self.total_job_count
+      get_metric "job_count"
+    end
+
+    def self.total_job_count_by_queue(queue)
+      get_metric "job_count:queue:#{queue}"
+    end
+
+    def self.total_job_count_by_job(job)
+      get_metric "job_count:job:#{job}"
+    end
+
+    def self.failed_job_count
+      get_metric "failed_job_count"
+    end
+
+    def self.failed_job_count_by_queue(queue)
+      get_metric "failed_job_count:queue:#{queue}"
+    end
+
+    def self.failed_job_count_by_job(job)
+      get_metric "failed_job_count:job:#{job}"
+    end
+
+    def self.total_payload_size
+      get_metric "payload_size"
+    end
+
+    def self.total_payload_size_by_queue(queue)
+      get_metric "payload_size:queue:#{queue}"
+    end
+
+    def self.total_payload_size_by_job(job)
+      get_metric "payload_size:job:#{job}"
+    end
+
+    def self.avg_payload_size
+      get_metric "avg_payload_size"
+    end
+
+    def self.avg_payload_size_by_queue(queue)
+      get_metric "avg_payload_size:queue:#{queue}"
+    end
+
+    def self.avg_payload_size_by_job(job)
+      get_metric "avg_payload_size:job:#{job}"
+    end
+
+    def self.avg_fork_time
+      get_metric "avg_fork_time"
+    end
+
+    def self.avg_fork_time_by_queue(queue)
+      get_metric "avg_fork_time:queue:#{queue}"
+    end
+
+    def self.avg_fork_time_by_job(job)
+      get_metric "avg_fork_time:job:#{job}"
+    end
+
+    def self.total_fork_time
+      get_metric "fork_time"
+    end
+
+    def self.total_fork_time_by_queue(queue)
+      get_metric "fork_time:queue:#{queue}"
+    end
+
+    def self.total_fork_time_by_job(job)
+      get_metric "fork_time:job:#{job}"
+    end
+
+    def self.total_fork_count
+      get_metric "fork_count"
+    end
+
+    def self.total_fork_count_by_queue(queue)
+      get_metric "fork_count:queue:#{queue}"
+    end
+
+    def self.total_fork_count_by_job(job)
+      get_metric "fork_count:job:#{job}"
+    end
+
+    def self.failed_depth
+      get_metric "depth:failed"
+    end
+
+    def self.pending_depth
+      get_metric "depth:pending"
+    end
+
+    def self.depth_by_queue(queue)
+      get_metric "depth:queue:#{queue}"
+    end
+
+    module Hooks
+
+      def after_enqueue_metrics(*args)
+        Resque::Metrics.record_job_enqueue(self, *args)
+      end
+
+      def around_perform_metrics(*args)
+        start = Time.now
+        yield
+        finish = ((Time.now.to_f - start.to_f) * 1000).to_i
+        Resque::Metrics.record_job_completion(self, finish)
+      end
+
+      def on_failure_metrics(e, *args)
+        Resque::Metrics.record_job_failure(self, e)
+      end
+
+    end
+
+  end
+end

data/lib/resque/metrics/backends.rb

@@ -0,0 +1,2 @@
+require 'resque/metrics/backends/redis'
+require 'resque/metrics/backends/statsd'

data/lib/resque/metrics/backends/redis.rb

@@ -0,0 +1,31 @@
+module Resque
+  module Metrics
+    module Backends
+      class Redis
+        attr_accessor :redis
+
+        def initialize(redis)
+          @redis = redis
+        end
+
+        def increment_metric(metric, by = 1)
+          redis.incrby("_metrics_:#{metric}", by)
+        end
+
+        def set_metric(metric, val)
+          redis.set("_metrics_:#{metric}", val)
+        end
+
+        def set_avg(metric, num, total)
+          val = total < 1 ? 0 : num / total
+          set_metric(metric, val)
+        end
+
+        def get_metric(metric)
+          redis.get("_metrics_:#{metric}").to_i
+        end
+
+      end
+    end
+  end
+end

data/lib/resque/metrics/backends/statsd.rb

@@ -0,0 +1,66 @@
+module Resque
+  module Metrics
+    module Backends
+      class Statsd
+        attr_accessor :statsd, :metric_prefix
+
+        def initialize(statsd, metric_prefix = 'resque')
+          @statsd = statsd
+          @metric_prefix = metric_prefix
+        end
+
+        def increment_metric(metric, by = 1)
+          if metric =~ /^(.+)(?:_job)?_(time|count)(?::(queue|job):(.*))?$/
+            event = $1
+            event = 'complete' if event == 'job'
+
+            time_or_count = $2
+            queue_or_job = $3
+            queue_or_job_name = $4
+            key = if queue_or_job && queue_or_job_name
+                    # ie resque.complete.queue.high.count, resque.failed.job.Index.timing
+                    "#{metric_prefix}.#{event}.#{queue_or_job}.#{queue_or_job_name}.#{time_or_count}"
+                  else
+
+                    # ie resque.complete.time
+                    "#{metric_prefix}.#{event}.#{time_or_count}"
+                  end
+            case time_or_count
+            when 'time'
+              statsd.timing key, by
+            when 'count'
+              statsd.increment key, by
+            else
+              raise "Not sure how to increment_metric for a #{time_or_count} metric (#{metric})"
+            end
+          elsif metric =~ /^payload_size(?::(queue|job):(.*))?$/
+            queue_or_job = $1
+            queue_or_job_name = $2
+            key = if queue_or_job && queue_or_job_name
+                    # ie resque.complete.queue.high.count, resque.failed.job.Index.timing
+                    "#{metric_prefix}.payload_size.#{queue_or_job}.#{queue_or_job_name}.#{time_or_count}"
+                  else
+                    "#{metric_prefix}.payload_size.#{time_or_count}"
+                  end
+            statsd.increment key, by
+          else
+            raise "Not sure how to increment_metric #{metric}"
+          end
+        end
+
+        def set_metric(metric, val)
+          if metric =~ /^depth(?::(failed|pending|queue)(?::(.+))?)?$/
+            key = "#{metric_prefix}.#{metric.gsub(':', '.')}"
+            statsd.gauge key, val
+          else
+            raise "Not sure how to set_metric #{metric}"
+          end
+        end
+        
+        # set_avg: let statsd & graphite handle that
+        # get_metric: would have to talk to graphite. but man, complicated
+      end
+    end
+  end
+end
+

data/resque-metrics.gemspec

@@ -0,0 +1,67 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: resque-metrics 0.0.6 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "vinted-resque-metrics"
+  s.version = "0.0.7"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["Aaron Quint", "Tomas Varaneckas"]
+  s.date = "2014-02-12"
+  s.description = "A simple Resque plugin that times and saves some simple metrics for Resque jobs back into redis. Based on this system\nyou could build some simple auto-scaling mechanism based on the speed and ETA of queues. Also includes a hook/callback\nmechanism for recording/sending the metrics to your favorite tool (AKA statsd/graphite).\n"
+  s.email = "aaron@quirkey.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "lib/resque-metrics.rb",
+    "lib/resque/metrics.rb",
+    "lib/resque/metrics/backends.rb",
+    "lib/resque/metrics/backends/redis.rb",
+    "lib/resque/metrics/backends/statsd.rb",
+    "resque-metrics.gemspec",
+    "test/helper.rb",
+    "test/redis-test.conf",
+    "test/test_resque-metrics.rb"
+  ]
+  s.homepage = "http://github.com/quirkey/resque-metrics"
+  s.licenses = ["MIT"]
+  s.rubygems_version = "2.2.1"
+  s.summary = "A Resque plugin for recording simple metrics for your jobs"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<resque>, ["~> 1.19"])
+      s.add_development_dependency(%q<ruby-debug19>, [">= 0"])
+      s.add_development_dependency(%q<minitest>, [">= 2"])
+      s.add_development_dependency(%q<bundler>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+    else
+      s.add_dependency(%q<resque>, ["~> 1.19"])
+      s.add_dependency(%q<ruby-debug19>, [">= 0"])
+      s.add_dependency(%q<minitest>, [">= 2"])
+      s.add_dependency(%q<bundler>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<resque>, ["~> 1.19"])
+    s.add_dependency(%q<ruby-debug19>, [">= 0"])
+    s.add_dependency(%q<minitest>, [">= 2"])
+    s.add_dependency(%q<bundler>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+  end
+end

data/test/helper.rb

@@ -0,0 +1,79 @@
+dir = File.dirname(File.expand_path(__FILE__))
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'minitest/autorun'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$TESTING = true
+
+require 'resque/metrics'
+
+#
+# make sure we can run redis
+#
+
+if ENV['BOXEN_REDIS_PORT']
+  redis_port = ENV['BOXEN_REDIS_PORT']
+else
+  if !system("which redis-server")
+    puts '', "** can't find `redis-server` in your path"
+    puts "** try running `sudo rake install`"
+    abort ''
+  end
+
+  #
+  # start our own redis when the tests start,
+  # kill it when they end
+  #
+
+  at_exit do
+    next if $!
+
+    if defined?(MiniTest)
+      exit_code = MiniTest::Unit.new.run(ARGV)
+    else
+      exit_code = Test::Unit::AutoRunner.run
+    end
+
+    pid = `ps -e -o pid,command | grep [r]edis-test`.split(" ")[0]
+    puts "Killing test redis server..."
+    Process.kill("KILL", pid.to_i)
+    FileUtils.rm_rf("#{dir}/dump.rdb")
+    exit exit_code
+  end
+
+  puts "Starting redis for testing at localhost:9736..."
+  `redis-server #{dir}/redis-test.conf`
+
+  redis_port = 9736
+end
+Resque.redis = "localhost:#{redis_port}:2"
+
+class SomeJob
+  extend Resque::Metrics
+
+  @queue = :jobs
+
+  def self.perform(x, y)
+    sleep rand * 0.01
+  end
+
+end
+
+class FailureJob
+  extend Resque::Metrics
+
+  @queue = :jobs
+
+  def self.perform
+    raise "failing lol"
+  end
+end

data/test/redis-test.conf

@@ -0,0 +1,130 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile ./test/redis-test.pid
+
+# Accept connections on the specified port, default is 6379
+port 9736
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir ./test/
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile stdout
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+
+# slaveof <masterip> <masterport>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+
+# maxmemory <bytes>
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes
+
+# Use object sharing. Can save a lot of memory if you have many common
+# string in your dataset, but performs lookups against the shared objects
+# pool so it uses more CPU and can be a bit slower. Usually it's a good
+# idea.
+#
+# When object sharing is enabled (shareobjects yes) you can use
+# shareobjectspoolsize to control the size of the pool used in order to try
+# object sharing. A bigger pool size will lead to better sharing capabilities.
+# In general you want this value to be at least the double of the number of
+# very common strings you have in your dataset.
+#
+# WARNING: object sharing is experimental, don't enable this feature
+# in production before of Redis 1.0-stable. Still please try this feature in
+# your development environment so that we can test it better.

data/test/test_resque-metrics.rb

@@ -0,0 +1,120 @@
+require 'helper'
+
+class TestResqueMetrics < MiniTest::Unit::TestCase
+  def setup
+    Resque.redis.flushall
+
+    Resque.before_first_fork = nil
+    Resque.before_fork = nil
+    Resque.after_fork = nil
+
+    @num_jobs = 4
+    @worker = Resque::Worker.new(:jobs)
+    @num_jobs.times do
+      work_job
+    end
+
+    @num_failed_jobs = 2
+    @num_failed_jobs.times do 
+      fail_job
+    end
+  end
+
+  def test_should_pass_resque_plugin_lint
+    assert Resque::Plugin.lint(Resque::Metrics::Hooks)
+  end
+
+  def test_should_perform_job
+    assert Resque::Job.new(:jobs, 'class' => SomeJob, 'args' => [1,2]).perform
+  end
+
+  def test_should_record_total_job_time
+    assert Resque::Metrics.total_job_time > 0
+    assert Resque::Metrics.total_job_time_by_queue(:jobs) > 0
+    assert Resque::Metrics.total_job_time_by_job(SomeJob) > 0
+  end
+
+  def test_should_record_enqueue_count
+    assert_equal @num_jobs + @num_failed_jobs, Resque::Metrics.total_enqueue_count
+    assert_equal @num_jobs + @num_failed_jobs, Resque::Metrics.total_enqueue_count_by_queue(:jobs)
+    assert_equal @num_jobs, Resque::Metrics.total_enqueue_count_by_job(SomeJob)
+    assert_equal @num_failed_jobs, Resque::Metrics.total_enqueue_count_by_job(FailureJob)
+  end
+
+  def test_should_record_job_count
+    assert Resque::Metrics.total_job_count > 0
+    assert Resque::Metrics.total_job_count_by_queue(:jobs) > 0
+    assert Resque::Metrics.total_job_count_by_job(SomeJob) > 0
+  end
+
+  def test_should_record_failed_job_count
+    assert Resque::Metrics.failed_job_count > 0, "Expected #{Resque::Metrics.failed_job_count} to be > 0, but wasn't"
+    assert Resque::Metrics.failed_job_count_by_queue(:jobs) > 0
+    assert Resque::Metrics.failed_job_count_by_job(FailureJob) > 0
+  end
+
+  def test_should_record_payload_size
+    assert Resque::Metrics.total_payload_size > 0
+    assert Resque::Metrics.total_payload_size_by_queue(:jobs) > 0
+    assert Resque::Metrics.total_payload_size_by_job(SomeJob) > 0
+  end
+
+  def test_should_record_avg_job_time
+    assert Resque::Metrics.avg_job_time > 0
+    assert Resque::Metrics.avg_job_time_by_queue(:jobs) > 0
+    assert Resque::Metrics.avg_job_time_by_job(SomeJob) > 0
+  end
+
+  def test_should_call_job_complete_callbacks
+    recorded = []
+    recorded_count = 0
+    Resque::Metrics.on_job_complete do |klass, queue, time|
+      recorded << {:klass => klass, :queue => queue, :time => time }
+    end
+    Resque::Metrics.on_job_complete do |klass, queue, time|
+      recorded_count += 1
+    end
+
+    work_job
+    work_job
+
+    assert_equal 2, recorded.length
+    assert_equal SomeJob, recorded[0][:klass]
+    assert_equal :jobs, recorded[0][:queue]
+    assert recorded[0][:time] > 0, "Expected #{recorded[0][:time]} to be > 0, but wasn't"
+    assert_equal 2, recorded_count
+  end
+
+  def test_should_call_job_failure_callbacks
+    recorded = []
+    recorded_count = 0
+    Resque::Metrics.on_job_failure do |klass, queue|
+      recorded << {:klass => klass, :queue => queue}
+    end
+    Resque::Metrics.on_job_failure do |klass, queue|
+      recorded_count += 1
+    end
+
+    fail_job
+    fail_job
+    fail_job
+
+    assert_equal 3, recorded.length
+    assert_equal FailureJob, recorded[0][:klass]
+    assert_equal :jobs, recorded[0][:queue]
+    assert_equal 3, recorded_count
+  end
+
+  private
+
+  def work_job
+    Resque.enqueue(SomeJob, 20, '/tmp')
+    @worker.work(0)
+  end
+
+  def fail_job
+    Resque.enqueue(FailureJob)
+    @worker.work(0)
+  end
+
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: vinted-resque-metrics
+version: !ruby/object:Gem::Version
+  version: 0.0.7
+platform: ruby
+authors:
+- Aaron Quint
+- Tomas Varaneckas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  A simple Resque plugin that times and saves some simple metrics for Resque jobs back into redis. Based on this system
+  you could build some simple auto-scaling mechanism based on the speed and ETA of queues. Also includes a hook/callback
+  mechanism for recording/sending the metrics to your favorite tool (AKA statsd/graphite).
+email: aaron@quirkey.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- ".document"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/resque-metrics.rb
+- lib/resque/metrics.rb
+- lib/resque/metrics/backends.rb
+- lib/resque/metrics/backends/redis.rb
+- lib/resque/metrics/backends/statsd.rb
+- resque-metrics.gemspec
+- test/helper.rb
+- test/redis-test.conf
+- test/test_resque-metrics.rb
+homepage: http://github.com/quirkey/resque-metrics
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A Resque plugin for recording simple metrics for your jobs
+test_files: []