resque-meta 1.0.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Lee Marlow
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+Software), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+Resque Meta
+===========
+
+A [Resque][rq] plugin. Requires Resque 1.8.
+
+If you want to be able to add metadata for a job
+to track anything you want, extend it with this module.
+
+For example:
+
+    class MyJob
+      extend Resque::Jobs::Meta
+
+      def self.perform(meta_id, *args)
+        heavy_lifting
+      end
+    end
+
+    meta0 = MyJob.enqueue('stuff')
+    meta0.enqueued_at # => 'Wed May 19 13:42:41 -0600 2010'
+    meta0.meta_id # => '03c9e1a045ad012dd20500264a19273c'
+    meta0['foo'] = 'bar' # => 'bar'
+
+    # later
+    meta1 = MyJob.get_meta('03c9e1a045ad012dd20500264a19273c')
+    meta1.job_class # => MyJob
+    meta1.enqueued_at # => 'Wed May 19 13:42:41 -0600 2010'
+    meta1['foo'] # => 'bar'
+
+[rq]: http://github.com/defunkt/resque

data/Rakefile

@@ -0,0 +1,38 @@
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+def command?(command)
+  system("type #{command} > /dev/null")
+end
+
+#
+# Tests
+#
+
+task :default => :test
+
+if command? :turn
+  desc "Run tests"
+  task :test do
+    suffix = "-n #{ENV['TEST']}" if ENV['TEST']
+    sh "turn test/*_test.rb #{suffix}"
+  end
+else
+  Rake::TestTask.new do |t|
+    t.libs << 'lib'
+    t.pattern = 'test/**/*_test.rb'
+    t.verbose = false
+  end
+end
+
+#
+# Gems
+#
+
+begin
+  require 'mg'
+  MG.new("resque-meta.gemspec")
+rescue LoadError
+  warn "mg not available."
+  warn "Install it with: gem i mg"
+end

data/lib/resque/plugins/meta.rb

@@ -0,0 +1,101 @@
+require 'digest/sha1'
+require 'resque'
+require 'resque/plugins/meta/version'
+require 'resque/plugins/meta/metadata'
+
+module Resque
+  module Plugins
+    # If you want to be able to add metadata for a job
+    # to track anything you want, extend it with this module.
+    #
+    # For example:
+    #
+    #     class MyJob
+    #       extend Resque::Jobs::Meta
+    #
+    #       def self.perform(meta_id, *args)
+    #         heavy_lifting
+    #       end
+    #     end
+    #
+    #     meta0 = MyJob.enqueue('stuff')
+    #     meta0.enqueued_at # => 'Wed May 19 13:42:41 -0600 2010'
+    #     meta0.meta_id # => '03c9e1a045ad012dd20500264a19273c'
+    #     meta0['foo'] = 'bar' # => 'bar'
+    #
+    #     # later
+    #     meta1 = MyJob.get_meta('03c9e1a045ad012dd20500264a19273c')
+    #     meta1.job_class # => MyJob
+    #     meta1.enqueued_at # => 'Wed May 19 13:42:41 -0600 2010'
+    #     meta1['foo'] # => 'bar'
+    module Meta
+
+      # Override in your job to control the metadata id. It is
+      # passed the same arguments as `perform`, that is, your job's
+      # payload.
+      def meta_id(*args)
+        Digest::SHA1.hexdigest([ Time.now.to_i, self, args ].join)
+      end
+
+      # Override in your job to control the how many seconds a job's
+      # metadata will live after it finishes.  Defaults to 24 hours.
+      # Return nil or 0 to set them to never expire.
+      def expire_meta_in
+        24 * 60 * 60
+      end
+
+      # Enqueues a job in Resque and return the association metadata.
+      # The meta_id in the returned object can be used to fetch the
+      # metadata again in the future.
+      def enqueue(*args)
+        meta = Metadata.new({'meta_id' => meta_id(args), 'job_class' => self.to_s})
+        meta.save
+        Resque.enqueue(self, meta.meta_id, *args)
+        meta
+      end
+
+      def store_meta(meta)
+        key = "meta:#{meta.meta_id}"
+        json = Resque.encode(meta.data)
+        Resque.redis.set(key, json)
+        Resque.redis.expireat("resque:#{key}", meta.expire_at) if meta.expire_at > 0
+        meta
+      end
+
+      # Retrieve the metadata for a given job.  If you call this
+      # from a class that extends Meta, then the metadata will
+      # only be returned if the metadata for that id is for the
+      # same class.  Explicitly, calling Meta.get_meta(some_id)
+      # will return the metadata for a job of any type.
+      def get_meta(meta_id)
+        key = "meta:#{meta_id}"
+        if json = Resque.redis.get(key)
+          data = Resque.decode(json)
+          if self == Meta || self.to_s == data['job_class']
+            Metadata.new(data)
+          end
+        end
+      end
+      module_function :get_meta
+      public :get_meta
+
+      def before_perform_meta(meta_id, *args)
+        if meta = get_meta(meta_id)
+          meta.start!
+        end
+      end
+
+      def after_perform_meta(meta_id, *args)
+        if meta = get_meta(meta_id)
+          meta.finish!
+        end
+      end
+
+      def on_failure_meta(e, meta_id, *args)
+        if meta = get_meta(meta_id)
+          meta.fail!
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/meta/metadata.rb

@@ -0,0 +1,123 @@
+require 'time'
+
+module Resque
+  module Plugins
+    module Meta
+      class Metadata
+        attr_reader :job_class, :meta_id, :data, :enqueued_at, :expire_in
+
+        def initialize(data_hash)
+          data_hash['enqueued_at'] ||= to_time_format_str(Time.now)
+          @data = data_hash
+          @meta_id = data_hash['meta_id'].dup
+          @enqueued_at = from_time_format_str('enqueued_at')
+          @job_class = data_hash['job_class']
+          if @job_class.is_a?(String)
+            @job_class = Resque.constantize(data_hash['job_class'])
+          else
+            data_hash['job_class'] = @job_class.to_s
+          end
+          @expire_in = @job_class.expire_meta_in || 0
+        end
+
+        def reload!
+          if new_meta = job_class.get_meta(meta_id)
+            @data = new_meta.data
+          end
+          self
+        end
+
+        def save
+          job_class.store_meta(self)
+          self
+        end
+
+        def [](key)
+          data[key]
+        end
+
+        def []=(key, val)
+          data[key.to_s] = val
+        end
+
+        def start!
+          self['started_at'] = to_time_format_str(Time.now)
+          save
+        end
+
+        def started_at
+          from_time_format_str('started_at')
+        end
+
+        def finish!
+          data['succeeded'] = true unless data.has_key?('succeeded')
+          self['finished_at'] = to_time_format_str(Time.now)
+          save
+        end
+
+        def finished_at
+          from_time_format_str('finished_at')
+        end
+
+        def expire_at
+          if finished? && expire_in > 0
+            finished_at.to_i + expire_in
+          else
+            0
+          end
+        end
+
+        def enqueued?
+          started_at ? false : true
+        end
+
+        def working?
+          started_at && !finished_at ? true : false
+        end
+
+        def started?
+          started_at ? true :false
+        end
+
+        def finished?
+          finished_at ? true : false
+        end
+
+        def fail!
+          self['succeeded'] = false
+          finish!
+        end
+
+        def succeeded?
+          finished? ? self['succeeded'] : nil
+        end
+
+        def failed?
+          finished? ? !self['succeeded'] : nil
+        end
+
+        def seconds_enqueued
+          (started? ? started_at : Time.now.to_i) - enqueued_at.to_i
+        end
+
+        def seconds_processing
+          if started?
+            (finished? ? finished_at : Time.now.to_i) - started_at.to_i
+          else
+            0
+          end
+        end
+
+        protected
+
+        def from_time_format_str(key)
+          (t = self[key]) && Time.parse(t)
+        end
+
+        def to_time_format_str(time)
+          time.utc.iso8601(6)
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/meta/version.rb

@@ -0,0 +1,7 @@
+module Resque
+  module Plugins
+    module Meta
+      Version = '1.0.0'
+    end
+  end
+end

data/test/meta_test.rb

@@ -0,0 +1,155 @@
+require File.dirname(__FILE__) + '/test_helper'
+require 'resque/plugins/meta'
+
+class MetaJob
+  extend Resque::Plugins::Meta
+  @queue = :test
+
+  def self.expire_meta_in
+    1
+  end
+
+  def self.perform(meta_id, key, val)
+    meta = get_meta(meta_id)
+    meta[key] = val
+    meta.save
+  end
+end
+
+class AnotherJob
+  extend Resque::Plugins::Meta
+  @queue = :test
+
+  def self.perform(meta_id)
+  end
+end
+
+class SlowJob
+  extend Resque::Plugins::Meta
+  @queue = :test
+
+  def self.expire_meta_in
+    1
+  end
+
+  def self.perform(meta_id, key, val)
+    meta = get_meta(meta_id)
+    meta[key] = val
+    meta.save
+    sleep 1
+  end
+end
+
+class FailingJob
+  extend Resque::Plugins::Meta
+  @queue = :test
+
+  def self.perform(*args)
+    raise 'boom'
+  end
+end
+
+class MetaTest < Test::Unit::TestCase
+  def setup
+    Resque.redis.flushall
+  end
+
+  def test_meta_version
+    assert_equal '1.0.0', Resque::Plugins::Meta::Version
+  end
+
+  def test_lint
+    assert_nothing_raised do
+      Resque::Plugin.lint(Resque::Plugins::Meta)
+    end
+  end
+
+  def test_resque_version
+    major, minor, patch = Resque::Version.split('.')
+    assert_equal 1, major.to_i
+    assert minor.to_i >= 8
+  end
+
+  def test_enqueued_metadata
+    now = Time.now
+    meta = MetaJob.enqueue('foo', 'bar')
+    assert_not_nil meta
+    assert_not_nil meta.meta_id
+    assert meta.enqueued_at.to_f > now.to_f, "#{meta.enqueued_at} should be after #{now}"
+    assert meta.enqueued?
+    assert !meta.started?
+    assert !meta.finished?
+    assert_nil meta['foo']
+    assert_equal Resque::Plugins::Meta::Metadata, meta.class
+    assert_equal MetaJob, meta.job_class
+  end
+
+  def test_processed_job
+    meta = MetaJob.enqueue('foo', 'bar')
+    assert_nil meta['foo']
+    worker = Resque::Worker.new(:test)
+    worker.work(0)
+
+    meta = MetaJob.get_meta(meta.meta_id)
+    assert_equal MetaJob, meta.job_class
+    assert meta.started?
+    assert meta.finished?, 'Job should be finished'
+    assert meta.succeeded?, 'Job should have succeeded'
+    assert !meta.enqueued?
+    assert_equal 'bar', meta['foo'], "'foo' not found in #{meta.inspect}"
+  end
+
+  def test_wrong_id_for_class
+    meta = MetaJob.enqueue('foo', 'bar')
+
+    assert_nil AnotherJob.get_meta(meta.meta_id)
+    assert_not_nil Resque::Plugins::Meta.get_meta(meta.meta_id)
+  end
+
+  def test_expired_metadata
+    meta = MetaJob.enqueue('foo', 'bar')
+    worker = Resque::Worker.new(:test)
+    worker.work(0)
+
+    sleep 2
+    meta = MetaJob.get_meta(meta.meta_id)
+    assert_nil meta
+  end
+
+  def test_slow_job
+    meta = SlowJob.enqueue('foo', 'bar')
+    worker = Resque::Worker.new(:test)
+    thread = Thread.new { worker.work(0) }
+
+    sleep 0.1
+    meta = SlowJob.get_meta(meta.meta_id)
+    assert !meta.enqueued?
+    assert meta.started?
+    assert meta.working?
+    assert !meta.finished?
+
+    thread.join # job should be done
+    meta.reload!
+    assert !meta.enqueued?
+    assert meta.started?
+    assert !meta.working?
+    assert meta.finished?
+    assert meta.succeeded?
+    assert !meta.failed?
+
+    sleep 2
+    assert_nil Resque::Plugins::Meta.get_meta(meta.meta_id)
+  end
+
+  def test_failing_job
+    meta = FailingJob.enqueue()
+    assert_nil meta.failed?
+    worker = Resque::Worker.new(:test)
+    worker.work(0)
+
+    meta.reload!
+    assert meta.finished?
+    assert meta.failed?
+    assert !meta.succeeded?
+  end
+end

data/test/redis-test.conf

@@ -0,0 +1,132 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile ./test/redis-test.pid
+
+# Accept connections on the specified port, default is 6379
+port 9736
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir ./test/
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile stdout
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+
+# slaveof <masterip> <masterport>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+
+# maxmemory <bytes>
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes
+
+# Use object sharing. Can save a lot of memory if you have many common
+# string in your dataset, but performs lookups against the shared objects
+# pool so it uses more CPU and can be a bit slower. Usually it's a good
+# idea.
+#
+# When object sharing is enabled (shareobjects yes) you can use
+# shareobjectspoolsize to control the size of the pool used in order to try
+# object sharing. A bigger pool size will lead to better sharing capabilities.
+# In general you want this value to be at least the double of the number of
+# very common strings you have in your dataset.
+#
+# WARNING: object sharing is experimental, don't enable this feature
+# in production before of Redis 1.0-stable. Still please try this feature in
+# your development environment so that we can test it better.
+shareobjects no
+shareobjectspoolsize 1024

data/test/test_helper.rb

@@ -0,0 +1,34 @@
+dir = File.dirname(File.expand_path(__FILE__))
+$LOAD_PATH.unshift dir + '/../lib'
+require 'test/unit'
+require 'rubygems'
+require 'resque'
+
+#
+# make sure we can run redis
+#
+
+if !system("which redis-server")
+  puts '', "** can't find `redis-server` in your path"
+  puts "** try running `sudo rake install`"
+  abort ''
+end
+
+
+#
+# start our own redis when the tests start,
+# kill it when they end
+#
+
+at_exit do
+  next if $!
+
+  pid = `ps -A -o pid,command | grep [r]edis-test`.split(" ")[0]
+  puts "Killing test redis server..."
+  `rm -f #{dir}/dump.rdb`
+  Process.kill("KILL", pid.to_i)
+end
+
+puts "Starting redis for testing at localhost:9736..."
+`redis-server #{dir}/redis-test.conf`
+Resque.redis = 'localhost:9736'

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification 
+name: resque-meta
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Lee Marlow
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-06-04 00:00:00 -06:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 55
+        segments: 
+        - 1
+        - 8
+        - 0
+        version: 1.8.0
+  type: :runtime
+  version_requirements: *id001
+description: |
+  A Resque plugin.  If you want to be able to add metadata for a job
+  to track anything you want, extend it with this module.
+  
+  For example:
+  
+      class MyJob
+        extend Resque::Jobs::Meta
+  
+        def self.perform(meta_id, *args)
+          heavy_lifting
+        end
+      end
+  
+      meta0 = MyJob.enqueue('stuff')
+      meta0.enqueued_at # => 'Wed May 19 13:42:41 -0600 2010'
+      meta0.meta_id # => '03c9e1a045ad012dd20500264a19273c'
+      meta0['foo'] = 'bar' # => 'bar'
+  
+      # later
+      meta1 = MyJob.get_meta('03c9e1a045ad012dd20500264a19273c')
+      meta1.job_class # => MyJob
+      meta1.enqueued_at # => 'Wed May 19 13:42:41 -0600 2010'
+      meta1['foo'] # => 'bar'
+
+email: lee.marlow@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- Rakefile
+- LICENSE
+- lib/resque/plugins/meta/metadata.rb
+- lib/resque/plugins/meta/version.rb
+- lib/resque/plugins/meta.rb
+- test/meta_test.rb
+- test/redis-test.conf
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/lmarlow/resque-meta
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A Resque plugin for storing job metadata.
+test_files: []
+