redis_safe_queue 0.0.1

data/README.md

@@ -0,0 +1,105 @@
+redis_safe_queue
+================
+
+RedisSafeQueue is a transactional queue for ruby/redis. It guarantees at least once 
+semantics.
+
+The queue may be used with multiple producers and consumers, each job is removed
+in a open/commit transaction; even if a worker dies while processing a job, it is
+automatically requeued after the specified `timeout`.
+
+You can create multiple instances of the RedisSafeQueue class with the same 
+`queue_id` (i.e. on multiple machines), they will share the same queue and 
+synchronize.
+
+Each queue provides: 
+
++ A `work` method that processes jobs in a loop and returns when the queue is empty.
+You may provide a maximum number of jobs as an integer argument and/or invoke the 
+`work` method in a loop to run a continuous queue.
+
++ A `finish` method that blocks until the queue is empty and then yields only once. 
+When multiple instances of the RedisSafeQueue class with the same `queue_id` exist 
+(i.e. on multiple machines) only one instance will yield and the others will return 
+without doing anything.
+
+
+### Example
+
+```ruby
+require "rubygems"
+require "redis_safe_queue"
+
+$processed = 0
+
+# create a new queue 'my_queue_1' with a 1-minute timeout, if a worker 
+# exceeds the timeout while processing a job it is implicitly requeued
+my_queue = RedisSafeQueue.new(
+  :redis => Redis.new,
+  :queue_id => "my_queue_1",
+  :timeout => 60
+)
+
+# push 10.000 items to the queue
+10_000.times do |i|
+  my_queue.push("fnord-#{rand(123)}")
+end
+
+# start 10 workers/threads
+10.times { Thread.new {
+
+  # this will process jobs in parallel until the queue is empty
+  my_queue.work do |job|
+    $processed += 1
+  end
+
+  # this will yield only once, on one of the 10 threads
+  my_queue.finish do
+    puts "queue finished, processed #{$processed} jobs on 10 threads"
+    exit 0
+  end
+
+} }
+```
+
+
+Limitations
+-----------
+
++ A job may be processed more than once if a worker takes longer than `timeout` to process a job (the transaction is only commited after the block passed to `work` returns)
++ A job may be processed more than once if a redis network roundtrip takes longer than `TX_COMMIT_TIMEOUT` (default: 10 seconds) during the `start_tx` procedure
++ Since queue appends are not synchronized there is a known race condition that occurs when producing and consuming from the same queue (same `queue_id`) at the same time; in this case the `work` method may return even though there are still jobs left in the queue. This can be mitigated by calling the `work` method in a loop and never invoking `finish`.
++ In a distributed setup the system clocks must be in sync on all machines since redis_safe_queue implements a timestamp-based lock expiration mechanism
+
+
+
+Installation
+------------
+
+    gem install redis_safe_queue
+
+or in your Gemfile:
+
+    gem 'redis_safe_queue', '~> 0.1'
+
+
+License
+-------
+
+Copyright (c) 2011 Paul Asmuth
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to use, copy and modify copies of the Software, subject 
+to the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/example.rb

@@ -0,0 +1,35 @@
+$: << ::File.expand_path("../lib", __FILE__)
+
+require "rubygems"
+require "redis_safe_queue"
+
+QUEUE = RedisSafeQueue.new(
+  :redis => Redis.new,
+  :redis_prefix => "fnordbar",
+  :queue_id => rand(8**16).to_s(36),
+  :timeout => 60
+)
+
+10_000.times do |i|
+  QUEUE.push("fnord-#{rand(123)}")
+end
+
+$processed = 0
+
+10.times { Thread.new {
+
+  QUEUE.work do |job|
+    $processed += 1
+  end
+
+  QUEUE.finish do
+    puts "queue finished, processed #{$processed} jobs on 10 threads"
+    exit 0
+  end
+
+} }
+
+loop do
+  puts "size: #{QUEUE.size}, processed: #{$processed}, iterations: #{QUEUE.iterations}"
+  sleep 1
+end

data/lib/redis_safe_queue.rb

@@ -0,0 +1,97 @@
+require "rubygems"
+require "redis"
+
+class RedisSafeQueue
+
+  # redis must process all commands within 10s
+  TX_COMMIT_TIMEOUT = 10
+
+  attr_accessor :timeout, :queue_id, :prefix, :redis, :iterations
+
+  def initialize(opts)
+    @redis = opts.fetch(:redis)
+    @prefix = opts[:redis_prefix] || :redis_safe_queue
+    @queue_id = opts.fetch(:queue_id)
+    @timeout = opts.fetch(:timeout).to_i
+    @iterations = 0
+  end
+
+  def push(job_data)
+    job_id = unique_id
+    @redis.set(key(job_id, :job), job_data)
+    @redis.sadd(key, job_id)
+  end
+
+  def work(max_jobs = nil)
+    loop do
+      break if max_jobs == 0
+      if job_id = start_tx
+        job = get_job(job_id)
+        yield(job)
+        commit_tx(job_id)
+        max_jobs -= 1 if max_jobs
+      else
+        break
+      end
+    end
+  end
+
+  def finish
+    sleep 1 while size > 0
+
+    lock = @redis.setnx(key(:finished), 1)
+    yield if lock
+
+    @redis.expire(key, @timeout)
+    @redis.expire(key(:finished), @timeout)
+  end
+
+  def size
+    @redis.scard(key)
+  end
+
+private
+
+  def start_tx
+    job_id = nil
+
+    loop do
+      @iterations +=1
+
+      candidate = @redis.srandmember(key)
+      return unless candidate
+
+      if lock = @redis.setnx(key(candidate, :lock), Time.now.to_i)
+        @redis.expire(key(candidate, :lock), @timeout)
+        return candidate
+      else
+        lock_time = @redis.get(key(candidate, :lock)).to_i
+
+        if (Time.now.to_i - lock_time) > @timeout
+          @redis.del(key(candidate, :lock))
+        end
+      end
+    end
+  end
+
+  def commit_tx(job_id)
+    @redis.pipelined do
+      @redis.expire(key(job_id, :lock), TX_COMMIT_TIMEOUT)
+      @redis.srem(key, job_id)
+      @redis.del(key(job_id, :job))
+    end
+  end
+
+  def unique_id
+    rand(8**32).to_s(36)
+  end
+
+  def get_job(job_id)
+    @redis.get(key(job_id, :job))
+  end
+
+  def key(*append)
+    [@prefix, @queue_id, append].flatten.compact * ":"
+  end
+
+end

data/redis_safe_queue.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "redis_safe_queue"
+  s.version     = "0.0.1"
+  s.date        = Date.today.to_s
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Paul Asmuth"]
+  s.email       = ["paul@paulasmuth.com"]
+  s.homepage    = "http://github.com/paulasmuth/redis_safe_queue"
+  s.summary     = %q{transactional queue for redis (at least once semantics)}
+  s.description = %q{RedisSafeQueue is a transactional queue for ruby/redis. It guarantees at least once semantics. The queue may be used with multiple producers and consumers, each job is removed in a open/commit transaction; even if a worker dies while processing a job, it is automatically requeued.}
+  s.licenses    = ["MIT"]
+
+  s.add_dependency "redis", ">= 2.2.2"
+
+  s.files         = `git ls-files`.split("\n") - [".gitignore", ".rspec", ".travis.yml"]
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: redis_safe_queue
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Paul Asmuth
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &6216580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *6216580
+description: RedisSafeQueue is a transactional queue for ruby/redis. It guarantees
+  at least once semantics. The queue may be used with multiple producers and consumers,
+  each job is removed in a open/commit transaction; even if a worker dies while processing
+  a job, it is automatically requeued.
+email:
+- paul@paulasmuth.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- example.rb
+- lib/redis_safe_queue.rb
+- redis_safe_queue.gemspec
+homepage: http://github.com/paulasmuth/redis_safe_queue
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.17
+signing_key: 
+specification_version: 3
+summary: transactional queue for redis (at least once semantics)
+test_files: []