resque_redis_composite 1.0.2

data/.gitignore

@@ -0,0 +1,2 @@
+*.gem
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,63 @@
+resque_redis_composite
+======================
+
+This gems allows Resque to handle queues located on seperate Redis servers.
+
+Requirements
+------------
+
+Currently working with Resque 1.23.x and 1.24.x
+I haven't looked into Resque 2 yet, but it might be possible to natively handle multiple Redis instances with the new `Resque::Backend` class.
+
+Install & Setup
+---------------
+
+Configure Resque in your application:
+
+```ruby
+Resque.after_fork do |job|
+  Resque::RedisComposite.reconnect_all(job)
+end
+```
+
+Usage
+-----
+
+```ruby
+config = {
+  "default" => "localhost:6379",
+  "some_other_queue" => "otherbox:6379"
+}
+
+Resque.redis = Resque::RedisComposite.create(config)
+```
+
+Alternatively, `config` can be one of :
+
+  * a single connection string, it will become the default connection:
+    
+    `config = "localhost:6379"`
+
+  * a hash with any combination of server values supported by Resque:
+    * a Redis connection url : `redis://...`
+    * a Redis client : `Redis.new(:host => ..., :port => ...)`
+    * a Redis namespace : `Redis::Namespace(..., :redis => ...)`
+
+Notes
+-----
+
+Resque Stats will always be stored on the `default` Redis server.
+
+TODO
+----
+
+  * Keep an eye out for Resque 2.0 and evalute if this gems is still going to be useful
+  * Review specs
+
+Authors
+-------
+
+Original work done by [Dave Hoover](https://github.com/redsquirrel) and [jiHyunBae](https://github.com/jiHyunBae).
+
+Updated and made into a gem by [Anthony Powles](https://github.com/yogin).
+

data/lib/resque/redis_composite.rb

@@ -0,0 +1,116 @@
+module Resque
+  class RedisComposite
+    class NoDefaultRedisServerError < StandardError; end
+
+    DEFAULT_SERVER_NAME = 'default'
+
+    attr_reader :mapping
+
+    class << self
+
+      def create(config)
+        Redis::Namespace.new(nil, :redis => RedisComposite.new(config))
+      end
+
+      def reconnect_all(job = nil)
+        return unless Resque.redis.redis.kind_of?(Resque::RedisComposite)
+        Resque.redis.mapping.each { |_, server| server.redis.client.reconnect }
+      end
+
+    end
+
+    def initialize(config)
+      config = { DEFAULT_SERVER_NAME => config } unless config.kind_of?(Hash)
+
+      config = HashWithIndifferentAccess.new(config)
+      raise NoDefaultRedisServerError, "No default server defined in configuration : #{config.inspect}" unless config.key?(DEFAULT_SERVER_NAME)
+
+      # quick backup for what's in Resque
+      original_resque_server = Resque.redis
+
+      @mapping = config.inject(HashWithIndifferentAccess.new) do |hash, (queue_name, server)|
+        # leaving Resque create Redis::Namespace instances
+        Resque.redis = server
+
+        hash[queue_name] = Resque.redis
+        hash
+      end
+
+      Resque.redis = original_resque_server
+    end
+
+    def method_missing(method_name, *args, &block)
+      # most redis command's first parameter is the key, so it should work most of the time
+      server_for(args.first).send(method_name, *args, &block)
+    end
+
+    def client(queue = DEFAULT_SERVER_NAME)
+      # TODO properly handle Resque.redis_id, probably need to return all our client ids
+      server_for(queue).client
+    end
+
+    # This is used to create a set of queue names, so needs some special treatment
+    def sadd(key, value)
+      if queues?(key)
+        server_for(value).sadd(key, value)
+      else
+        server_for(key).sadd(key, value)
+      end
+    end
+
+    # If we're using smembers to get queue names, we aggregate across all servers
+    def smembers(key)
+      if queues?(key)
+        servers.inject([]) { |a, s| a + s.smembers(key) }.uniq
+      else
+        server_for(key).smembers(key)
+      end
+    end
+
+    # This is used to completely delete a queue, so needs some special treatment
+    def srem(key, value)
+      if queues?(key)
+        server_for(value).srem(key, value)
+      else
+        server_for(key).srem(key, value)
+      end
+    end
+
+    # Sometimes we're pushing onto the 'failed' queue, and we want to make sure
+    # the failures are pushed into the same Redis server as the queue is hosted on.
+    def rpush(key, value)
+      if failed?(key)
+        queue_with_failure = Resque.decode(value)["queue"]
+        server_for(queue_with_failure).rpush(key, value)
+      else
+        server_for(key).rpush(key, value)
+      end
+    end
+
+    protected
+
+    def servers
+      @mapping.values
+    end
+
+    def default_server
+      @mapping[DEFAULT_SERVER_NAME]
+    end
+
+    def server_for(queue)
+      # queue_name = queue.to_s.sub(/^queue:/, "")
+      # queue parsing : not match regular expression
+      queue_name = queue.to_s.sub(/[\W\w]*queue:/,"")
+      @mapping[queue_name] || default_server
+    end
+
+    def queues?(key)
+      key.to_s == "queues"
+    end
+
+    def failed?(key)
+      key.to_s == "failed"
+    end
+
+  end
+end

data/lib/resque_redis_composite.rb

@@ -0,0 +1,3 @@
+require 'resque'
+require 'resque/redis_composite'
+

data/resque_redis_composite.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s|
+  s.name        = "resque_redis_composite"
+  s.description = "Allowing Resque to handle queues accross multiple Redis instances"
+  s.version     = "1.0.2"
+  s.authors     = [ "redsquirrel", "jiHyunBae", "Anthony Powles" ]
+  s.email       = "rubygems+resque_redis_composite@idreamz.net"
+  s.summary     = "Resque support for multiple Redis instances"
+  s.homepage    = "https://github.com/yogin/resque_redis_composite"
+  s.files       = `git ls-files`.split($/)
+
+  s.add_development_dependency "bundler", "~> 1.3"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "pry"
+
+  s.add_dependency "resque"
+end

data/spec/resque/redis_composite_spec.rb

@@ -0,0 +1,118 @@
+require 'rubygems'
+require 'rspec'
+require 'resque'
+
+$:.unshift "#{File.dirname(__FILE__)}/../lib"
+require 'resque/redis_composite'
+
+describe Resque::RedisComposite do
+  describe "initialize" do
+
+    it "accepts a redis client" do
+      Resque.redis = Redis.new(:host => 'localhost', :port => 5380)
+      Resque.redis.should be_a(Redis::Namespace)
+      Resque.redis.client.port.should == 5380
+    end
+
+    it "accepts redis composite with a connection string" do
+      Resque.redis = Resque::RedisComposite.new("localhost:5380")
+      Resque.redis.client.port.should == 5380
+    end
+  
+    it "accepts redis composite with a hash of connection strings" do
+      Resque.redis = Resque::RedisComposite.new("default" => "localhost:5380", "test" => "localhost:5381")
+      Resque.redis.client.port.should == 5380
+      Resque.redis.client("test").port.should == 5381
+    end
+  
+    it "raises exception Resque::RedisComposite::MissingDefaultRedisInstance if there are no default instance" do
+      expect {
+        Resque::RedisComposite.new("foo" => "localhost:5380", "bar" => "localhost:5381")
+      }.to raise_error(Resque::RedisComposite::MissingDefaultRedisInstance)
+    end    
+  end
+  
+  describe "redis interface" do
+#    before do
+      #let(:default_instance) { mock("default") }
+      #let(:special_instance) { mock("special") }
+    #end
+
+    #it "properly namespaces the resque queues" do
+      #special_instance.should_receive(:sadd).with("resque:queues", "stuff")
+      #special_instance.should_receive(:rpush).with("resque:queue:stuff", '{"class":"SomeClass","args":"do something"}')
+
+      #Resque.redis = Resque::RedisComposite.new("default" => default_instance, "stuff" => special_instance)
+      #Resque.push("stuff", :class => "SomeClass", :args => "do something")
+    #end
+    
+    #it "delegates most calls to the default server" do
+      #@default_server.should_receive(:srem)
+      #@default_server.should_receive(:get)
+
+      #redis = Resque::RedisComposite.new("default" => @default_server, "stuff" => @special_server)
+      #redis.srem(:workers, self)
+      #redis.get(:workers)
+    #end
+
+    #describe "Resque.push" do
+      #it "pushes jobs onto the specific Redis server" do
+        #@special_server.should_receive(:sadd)
+        #@special_server.should_receive(:rpush)
+
+        #Resque.redis = Resque::RedisComposite.new("default" => @default_server, "stuff" => @special_server)
+        #Resque.push("stuff", :class => "SomeClass", :args => "do something")
+      #end
+
+      #it "pushes jobs onto the default Redis server" do
+        #@default_server.should_receive(:sadd)
+        #@default_server.should_receive(:rpush)
+
+        #Resque.redis = Resque::RedisComposite.new("default" => @default_server, "stuff" => @special_server)
+        #Resque.push("email", :class => "SomeOtherClass", :args => "do something else")
+      #end
+    #end
+
+    #describe "Resque.pop" do
+      #it "pops jobs off the specific Redis server" do
+        #@special_server.should_receive(:lpop)
+
+        #Resque.redis = Resque::RedisComposite.new("default" => @default_server, "stuff" => @special_server)
+        #Resque.pop("stuff")
+      #end
+
+      #it "pops jobs off the default Redis server" do
+        #@default_server.should_receive(:lpop)
+
+        #Resque.redis = Resque::RedisComposite.new("default" => @default_server, "stuff" => @special_server)
+        #Resque.pop("email")
+      #end
+    #end
+
+    #describe "Resque.queues" do
+      #it "aggregates all the queues" do
+        #@default_server.should_receive(:smembers).with("resque:queues").and_return(%w(a b c d))
+        #@special_server.should_receive(:smembers).with("resque:queues").and_return(%w(d e f))
+      
+        #Resque.redis = Resque::RedisComposite.new("default" => @default_server, "stuff" => @special_server)
+        #Resque.queues.sort.should == %w(a b c d e f)
+      #end
+    #end
+    
+    #describe "Resque::Failure.create" do
+      #it "pushes the failure onto the specific Redis server" do
+        #@special_server.should_receive(:rpush)
+      
+        #Resque.redis = Resque::RedisComposite.new("default" => @default_server, "stuff" => @special_server)
+        #Resque::Failure.create(:queue => "stuff", :exception => StandardError.new)
+      #end
+
+      #it "pushes the failure onto the default Redis server" do
+        #@default_server.should_receive(:rpush)
+      
+        #Resque.redis = Resque::RedisComposite.new("default" => @default_server, "stuff" => @special_server)
+        #Resque::Failure.create(:queue => "transactions", :exception => StandardError.new)
+      #end
+    #end
+  end
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: resque_redis_composite
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+  prerelease: 
+platform: ruby
+authors:
+- redsquirrel
+- jiHyunBae
+- Anthony Powles
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Allowing Resque to handle queues accross multiple Redis instances
+email: rubygems+resque_redis_composite@idreamz.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- lib/resque/redis_composite.rb
+- lib/resque_redis_composite.rb
+- resque_redis_composite.gemspec
+- spec/resque/redis_composite_spec.rb
+homepage: https://github.com/yogin/resque_redis_composite
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Resque support for multiple Redis instances
+test_files: []