fairway 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2b3057c6ab8a45805ace676a8d13f8775ca5b42b
-  data.tar.gz: 0520d51ae3dc1aa63557d1aa253d62f1cf03b1e8
+  metadata.gz: 560fe83c9779d40b22fe2d5e093d3ad9310dd9ed
+  data.tar.gz: f1c2d3c7933944f75a0df7c987430a5a524193bc
 SHA512:
-  metadata.gz: 6592bfbf636f74b6f9deb95c508d74464202b70fab905c8c293e393179c3904c88353aaba1b1fd8784c29c7e94abd4fb993f63d3447e4253f8c1cada4404fcbd
-  data.tar.gz: b6d6099a051fdf43df55f8070ccfc8a0203e3463d2bfbb303ebecaf9e62c31a0c46417b7e80bf413e4f77a41fdaeef0442e9722586c8a807473df989ff45e3d7
+  metadata.gz: 3c20f8ce9420514cd5ca9c1b54382aa5a8ce0bdd9f7e9947981864edfb169c54d6557ee0eb6c0f323727406d059c9054885386e97d5199f45ded8d4bceaea58b
+  data.tar.gz: fb5e08076353e022fd4fbc9207d79ea566b014c3e789b056a8107a3bdf0cf2615c5d366a5fd577b66c18b621ecbfa3bd387178f08b47483e5fe9619c5a17c9b2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fairway (0.2.1)
+    fairway (0.2.6)
       activesupport
       connection_pool
       redis

data/lib/fairway/config.rb

@@ -1,15 +1,60 @@
 module Fairway
+  class RandomDistribution
+    class CannotConnect < RuntimeError; end
+
+    EXCEPTIONS = [
+      Redis::CannotConnectError,
+      Errno::ETIMEDOUT,
+      Errno::EHOSTUNREACH
+    ]
+
+    attr_reader :pools
+
+    def initialize(pools)
+      @pools = pools
+    end
+
+    def with(&block)
+      valid_pools = @pools
+
+      while valid_pools.any?
+        pool = valid_pools.sample
+
+        pool.with do |conn|
+          begin
+            return yield(conn)
+          rescue *EXCEPTIONS
+            valid_pools -= [pool]
+          end
+        end
+      end
+
+      raise CannotConnect.new
+    end
+
+    def with_each(&block)
+      @pools.shuffle.each do |pool|
+        pool.with do |conn|
+          begin
+            yield(conn)
+          end
+        end
+      end
+    end
+  end
+
   class Config
     attr_accessor :namespace
-    attr_reader :defined_queues, :redis_options
+    attr_reader :defined_queues, :redis_options, :distribute
 
     DEFAULT_FACET = "default"
 
     QueueDefinition = Struct.new(:name, :channel)
 
     def initialize
-      @redis_options  = {}
+      @redis_options  = []
       @namespace      = nil
+      @distribute     = RandomDistribution
       @facet          = lambda { |message| DEFAULT_FACET }
       @defined_queues = []
 
@@ -29,32 +74,54 @@ module Fairway
     end
 
     def redis=(options)
-      @redis_options = options
+      @redis_options = [options].flatten
     end
 
     def redis
-      @redis ||= pool { Redis::Namespace.new(@namespace, redis: raw_redis) }
+      @redis ||= redises
     end
 
     def scripts
       @scripts ||= begin
-        Scripts.new(pool { raw_redis }, @namespace)
+        Scripts.new(raw_redises, @namespace)
       end
     end
 
     private
 
-    def pool(&block)
-      pool_size    = @redis_options[:pool]    || 1
-      pool_timeout = @redis_options[:timeout] || 5
+    def redises
+      @redises ||= begin
+        @redis_options << {} if @redis_options.empty?
+        pools = @redis_options.map do |options|
+          pool(options) { Redis::Namespace.new(@namespace, redis: raw_redis(options)) }
+        end
+
+        @distribute.new(pools)
+      end
+    end
+
+    def raw_redises
+      @raw_redises ||= begin
+        @redis_options << {} if @redis_options.empty?
+        pools = @redis_options.map do |options|
+          pool(options) { raw_redis(options) }
+        end
+
+        @distribute.new(pools)
+      end
+    end
+
+    def pool(options, &block)
+      pool_size    = options[:pool]    || 1
+      pool_timeout = options[:timeout] || 5
 
       ConnectionPool.new(size: pool_size, timeout: pool_timeout) do
         yield
       end
     end
 
-    def raw_redis
-      Redis.new(@redis_options)
+    def raw_redis(options)
+      Redis.new(options)
     end
   end
 end

data/lib/fairway/facet.rb

@@ -10,11 +10,13 @@ module Fairway
     end
 
     def length
-      redis.with do |conn|
-        each_queue do |queue|
-          conn.llen(facet_key(queue))
-        end.sum
-      end
+      redis.pools.map do |pool|
+        pool.with do |conn|
+          each_queue do |queue|
+            conn.llen(facet_key(queue))
+          end.sum
+        end
+      end.sum
     end
 
     def priority

data/lib/fairway/queue.rb

@@ -8,21 +8,25 @@ module Fairway
     end
 
     def active_facets
-      redis.with do |conn|
-        facet_names = unique_queues.map do |queue|
+      facet_names = []
+
+      redis.with_each do |conn|
+        facet_names += unique_queues.map do |queue|
           conn.smembers("#{queue}:active_facets")
-        end.flatten.uniq
-      
-        facet_names.map do |name|
-          Facet.new(self, name)
-        end
+        end.flatten
+      end
+
+      facet_names.uniq.map do |name|
+        Facet.new(self, name)
       end
     end
 
     def length
-      redis.with do |conn|
-        conn.mget(unique_queues.map{|q| "#{q}:length" }).map(&:to_i).sum
-      end
+      redis.pools.map do |pool|
+        pool.with do |conn|
+          conn.mget(unique_queues.map{|q| "#{q}:length" }).map(&:to_i).sum
+        end
+      end.sum
     end
 
     def peek

data/lib/fairway/scripts.rb

@@ -15,13 +15,13 @@ module Fairway
     end
 
     def register_queue(name, channel)
-      redis.with do |conn|
+      redis.with_each do |conn|
         conn.hset(registered_queues_key, name, channel)
       end
     end
 
     def unregister_queue(name)
-      redis.with do |conn|
+      redis.with_each do |conn|
         conn.hdel(registered_queues_key, name)
       end
     end
@@ -35,12 +35,22 @@ module Fairway
     def method_missing(method_name, *args)
       loaded = false
 
-      redis.with do |conn|
-        conn.evalsha(script_sha(method_name), [namespace], args)
+      if multi?(method_name)
+        redis.with_each do |conn|
+          conn.evalsha(script_sha(method_name), [namespace], args)
+        end
+      elsif first?(method_name)
+        first_pool do |conn|
+          conn.evalsha(script_sha(method_name), [namespace], args)
+        end
+      else
+        redis.with do |conn|
+          conn.evalsha(script_sha(method_name), [namespace], args)
+        end
       end
     rescue Redis::CommandError => ex
       if ex.message.include?("NOSCRIPT") && !loaded
-        redis.with do |conn|
+        redis.with_each do |conn|
           conn.script(:load, script_source(method_name))
         end
 
@@ -53,6 +63,23 @@ module Fairway
 
   private
 
+    def first?(script)
+      ["fairway_pull", "fairway_peek"].include?(script.to_s)
+    end
+
+    def multi?(script)
+      ["fairway_priority", "fairway_destroy"].include?(script.to_s)
+    end
+
+    def first_pool(&block)
+      redis.with_each do |conn|
+        val = yield(conn)
+        return val if val
+      end
+
+      nil
+    end
+
     def registered_queues_key
       "#{namespace}registered_queues"
     end

data/lib/fairway/version.rb

@@ -1,3 +1,3 @@
 module Fairway
-  VERSION = "0.2.6"
+  VERSION = "0.2.7"
 end

data/spec/lib/fairway/config_spec.rb

@@ -29,19 +29,70 @@ module Fairway
       end
     end
 
+    it "allows multiple redis config" do
+      config = Config.new do |config|
+        config.redis = [
+          { host: "127.0.0.1", port: 6379 },
+          { host: "127.0.0.1", port: 6380 }
+        ]
+      end
+
+      config.redis.pools.length.should == 2
+    end
+
+    it "distributes requests randomly between multiple redis's" do
+      config = Config.new do |config|
+        config.redis = [
+          { host: "127.0.0.1", port: 6379 },
+          { host: "127.0.0.1", port: 6380 }
+        ]
+      end
+      
+      ports = []
+
+      20.times do
+        config.redis.with do |conn|
+          ports << conn.info["tcp_port"]
+        end
+      end
+
+      ports.uniq.sort.should == ["6379", "6380"]
+    end
+
+    it "doesn't lose requests if one redis is down" do
+      config = Config.new do |config|
+        config.redis = [
+          { host: "127.0.0.1", port: 6379 },
+          { host: "127.0.0.1", port: 6380 },
+          { host: "127.0.0.1", port: 9999 }
+        ]
+      end
+      
+      ports = []
+
+      20.times do
+        config.redis.with do |conn|
+          ports << conn.info["tcp_port"]
+        end
+      end
+
+      ports.length.should == 20
+      ports.uniq.sort.should == ["6379", "6380"]
+    end
+
     it "allows setting of connection pooling" do
       config = Config.new do |config|
         config.redis = { pool: 10 }
       end
 
-      config.redis.instance_variable_get("@size").should == 10
+      config.redis.pools.first.instance_variable_get("@size").should == 10
     end
 
     it "defaults to pool of 1" do
       config = Config.new do |config|
       end
 
-      config.redis.instance_variable_get("@size").should == 1
+      config.redis.pools.first.instance_variable_get("@size").should == 1
     end
 
     it "allows setting of redis namespace" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fairway
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - John Allison