fairway 0.3.1 → 0.3.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OGE1MjlkNzEzY2FiZDc0MWMzMGNkZDhlYjkxODg4MzY3ZmJhNGE0NA==
+    ODBhZmUyMTQyYmZkMTk0ZmJkNGM3ZmQ4ZjdkZWUzNzc5MjRlMTE3Nw==
   data.tar.gz: !binary |-
-    ZjBiMzdjNmQ3YWI4Mjg1OTE5MDVkNTU2OTUwMTZiNTIwODhjZDg5OQ==
+    MzcxMWQxYjFkMzVhYTY4M2Y1M2M0YjRlY2NlZGU0YzVjYjIwZjFmNA==
 SHA512:
   metadata.gz: !binary |-
-    ODAyMTVhODhiZWRmY2NhMzgyYTkyMzliZDIyNDUyOGM2NGJkMWY3Nzg1MmVm
-    NmZhM2Q5OWI4NWZkNGQzZWIxOWQyMGJmZGM2NWUxOTY5ZjI0MmUzNmZiMTcx
-    MTkwYzlhYzJkNWQyMzVmMjdhYThhNzAzMzI2YzMxNmFiZWY2Y2Y=
+    NGE4NmM1Yzk5NGE2NzYxNTdiNjIxYzQzZDU3ZjZiZmVjNjNkYjZmNTNjNDUz
+    Y2IwNmViOTgxNmU5NjAzM2U2OTE2MWJmNTk1YTEyYWVmZWEyMDFlZmI2NDgy
+    NDE5OTJiNDk1YTY5YTRlMDNiODFmNWU2YTQ3ODZiZjE3MjA0YWU=
   data.tar.gz: !binary |-
-    YWRjM2RhYWI3NzJjNDljZmQwYjgxYzIzZGMwNGI4MDliZmY2NTFkZGY0ZDlm
-    Y2U3MGU0YTUyZjU4ODNhNzA3N2RkOTI4ODYwYWY2NDM1YzYzMDk4NzQ5NDM1
-    MjRiZGM1MjdkYjU1N2M1NTI0YjgzYjYwMGE0MzJlNjA5NjExMjA=
+    MDEwZDIwMzI0YzQ4Yzc0NDU2NWRkNzMxOGM5ZjkxMjMwY2I5NGFiYmU2NTE0
+    NjA1YTAxMmNmZjNmNjEzNWJmZWFlMjBkNzljZmVjZDZkYmY4YTQwODFiMWMy
+    NDFjZjA1OTNhZTc0MWM5YmVmZjU0MzMxMjRkMGEzNmE0MzRjNWQ=

data/Gemfile

@@ -2,6 +2,8 @@ source :rubygems
 
 gem "rake"
 
+gem "activesupport", "4.0.2"
+
 # Specify your gem's dependencies in fairway.gemspec
 gemspec
 

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fairway (0.2.7)
-      activesupport (= 4.0.2)
+    fairway (0.3.1)
+      activesupport
       connection_pool
       redis
       redis-namespace (>= 1.3.0)
@@ -51,6 +51,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activesupport (= 4.0.2)
   fairway!
   rake
   rspec

data/go/fairway_ack.go

@@ -49,8 +49,9 @@ if removed > 0 then
     redis.call('hset', facet_pool, facet, current + 1);
   end
 
-  if (current == 0 and length == 0 and inflight_cur == 0 and n == 0) then
+  if (length == 0 and inflight_cur == 0 and n == 0) then
     redis.call('del', inflight_total);
+    redis.call('hdel', facet_pool, facet);
     redis.call('srem', active_facets, facet);
   end
 end

data/go/fairway_deliver.go

@@ -36,7 +36,6 @@ for i = 1, #registered_queues, 2 do
     local length = redis.call('lpush', k(queue, facet), message)
     redis.call('incr', k(queue, 'length'));
 
-
     -- Manage facet queue and active facets
     local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
     local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;

data/go/fairway_pull.go

@@ -10,38 +10,15 @@ local k = function (queue, subkey)
   return namespace .. queue .. ':' .. subkey;
 end
 
--- Multiple queues can be passed through
--- fairway_pull. We'll loop through all 
--- provided queues, and return a message
--- from the first one that isn't empty.
-for i, queue in ipairs(ARGV) do
-  local active_facets  = k(queue, 'active_facets');
+local pull = function (queue)
   local round_robin    = k(queue, 'facet_queue');
   local inflight       = k(queue, 'inflight');
-  local inflight_limit = k(queue, 'limit');
-  local priorities     = k(queue, 'priorities');
-  local facet_pool     = k(queue, 'facet_pool');
-
-  if wait ~= -1 then
-    -- Check if any current inflight messages
-    -- have been inflight for a long time.
-    local inflightmessage = redis.call('zrange', inflight, 0, 0, 'WITHSCORES');
-
-    -- If we have an inflight message and it's score
-    -- is less than the current pull timestamp, reset
-    -- the inflight score for the the message and resend.
-    if #inflightmessage > 0 then
-      if tonumber(inflightmessage[2]) <= timestamp then
-        redis.call('zadd', inflight, timestamp + wait, inflightmessage[1]);
-        return {queue, inflightmessage[1]}
-      end
-    end
-  end
 
   -- Pull a facet from the round-robin list.
   -- This list guarantees each active facet will have a
   -- message pulled from the queue every time through..
   local facet = redis.call('rpop', round_robin);
+  local message = nil
 
   if facet then
     -- If we found an active facet, we know the facet
@@ -50,7 +27,7 @@ for i, queue in ipairs(ARGV) do
     local messages       = k(queue, facet);
     local inflight_total = k(queue, facet .. ':inflight');
 
-    local message = redis.call('rpop', messages);
+    message = redis.call('rpop', messages);
 
     if message then
       if wait ~= -1 then
@@ -60,43 +37,99 @@ for i, queue in ipairs(ARGV) do
 
       redis.call('decr', k(queue, 'length'));
     end
+  end
 
-    -- Manage facet queue and active facets
-    local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
-    local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;
-    local length        = redis.call('llen', messages);
-    local inflight_cur  = tonumber(redis.call('get', inflight_total)) or 0;
-    local inflight_max  = tonumber(redis.call('get', inflight_limit)) or 0;
+  return {facet, message};
+end
+
+local manage = function (queue, facet)
+  local active_facets  = k(queue, 'active_facets');
+  local round_robin    = k(queue, 'facet_queue');
+  local inflight_limit = k(queue, 'limit');
+  local priorities     = k(queue, 'priorities');
+  local facet_pool     = k(queue, 'facet_pool');
+  local messages       = k(queue, facet);
+  local inflight_total = k(queue, facet .. ':inflight');
+
+  local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
+  local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;
+  local length        = redis.call('llen', messages);
+  local inflight_cur  = tonumber(redis.call('get', inflight_total)) or 0;
+  local inflight_max  = tonumber(redis.call('get', inflight_limit)) or 0;
+  
+  local n = 0
+  
+  -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
+  
+  if inflight_max > 0 then
+    n = math.min(length, priority, inflight_max - inflight_cur);
+  else
+    n = math.min(length, priority);
+  end
+  
+  -- redis.log(redis.LOG_WARNING, "PULL: "..current.."/"..n);
+  
+  if n < current then
+    -- redis.log(redis.LOG_WARNING, "shrinking");
+    redis.call('hset', facet_pool, facet, current - 1);
+  elseif n > current then
+    -- redis.log(redis.LOG_WARNING, "growing");
+    redis.call('lpush', round_robin, facet);
+    redis.call('lpush', round_robin, facet);
+    redis.call('hset', facet_pool, facet, current + 1);
+  else
+    -- redis.log(redis.LOG_WARNING, "maintaining");
+    redis.call('lpush', round_robin, facet);
+  end
+  
+  if (length == 0 and inflight_cur == 0 and n == 0) then
+    redis.call('del', inflight_total);
+    redis.call('hdel', facet_pool, facet);
+    redis.call('srem', active_facets, facet);
+  end
+end
 
-    local n = 0
+-- Multiple queues can be passed through
+-- fairway_pull. We'll loop through all 
+-- provided queues, and return a message
+-- from the first one that isn't empty.
+for i, queue in ipairs(ARGV) do
+  local inflight = k(queue, 'inflight');
 
-    -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
+  if wait ~= -1 then
+    -- Check if any current inflight messages
+    -- have been inflight for a long time.
+    local inflightmessage = redis.call('zrange', inflight, 0, 0, 'WITHSCORES');
 
-    if inflight_max > 0 then
-      n = math.min(length, priority, inflight_max - inflight_cur);
-    else
-      n = math.min(length, priority);
+    -- If we have an inflight message and it's score
+    -- is less than the current pull timestamp, reset
+    -- the inflight score for the the message and resend.
+    if #inflightmessage > 0 then
+      if tonumber(inflightmessage[2]) <= timestamp then
+        redis.call('zadd', inflight, timestamp + wait, inflightmessage[1]);
+        return {queue, inflightmessage[1]}
+      end
     end
+  end
 
-    -- redis.log(redis.LOG_WARNING, "PULL: "..current.."/"..n);
-
-    if n < current then
-      -- redis.log(redis.LOG_WARNING, "shrinking");
-      redis.call('hset', facet_pool, facet, current - 1);
-    elseif n > current then
-      -- redis.log(redis.LOG_WARNING, "growing");
-      redis.call('lpush', round_robin, facet);
-      redis.call('lpush', round_robin, facet);
-      redis.call('hset', facet_pool, facet, current + 1);
-    else
-      -- redis.log(redis.LOG_WARNING, "maintaining");
-      redis.call('lpush', round_robin, facet);
-    end
+  local pulled = pull(queue);
+  local facet = pulled[1];
+  local message = pulled[2];
 
-    if (current == 1 and length == 0 and inflight_cur == 0 and n == 0) then
-      redis.call('del', inflight_total);
-      redis.call('srem', active_facets, facet);
-    end
+  if facet then
+    manage(queue, facet);
+    
+    -- if message then
+    -- else
+    --   -- TODO loop through until we find a message
+    --   pulled = pull(queue);
+    --   facet = pulled[1];
+    --   message = pulled[2]; 		  
+    
+    --   if facet then
+    --     manage(queue, facet);
+    --   end
+    -- end
 
     return {queue, message};
   end

data/go/message.go

@@ -2,7 +2,8 @@ package fairway
 
 import (
 	"encoding/json"
-	"github.com/customerio/go-simplejson"
+
+	"github.com/bitly/go-simplejson"
 )
 
 type Msg struct {

data/go/queue_test.go

@@ -62,6 +62,87 @@ func QueueSpec(c gospec.Context) {
 			c.Expect(message.json(), Equals, msg2.json())
 		})
 
+		c.Specify("skips over facets in invalid state", func() {
+			config.Facet = func(msg *Msg) string {
+				str, _ := msg.Get("facet").String()
+				return str
+			}
+
+			msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
+			msg2, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage2"})
+			msg3, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage3"})
+
+			conn.Deliver(msg1)
+			conn.Deliver(msg2)
+			conn.Deliver(msg3)
+
+			r := config.Pool.Get()
+			defer r.Close()
+
+			count, _ := redis.Int(r.Do("llen", "fairway:myqueue:1"))
+			c.Expect(count, Equals, 2)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
+			c.Expect(count, Equals, 1)
+			count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
+			c.Expect(count, Equals, 2)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
+			c.Expect(count, Equals, 2)
+			count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
+			c.Expect(count, Equals, 1)
+			count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
+			c.Expect(count, Equals, 1)
+
+			queueName, message := queue.Pull(-1)
+			c.Expect(queueName, Equals, "myqueue")
+			c.Expect(message.json(), Equals, msg1.json())
+
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
+			c.Expect(count, Equals, 1)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
+			c.Expect(count, Equals, 1)
+			count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
+			c.Expect(count, Equals, 2)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
+			c.Expect(count, Equals, 2)
+			count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
+			c.Expect(count, Equals, 1)
+			count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
+			c.Expect(count, Equals, 1)
+
+			// We expect a message to be in here
+			r.Do("del", "fairway:myqueue:2")
+
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
+			c.Expect(count, Equals, 1)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
+			c.Expect(count, Equals, 0)
+			count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
+			c.Expect(count, Equals, 2)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
+			c.Expect(count, Equals, 2)
+			count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
+			c.Expect(count, Equals, 1)
+			count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
+			c.Expect(count, Equals, 1)
+
+			queueName, message = queue.Pull(-1)
+			c.Expect(queueName, Equals, "myqueue")
+			c.Expect(message.json(), Equals, msg3.json())
+
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
+			c.Expect(count, Equals, 0)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
+			c.Expect(count, Equals, 0)
+			count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
+			c.Expect(count, Equals, 0)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
+			c.Expect(count, Equals, 0)
+			count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
+			c.Expect(count, Equals, 0)
+			count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
+			c.Expect(count, Equals, 0)
+		})
+
 		c.Specify("places pulled message on inflight sorted set until acknowledged", func() {
 			msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
 

data/lib/fairway/version.rb

@@ -1,3 +1,3 @@
 module Fairway
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/redis/fairway_deliver.lua

@@ -32,7 +32,6 @@ for i = 1, #registered_queues, 2 do
     local length = redis.call('lpush', k(queue, facet), message)
     redis.call('incr', k(queue, 'length'));
 
-
     -- Manage facet queue and active facets
     local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
     local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;

data/redis/fairway_pull.lua

@@ -6,38 +6,15 @@ local k = function (queue, subkey)
   return namespace .. queue .. ':' .. subkey;
 end
 
--- Multiple queues can be passed through
--- fairway_pull. We'll loop through all 
--- provided queues, and return a message
--- from the first one that isn't empty.
-for i, queue in ipairs(ARGV) do
-  local active_facets  = k(queue, 'active_facets');
+local pull = function (queue)
   local round_robin    = k(queue, 'facet_queue');
   local inflight       = k(queue, 'inflight');
-  local inflight_limit = k(queue, 'limit');
-  local priorities     = k(queue, 'priorities');
-  local facet_pool     = k(queue, 'facet_pool');
-
-  if wait ~= -1 then
-    -- Check if any current inflight messages
-    -- have been inflight for a long time.
-    local inflightmessage = redis.call('zrange', inflight, 0, 0, 'WITHSCORES');
-
-    -- If we have an inflight message and it's score
-    -- is less than the current pull timestamp, reset
-    -- the inflight score for the the message and resend.
-    if #inflightmessage > 0 then
-      if tonumber(inflightmessage[2]) <= timestamp then
-        redis.call('zadd', inflight, timestamp + wait, inflightmessage[1]);
-        return {queue, inflightmessage[1]}
-      end
-    end
-  end
 
   -- Pull a facet from the round-robin list.
   -- This list guarantees each active facet will have a
   -- message pulled from the queue every time through..
   local facet = redis.call('rpop', round_robin);
+  local message = nil
 
   if facet then
     -- If we found an active facet, we know the facet
@@ -46,7 +23,7 @@ for i, queue in ipairs(ARGV) do
     local messages       = k(queue, facet);
     local inflight_total = k(queue, facet .. ':inflight');
 
-    local message = redis.call('rpop', messages);
+    message = redis.call('rpop', messages);
 
     if message then
       if wait ~= -1 then
@@ -56,44 +33,101 @@ for i, queue in ipairs(ARGV) do
 
       redis.call('decr', k(queue, 'length'));
     end
+  end
 
-    -- Manage facet queue and active facets
-    local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
-    local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;
-    local length        = redis.call('llen', messages);
-    local inflight_cur  = tonumber(redis.call('get', inflight_total)) or 0;
-    local inflight_max  = tonumber(redis.call('get', inflight_limit)) or 0;
+  return {facet, message};
+end
 
-    local n = 0
+local manage = function (queue, facet)
+  local active_facets  = k(queue, 'active_facets');
+  local round_robin    = k(queue, 'facet_queue');
+  local inflight_limit = k(queue, 'limit');
+  local priorities     = k(queue, 'priorities');
+  local facet_pool     = k(queue, 'facet_pool');
+  local messages       = k(queue, facet);
+  local inflight_total = k(queue, facet .. ':inflight');
+
+  local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
+  local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;
+  local length        = redis.call('llen', messages);
+  local inflight_cur  = tonumber(redis.call('get', inflight_total)) or 0;
+  local inflight_max  = tonumber(redis.call('get', inflight_limit)) or 0;
+  
+  local n = 0
+  
+  -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
+  
+  if inflight_max > 0 then
+    n = math.min(length, priority, inflight_max - inflight_cur);
+  else
+    n = math.min(length, priority);
+  end
+  
+  -- redis.log(redis.LOG_WARNING, "PULL: "..current.."/"..n);
+  
+  if n < current then
+    -- redis.log(redis.LOG_WARNING, "shrinking");
+    redis.call('hset', facet_pool, facet, current - 1);
+  elseif n > current then
+    -- redis.log(redis.LOG_WARNING, "growing");
+    redis.call('lpush', round_robin, facet);
+    redis.call('lpush', round_robin, facet);
+    redis.call('hset', facet_pool, facet, current + 1);
+  else
+    -- redis.log(redis.LOG_WARNING, "maintaining");
+    redis.call('lpush', round_robin, facet);
+  end
+  
+  if (length == 0 and inflight_cur == 0 and n == 0) then
+    redis.call('del', inflight_total);
+    redis.call('hdel', facet_pool, facet);
+    redis.call('srem', active_facets, facet);
+  end
+end
 
-    -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
+-- Multiple queues can be passed through
+-- fairway_pull. We'll loop through all 
+-- provided queues, and return a message
+-- from the first one that isn't empty.
+for i, queue in ipairs(ARGV) do
+  local inflight = k(queue, 'inflight');
 
-    if inflight_max > 0 then
-      n = math.min(length, priority, inflight_max - inflight_cur);
-    else
-      n = math.min(length, priority);
-    end
+  if wait ~= -1 then
+    -- Check if any current inflight messages
+    -- have been inflight for a long time.
+    local inflightmessage = redis.call('zrange', inflight, 0, 0, 'WITHSCORES');
 
-    -- redis.log(redis.LOG_WARNING, "PULL: "..current.."/"..n);
-
-    if n < current then
-      -- redis.log(redis.LOG_WARNING, "shrinking");
-      redis.call('hset', facet_pool, facet, current - 1);
-    elseif n > current then
-      -- redis.log(redis.LOG_WARNING, "growing");
-      redis.call('lpush', round_robin, facet);
-      redis.call('lpush', round_robin, facet);
-      redis.call('hset', facet_pool, facet, current + 1);
-    else
-      -- redis.log(redis.LOG_WARNING, "maintaining");
-      redis.call('lpush', round_robin, facet);
+    -- If we have an inflight message and it's score
+    -- is less than the current pull timestamp, reset
+    -- the inflight score for the the message and resend.
+    if #inflightmessage > 0 then
+      if tonumber(inflightmessage[2]) <= timestamp then
+        redis.call('zadd', inflight, timestamp + wait, inflightmessage[1]);
+        return {queue, inflightmessage[1]}
+      end
     end
+  end
 
-    if (current == 1 and length == 0 and inflight_cur == 0 and n == 0) then
-      redis.call('del', inflight_total);
-      redis.call('srem', active_facets, facet);
-    end
+  local pulled = pull(queue);
+  local facet = pulled[1];
+  local message = pulled[2];
+
+  if facet then
+    manage(queue, facet);
+    
+    -- if message then
+    -- else
+    --   -- TODO loop through until we find a message
+    --   pulled = pull(queue);
+    --   facet = pulled[1];
+    --   message = pulled[2]; 		  
+    
+    --   if facet then
+    --     manage(queue, facet);
+    --   end
+    -- end
 
     return {queue, message};
   end
 end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fairway
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - John Allison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-15 00:00:00.000000000 Z
+date: 2016-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport