fairway 0.2.7 → 0.3.1

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 560fe83c9779d40b22fe2d5e093d3ad9310dd9ed
-  data.tar.gz: f1c2d3c7933944f75a0df7c987430a5a524193bc
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OGE1MjlkNzEzY2FiZDc0MWMzMGNkZDhlYjkxODg4MzY3ZmJhNGE0NA==
+  data.tar.gz: !binary |-
+    ZjBiMzdjNmQ3YWI4Mjg1OTE5MDVkNTU2OTUwMTZiNTIwODhjZDg5OQ==
 SHA512:
-  metadata.gz: 3c20f8ce9420514cd5ca9c1b54382aa5a8ce0bdd9f7e9947981864edfb169c54d6557ee0eb6c0f323727406d059c9054885386e97d5199f45ded8d4bceaea58b
-  data.tar.gz: fb5e08076353e022fd4fbc9207d79ea566b014c3e789b056a8107a3bdf0cf2615c5d366a5fd577b66c18b621ecbfa3bd387178f08b47483e5fe9619c5a17c9b2
+  metadata.gz: !binary |-
+    ODAyMTVhODhiZWRmY2NhMzgyYTkyMzliZDIyNDUyOGM2NGJkMWY3Nzg1MmVm
+    NmZhM2Q5OWI4NWZkNGQzZWIxOWQyMGJmZGM2NWUxOTY5ZjI0MmUzNmZiMTcx
+    MTkwYzlhYzJkNWQyMzVmMjdhYThhNzAzMzI2YzMxNmFiZWY2Y2Y=
+  data.tar.gz: !binary |-
+    YWRjM2RhYWI3NzJjNDljZmQwYjgxYzIzZGMwNGI4MDliZmY2NTFkZGY0ZDlm
+    Y2U3MGU0YTUyZjU4ODNhNzA3N2RkOTI4ODYwYWY2NDM1YzYzMDk4NzQ5NDM1
+    MjRiZGM1MjdkYjU1N2M1NTI0YjgzYjYwMGE0MzJlNjA5NjExMjA=

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    fairway (0.2.6)
-      activesupport
+    fairway (0.2.7)
+      activesupport (= 4.0.2)
       connection_pool
       redis
       redis-namespace (>= 1.3.0)
@@ -16,14 +16,13 @@ GEM
       multi_json (~> 1.3)
       thread_safe (~> 0.1)
       tzinfo (~> 0.3.37)
-    atomic (1.1.14)
     celluloid (0.12.4)
       facter (>= 1.6.12)
       timers (>= 1.0.0)
     connection_pool (1.0.0)
     diff-lcs (1.1.3)
     facter (1.6.17)
-    i18n (0.6.9)
+    i18n (0.7.0)
     minitest (4.7.5)
     multi_json (1.5.0)
     rake (10.1.0)
@@ -44,10 +43,9 @@ GEM
       multi_json (~> 1)
       redis (~> 3)
       redis-namespace
-    thread_safe (0.1.3)
-      atomic
+    thread_safe (0.3.5)
     timers (1.1.0)
-    tzinfo (0.3.38)
+    tzinfo (0.3.46)
 
 PLATFORMS
   ruby
@@ -57,3 +55,6 @@ DEPENDENCIES
   rake
   rspec
   sidekiq
+
+BUNDLED WITH
+   1.10.6

data/go/channeled_connection_test.go

@@ -2,6 +2,7 @@ package fairway
 
 import (
 	"fmt"
+
 	"github.com/customerio/gospec"
 	. "github.com/customerio/gospec"
 	"github.com/customerio/redigo/redis"
@@ -37,4 +38,25 @@ func ChanneledConnectionSpec(c gospec.Context) {
 			c.Expect(count, Equals, 0)
 		})
 	})
+
+	c.Specify("DeliverBytes", func() {
+		c.Specify("only queues up message for matching queues", func() {
+			r := config.Pool.Get()
+			defer r.Close()
+
+			count, _ := redis.Int(r.Do("llen", "fairway:myqueue:default"))
+			c.Expect(count, Equals, 0)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue2:default"))
+			c.Expect(count, Equals, 0)
+
+			msg, _ := NewMsg(map[string]string{"type": "a"})
+
+			conn.DeliverBytes("channel:typea:channel", "default", []byte(msg.json()))
+
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue:default"))
+			c.Expect(count, Equals, 1)
+			count, _ = redis.Int(r.Do("llen", "fairway:myqueue2:default"))
+			c.Expect(count, Equals, 0)
+		})
+	})
 }

data/go/config.go

@@ -32,7 +32,7 @@ func NewConfig(server string, db string, poolSize int) *Config {
 		[]*QueueDefinition{},
 		&redis.Pool{
 			MaxIdle:     poolSize,
-			MaxActive:   poolSize,
+			MaxActive:   0,
 			IdleTimeout: 240 * time.Second,
 			Dial: func() (redis.Conn, error) {
 				c, err := redis.Dial("tcp", server)

data/go/config_test.go

@@ -25,10 +25,10 @@ func ConfigSpec(c gospec.Context) {
 
 	c.Specify("sets redis pool size", func() {
 		c.Expect(config.Pool.MaxIdle, Equals, 10)
-		c.Expect(config.Pool.MaxActive, Equals, 10)
+		c.Expect(config.Pool.MaxActive, Equals, 0)
 		config = NewConfig("localhost:6379", "15", 20)
 		c.Expect(config.Pool.MaxIdle, Equals, 20)
-		c.Expect(config.Pool.MaxActive, Equals, 20)
+		c.Expect(config.Pool.MaxActive, Equals, 0)
 	})
 
 	c.Specify("can specify custom namespace", func() {

data/go/connection.go

@@ -5,6 +5,7 @@ type Connection interface {
 	Queues() []*Queue
 	Channel(*Msg) string
 	Deliver(*Msg) error
+	DeliverBytes(channel, facet string, bytes []byte) error
 	Configuration() *Config
 }
 
@@ -40,6 +41,10 @@ func (c *conn) Deliver(msg *Msg) error {
 	return c.scripts.deliver(channel, facet, msg)
 }
 
+func (c *conn) DeliverBytes(channel, facet string, msg []byte) error {
+	return c.scripts.deliverBytes(channel, facet, msg)
+}
+
 func (c *conn) Configuration() *Config {
 	return c.config
 }

data/go/connection_test.go

@@ -97,6 +97,7 @@ func ConnectionSpec(c gospec.Context) {
 			defer r.Close()
 
 			r.Do("sadd", "fairway:myqueue:active_facets", "default")
+			r.Do("hset", "fairway:myqueue:facet_pool", "default", "1")
 
 			msg, _ := NewMsg(map[string]string{})
 

data/go/fairway_ack.go

@@ -0,0 +1,60 @@
+package fairway
+
+func FairwayAck() string {
+	return `
+local namespace = KEYS[1];
+
+local k = function (queue, subkey)
+  return namespace .. queue .. ':' .. subkey;
+end
+
+local queue   = ARGV[1];
+local facet   = ARGV[2];
+local message = ARGV[3];
+
+local active_facets  = k(queue, 'active_facets');
+local round_robin    = k(queue, 'facet_queue');
+local facet_pool     = k(queue, 'facet_pool');
+local inflight       = k(queue, 'inflight');
+local messages       = k(queue, facet);
+local inflight_total = k(queue, facet .. ':inflight');
+local inflight_limit = k(queue, 'limit');
+local priorities     = k(queue, 'priorities');
+
+local removed = tonumber(redis.call('zrem', inflight, message))
+
+if removed > 0 then
+  -- Manage facet queue and active facets
+  local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
+  local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;
+  local length        = redis.call('llen', messages);
+  local inflight_cur  = tonumber(redis.call('decr', inflight_total)) or 0;
+  local inflight_max  = tonumber(redis.call('get', inflight_limit)) or 0;
+
+  local n = 0
+
+  -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur)
+
+  if inflight_max > 0 then
+    n = math.min(length, priority, inflight_max - inflight_cur);
+  else
+    n = math.min(length, priority);
+  end
+
+  -- redis.log(redis.LOG_WARNING, "ACK: "..current.."/"..n);
+
+  if n > current then
+    -- redis.log(redis.LOG_WARNING, "growing");
+    redis.call('lpush', round_robin, facet);
+    redis.call('hset', facet_pool, facet, current + 1);
+  end
+
+  if (current == 0 and length == 0 and inflight_cur == 0 and n == 0) then
+    redis.call('del', inflight_total);
+    redis.call('srem', active_facets, facet);
+  end
+end
+
+return removed
+`
+}

data/go/fairway_deliver.go

@@ -1,7 +1,7 @@
 package fairway
 
 func FairwayDeliver() string {
-  return `
+	return `
 local namespace = KEYS[1];
 local topic     = ARGV[1];
 local facet     = ARGV[2];
@@ -23,45 +23,50 @@ for i = 1, #registered_queues, 2 do
   -- If the message topic matches the queue topic,
   -- we deliver the message to the queue.
   if string.find(topic, queue_topic) then
-    local priorities    = k(queue, 'priorities');
-    local active_facets = k(queue, 'active_facets');
-    local round_robin   = k(queue, 'facet_queue');
-    local facet_pool    = k(queue, 'facet_pool');
+    local priorities     = k(queue, 'priorities');
+    local active_facets  = k(queue, 'active_facets');
+    local round_robin    = k(queue, 'facet_queue');
+    local facet_pool     = k(queue, 'facet_pool');
+    local inflight_total = k(queue, facet .. ':inflight');
+    local inflight_limit = k(queue, 'limit');
 
     -- Delivering the message to a queue is as simple as
     -- pushing it onto the facet's message list, and
     -- incrementing the length of the queue itself.
-    redis.call('lpush', k(queue, facet), message)
+    local length = redis.call('lpush', k(queue, facet), message)
     redis.call('incr', k(queue, 'length'));
 
-    -- If the facet just became active, we need to add
-    -- the facet to the round-robin queue, so it's
-    -- messages will be processed.
-    if redis.call('sadd', active_facets, facet) == 1 then
-      local priority = tonumber(redis.call('hget', priorities, facet)) or 1
 
-      -- If the facet currently has a priority
-      -- we need to jump start the facet by adding
-      -- it to the round-robin queue and updating
-      -- the current priority.
-      if priority > 0 then
-        redis.call('lpush', round_robin, facet);
-        redis.call('hset', facet_pool, facet, 1);
-      
-      -- If the facet has no set priority, just set the
-      -- current priority to zero. Since the facet just
-      -- became active, we can be sure it's already zero
-      -- or undefined.
-      else
-        redis.call('hset', facet_pool, facet, 0);
-      end
+    -- Manage facet queue and active facets
+    local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
+    local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;
+    local inflight_cur  = tonumber(redis.call('get', inflight_total)) or 0;
+    local inflight_max  = tonumber(redis.call('get', inflight_limit)) or 0;
+
+    local n = 0
+
+    -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
+
+    if inflight_max > 0 then
+      n = math.min(length, priority, inflight_max - inflight_cur);
+    else
+      n = math.min(length, priority);
+    end
+
+    -- redis.log(redis.LOG_WARNING, "PUSH: "..current.."/"..n);
+
+    if n > current then
+      -- redis.log(redis.LOG_WARNING, "growing");
+      redis.call('lpush', round_robin, facet);
+      redis.call('hset', facet_pool, facet, current + 1);
     end
+
+    redis.call('sadd', active_facets, facet)
   end
 end
 
 -- For any clients listening over pub/sub,
 -- we should publish the message.
 redis.call('publish', namespace .. topic, message);
-
 `
 }

data/go/fairway_destroy.go

@@ -1,7 +1,7 @@
 package fairway
 
 func FairwayDestroy() string {
-  return `
+	return `
 local namespace = KEYS[1];
 
 local k = function (queue, subkey)
@@ -13,15 +13,18 @@ end
 -- provided queues, and delete related keys
 -- for each queue.
 for i, queue in ipairs(ARGV) do
-  local priorities    = k(queue, 'priorities');
-  local active_facets = k(queue, 'active_facets');
-  local round_robin   = k(queue, 'facet_queue');
-  local facet_pool    = k(queue, 'facet_pool');
-  local length        = k(queue, 'length');
+  local priorities     = k(queue, 'priorities');
+  local active_facets  = k(queue, 'active_facets');
+  local round_robin    = k(queue, 'facet_queue');
+  local facet_pool     = k(queue, 'facet_pool');
+  local inflight       = k(queue, 'inflight');
+  local inflight_limit = k(queue, 'limit');
+  local length         = k(queue, 'length');
 
   local facets = redis.call('smembers', active_facets);
 
   for i = 1, #facets, 1 do
+    redis.call('del', k(queue, facets[i] .. ':inflight'));
     redis.call('del', k(queue, facets[i]));
   end
 
@@ -29,6 +32,8 @@ for i, queue in ipairs(ARGV) do
   redis.call('del', active_facets);
   redis.call('del', round_robin);
   redis.call('del', facet_pool);
+  redis.call('del', inflight);
+  redis.call('del', inflight_limit);
   redis.call('del', length);
 end
 

data/go/fairway_inflight.go

@@ -0,0 +1,15 @@
+package fairway
+
+func FairwayInflight() string {
+	return `
+local namespace = KEYS[1];
+
+local k = function (queue, subkey)
+  return namespace .. queue .. ':' .. subkey;
+end
+
+for i, queue in ipairs(ARGV) do
+  return redis.call('zrange', k(queue, 'inflight'), 0, -1);
+end
+`
+}

data/go/fairway_peek.go

@@ -1,11 +1,10 @@
 package fairway
 
 func FairwayPeek() string {
-  return `
+	return `
 local namespace = KEYS[1];
 
 for index, queue_name in ipairs(ARGV) do
-  local active_facets = namespace .. queue_name .. ':active_facets';
   local facet_queue   = namespace .. queue_name .. ':facet_queue';
 
   local facet = redis.call('lrange', facet_queue, -1, -1)[1];
@@ -17,6 +16,5 @@ for index, queue_name in ipairs(ARGV) do
     return {queue_name, message};
   end
 end
-
 `
 }

data/go/fairway_ping.go

@@ -0,0 +1,20 @@
+package fairway
+
+func FairwayPing() string {
+	return `
+local namespace = KEYS[1];
+local timestamp = tonumber(KEYS[2]);
+local wait = tonumber(KEYS[3]);
+
+local k = function (queue, subkey)
+  return namespace .. queue .. ':' .. subkey;
+end
+
+local queue = ARGV[1];
+local message = ARGV[2];
+
+local inflight = k(queue, 'inflight');
+
+redis.call('zadd', inflight, timestamp + wait, message);
+`
+}

data/go/fairway_priority.go

@@ -1,7 +1,7 @@
 package fairway
 
 func FairwayPriority() string {
-  return `
+	return `
 local namespace    = KEYS[1];
 local queue        = ARGV[1];
 local facet        = ARGV[2];
@@ -17,7 +17,7 @@ local facet_pool  = k(queue, 'facet_pool');
 
 -- Find the current state of the facet for the queue
 local priority = tonumber(redis.call('hget', priorities, facet)) or 1;
-local current  = tonumber(redis.call('hget', facet_pool, facet));
+local current  = tonumber(redis.call('hget', facet_pool, facet)) or 0;
 
 -- If priority is currently zero, we need to jump
 -- start the facet by adding it to the round-robin
@@ -31,6 +31,5 @@ end
 -- set the new priority, and the real priority
 -- will update lazily on pull.
 redis.call('hset', priorities, facet, new_priority);
-
 `
 }

data/go/fairway_pull.go

@@ -1,8 +1,10 @@
 package fairway
 
 func FairwayPull() string {
-  return `
+	return `
 local namespace = KEYS[1];
+local timestamp = tonumber(KEYS[2]);
+local wait = tonumber(KEYS[3]);
 
 local k = function (queue, subkey)
   return namespace .. queue .. ':' .. subkey;
@@ -13,10 +15,28 @@ end
 -- provided queues, and return a message
 -- from the first one that isn't empty.
 for i, queue in ipairs(ARGV) do
-  local priorities    = k(queue, 'priorities');
-  local active_facets = k(queue, 'active_facets');
-  local round_robin   = k(queue, 'facet_queue');
-  local facet_pool    = k(queue, 'facet_pool');
+  local active_facets  = k(queue, 'active_facets');
+  local round_robin    = k(queue, 'facet_queue');
+  local inflight       = k(queue, 'inflight');
+  local inflight_limit = k(queue, 'limit');
+  local priorities     = k(queue, 'priorities');
+  local facet_pool     = k(queue, 'facet_pool');
+
+  if wait ~= -1 then
+    -- Check if any current inflight messages
+    -- have been inflight for a long time.
+    local inflightmessage = redis.call('zrange', inflight, 0, 0, 'WITHSCORES');
+
+    -- If we have an inflight message and it's score
+    -- is less than the current pull timestamp, reset
+    -- the inflight score for the the message and resend.
+    if #inflightmessage > 0 then
+      if tonumber(inflightmessage[2]) <= timestamp then
+        redis.call('zadd', inflight, timestamp + wait, inflightmessage[1]);
+        return {queue, inflightmessage[1]}
+      end
+    end
+  end
 
   -- Pull a facet from the round-robin list.
   -- This list guarantees each active facet will have a
@@ -27,73 +47,59 @@ for i, queue in ipairs(ARGV) do
     -- If we found an active facet, we know the facet
     -- has at least one message available to be pulled
     -- from it's message queue.
-    local messages = k(queue, facet);
-    local message  = redis.call('rpop', messages);
+    local messages       = k(queue, facet);
+    local inflight_total = k(queue, facet .. ':inflight');
+
+    local message = redis.call('rpop', messages);
 
     if message then
+      if wait ~= -1 then
+        redis.call('zadd', inflight, timestamp + wait, message);
+        redis.call('incr', inflight_total);
+      end
+
       redis.call('decr', k(queue, 'length'));
     end
 
-    local length = redis.call('llen', messages);
+    -- Manage facet queue and active facets
+    local current       = tonumber(redis.call('hget', facet_pool, facet)) or 0;
+    local priority      = tonumber(redis.call('hget', priorities, facet)) or 1;
+    local length        = redis.call('llen', messages);
+    local inflight_cur  = tonumber(redis.call('get', inflight_total)) or 0;
+    local inflight_max  = tonumber(redis.call('get', inflight_limit)) or 0;
 
-    -- If the length of the facet's message queue
-    -- is empty, then it is no longer active as
-    -- it no longer has any messages.
-    if length == 0 then
-      -- We remove the facet from the set of active
-      -- facets and don't push the facet back on the
-      -- round-robin queue.
-      redis.call('srem', active_facets, facet);
-     
-    -- If the facet still has messages to process,
-    -- it remains in the active facet set, and is
-    -- pushed back on the round-robin queue.
-    --
-    -- Additionally, the priority of the facet may
-    -- have changed, so we'll check and update the
-    -- current facet's priority if needed.
+    local n = 0
+
+    -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
+
+    if inflight_max > 0 then
+      n = math.min(length, priority, inflight_max - inflight_cur);
     else
-      local priority = tonumber(redis.call('hget', priorities, facet)) or 1
-      local current  = tonumber(redis.call('hget', facet_pool, facet)) or 1
-
-      -- If the current priority is less than the
-      -- desired priority, let's increase the priority
-      -- by pushing the current facet on the round-robin
-      -- queue twice, and incrementing the current
-      -- priority.
-      --
-      -- Note: If there aren't enough messages left
-      -- on the facet, we don't increase priority.
-      if current < priority and length > current then
-        redis.call('lpush', round_robin, facet);
-        redis.call('lpush', round_robin, facet);
-        redis.call('hset', facet_pool, facet, current + 1);
-        
-      -- If the current priority is greater than the
-      -- desired priority, let's decrease the priority
-      -- by not pushing the current facet on the round-robin
-      -- queue, and decrementing the current priority.
-      --
-      -- Note: Also decrement priority if there aren't
-      -- enough messages for the current priority. This
-      -- ensures priority (entries in the round-robin queue)
-      -- never exceeds the number of messages for a given
-      -- facet.
-      elseif current > priority or current > length then
-        redis.call('hset', facet_pool, facet, current - 1);
-      
-      -- If the current priority is equals the
-      -- desired priority, let's maintain the current priority
-      -- by pushing the current facet on the round-robin
-      -- queue once.
-      else
-        redis.call('lpush', round_robin, facet);
-      end
+      n = math.min(length, priority);
+    end
+
+    -- redis.log(redis.LOG_WARNING, "PULL: "..current.."/"..n);
+
+    if n < current then
+      -- redis.log(redis.LOG_WARNING, "shrinking");
+      redis.call('hset', facet_pool, facet, current - 1);
+    elseif n > current then
+      -- redis.log(redis.LOG_WARNING, "growing");
+      redis.call('lpush', round_robin, facet);
+      redis.call('lpush', round_robin, facet);
+      redis.call('hset', facet_pool, facet, current + 1);
+    else
+      -- redis.log(redis.LOG_WARNING, "maintaining");
+      redis.call('lpush', round_robin, facet);
+    end
+
+    if (current == 1 and length == 0 and inflight_cur == 0 and n == 0) then
+      redis.call('del', inflight_total);
+      redis.call('srem', active_facets, facet);
     end
 
     return {queue, message};
   end
 end
-
 `
 }

data/go/message.go

@@ -6,6 +6,7 @@ import (
 )
 
 type Msg struct {
+	Original string
 	*simplejson.Json
 }
 
@@ -20,7 +21,7 @@ func NewMsg(body interface{}) (*Msg, error) {
 		return nil, err
 	}
 
-	return &Msg{simplej}, nil
+	return &Msg{string(bytes), simplej}, nil
 }
 
 func NewMsgFromString(body string) (*Msg, error) {
@@ -29,7 +30,7 @@ func NewMsgFromString(body string) (*Msg, error) {
 		return nil, err
 	}
 
-	return &Msg{simplej}, nil
+	return &Msg{body, simplej}, nil
 }
 
 func (m *Msg) json() string {

data/go/queue.go

@@ -9,6 +9,35 @@ func NewQueue(conn Connection, name string) *Queue {
 	return &Queue{conn, name}
 }
 
-func (q *Queue) Pull() (string, *Msg) {
-	return q.conn.Configuration().scripts().pull(q.name)
+func (q *Queue) Name() string {
+	return q.name
+}
+
+func (q *Queue) Length() (int, error) {
+	return q.conn.Configuration().scripts().length(q.name)
+}
+
+func (q *Queue) Pull(resendTimeframe int) (string, *Msg) {
+	return q.conn.Configuration().scripts().pull(q.name, resendTimeframe)
+}
+
+func (q *Queue) Inflight() []string {
+	return q.conn.Configuration().scripts().inflight(q.name)
+}
+
+func (q *Queue) InflightLimit() (int, error) {
+	return q.conn.Configuration().scripts().inflightLimit(q.name)
+}
+
+func (q *Queue) SetInflightLimit(limit int) error {
+	return q.conn.Configuration().scripts().setInflightLimit(q.name, limit)
+}
+
+func (q *Queue) Ping(message *Msg, resendTimeframe int) error {
+	return q.conn.Configuration().scripts().ping(q.name, message, resendTimeframe)
+}
+
+func (q *Queue) Ack(message *Msg) error {
+	facet := q.conn.Configuration().Facet(message)
+	return q.conn.Configuration().scripts().ack(q.name, facet, message)
 }