fairway 0.3.5 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +8 -8
  2. data/Gemfile.lock +1 -1
  3. data/go/fairway_pull.go +39 -22
  4. data/go/queue.go +13 -1
  5. data/go/queue_test.go +430 -80
  6. data/go/scripts.go +39 -35
  7. data/lib/fairway/config.rb +13 -2
  8. data/lib/fairway/scripts.rb +1 -1
  9. data/lib/fairway/version.rb +1 -1
  10. metadata +2 -2
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- OTY4Y2NjNWRkYTM0NzYwY2U3NWRmNzhmMWE3ZmVhY2M3MTlhMjc0Mg==
4
+ OGQ1NDFhZjE3MDk0MmIzN2RmNThmN2IzZDczYTliY2M5ZjNiOGIxMQ==
5
5
  data.tar.gz: !binary |-
6
- NzJjNmNjMTNmMGUxN2MyNjNlZWQ2MGE4MTY3OTg5NWMxZjU2NzZkYg==
6
+ OGM2NTcyYjMyODQwOGUwYTI3OGQxYmZmZjYxY2I0MDYzY2FmOTQ1YQ==
7
7
  SHA512:
8
8
  metadata.gz: !binary |-
9
- ZWZmNmY3YmI5NDkyZGZmMDBhNWZkZDEyOGY3MWYxNmMzMWE3YTM3ZDRmNjdk
10
- NGZmMzMwZjk3NDdmNGI5YTIzYjI2YzkyN2M0Nzc0YjA2Yzc4MDYxZTllZWE2
11
- MWYwMDdhZmVmODE1ODcyZTRiNzViZDQ3ZWJiNDZlNGM2Y2M2ZTk=
9
+ NmI3ZWRjNTQxMWUyODVjY2Q4YzJlODg0MTBiY2FkNzZmMDJjZTVkYzM2ZmRh
10
+ MjUyYTM3ZDJlOWUyYjFkYTE0YjlkMGI3MjNiZWFiNzFjNmUxZTgwZTcyYWZk
11
+ OTQxMWI1NTA1OGRjY2QwY2Y2NWYzYjg1NTExNWQ2NzUyY2EwNzE=
12
12
  data.tar.gz: !binary |-
13
- Mzk4MDdhNGNhNWU1ZWM3OGVjZTZkZTI5YzMxNzE3YWY2NDM4YmExMzRiMjI4
14
- M2U5YzBmZjFjYjVlNzE3MjhmZDlkZTQwYTBmMGU3MTM3OWViZjFlNjQzNGY4
15
- MzMwOTM3ZjU0ODQ1YWFlZjZjNmJhYmU1MTdiMjQ3N2RjMTM5NjM=
13
+ MGQ4OTQwNzU1MGVhYzVkZDM5MjMxZDlkMGE5NDg2NTZjZTFjODgyZDc1NGFj
14
+ YTM4MDYzZDk2MGEyMWNiNWZjMjY5NTE4MzkwMzk3NzY5ZWMzNjcwNWNlMDJl
15
+ YWRmZjgxYzJmYWZkMzBkNjU5YmE1ZGNkZDE2YjRhMWFiNGQ2MjM=
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fairway (0.3.4)
4
+ fairway (0.3.5)
5
5
  activesupport
6
6
  connection_pool
7
7
  redis
data/go/fairway_pull.go CHANGED
@@ -4,13 +4,14 @@ func FairwayPull() string {
4
4
  return `
5
5
  local namespace = KEYS[1];
6
6
  local timestamp = tonumber(KEYS[2]);
7
- local wait = tonumber(KEYS[3]);
7
+ local n = tonumber(KEYS[3]);
8
+ local wait = tonumber(KEYS[4]);
8
9
 
9
10
  local k = function (queue, subkey)
10
11
  return namespace .. queue .. ':' .. subkey;
11
12
  end
12
13
 
13
- local pull = function (queue)
14
+ local pull = function (queue, n)
14
15
  local round_robin = k(queue, 'facet_queue');
15
16
  local inflight = k(queue, 'inflight');
16
17
 
@@ -18,7 +19,7 @@ local pull = function (queue)
18
19
  -- This list guarantees each active facet will have a
19
20
  -- message pulled from the queue every time through..
20
21
  local facet = redis.call('rpop', round_robin);
21
- local message = nil
22
+ local msgs = {};
22
23
 
23
24
  if facet then
24
25
  -- If we found an active facet, we know the facet
@@ -27,19 +28,23 @@ local pull = function (queue)
27
28
  local messages = k(queue, facet);
28
29
  local inflight_facet = k(queue, facet .. ':inflight');
29
30
 
30
- message = redis.call('rpop', messages);
31
+ for i = 1, n, 1 do
32
+ local msg = redis.call('rpop', messages);
31
33
 
32
- if message then
33
- if wait ~= -1 then
34
- redis.call('zadd', inflight, timestamp + wait, message);
35
- redis.call('sadd', inflight_facet, message);
36
- end
34
+ if msg then
35
+ if wait ~= -1 then
36
+ redis.call('zadd', inflight, timestamp + wait, msg);
37
+ redis.call('sadd', inflight_facet, msg);
38
+ end
37
39
 
38
- redis.call('decr', k(queue, 'length'));
40
+ table.insert(msgs, msg);
41
+ end
39
42
  end
43
+
44
+ redis.call('decrby', k(queue, 'length'), #msgs);
40
45
  end
41
46
 
42
- return {facet, message};
47
+ return {facet, msgs};
43
48
  end
44
49
 
45
50
  local manage = function (queue, facet)
@@ -77,7 +82,7 @@ local manage = function (queue, facet)
77
82
  redis.call('lpush', round_robin, facet);
78
83
  redis.call('lpush', round_robin, facet);
79
84
  redis.call('hset', facet_pool, facet, current + 1);
80
- else
85
+ elseif n > 0 then
81
86
  -- redis.log(redis.LOG_WARNING, "maintaining");
82
87
  redis.call('lpush', round_robin, facet);
83
88
  end
@@ -99,39 +104,51 @@ for i, queue in ipairs(ARGV) do
99
104
  if wait ~= -1 then
100
105
  -- Check if any current inflight messages
101
106
  -- have been inflight for a long time.
102
- local inflightmessage = redis.call('zrange', inflight, 0, 0, 'WITHSCORES');
107
+ local inflightmessages = redis.call('zrange', inflight, 0, n-1, 'WITHSCORES');
108
+
109
+ local msgs = {}
103
110
 
104
111
  -- If we have an inflight message and it's score
105
112
  -- is less than the current pull timestamp, reset
106
113
  -- the inflight score for the the message and resend.
107
- if #inflightmessage > 0 then
108
- if tonumber(inflightmessage[2]) <= timestamp then
109
- redis.call('zadd', inflight, timestamp + wait, inflightmessage[1]);
110
- return {queue, inflightmessage[1]}
114
+ if #inflightmessages > 0 then
115
+ for i = 1, #inflightmessages, 2 do
116
+ local msg = inflightmessages[i];
117
+ local ts = tonumber(inflightmessages[i+1]);
118
+
119
+ if tonumber(ts) <= timestamp then
120
+ redis.call('zadd', inflight, timestamp + wait, msg);
121
+ table.insert(msgs, msg);
122
+
123
+ end
111
124
  end
112
125
  end
126
+
127
+ if #msgs > 0 then
128
+ return {queue, msgs}
129
+ end
113
130
  end
114
131
 
115
- local pulled = pull(queue);
132
+ local pulled = pull(queue, n);
116
133
  local facet = pulled[1];
117
- local message = pulled[2];
134
+ local msgs = pulled[2];
118
135
 
119
136
  if facet then
120
137
  manage(queue, facet);
121
138
 
122
- -- if message then
139
+ -- if #msgs > 0 then
123
140
  -- else
124
141
  -- -- TODO loop through until we find a message
125
142
  -- pulled = pull(queue);
126
143
  -- facet = pulled[1];
127
- -- message = pulled[2];
144
+ -- message = pulled[2];
128
145
 
129
146
  -- if facet then
130
147
  -- manage(queue, facet);
131
148
  -- end
132
149
  -- end
133
150
 
134
- return {queue, message};
151
+ return {queue, msgs};
135
152
  end
136
153
  end
137
154
  `
data/go/queue.go CHANGED
@@ -18,7 +18,19 @@ func (q *Queue) Length() (int, error) {
18
18
  }
19
19
 
20
20
  func (q *Queue) Pull(resendTimeframe int) (string, *Msg) {
21
- return q.conn.Configuration().scripts().pull(q.name, resendTimeframe)
21
+ name, msgs := q.conn.Configuration().scripts().pull(q.name, 1, resendTimeframe)
22
+
23
+ var m *Msg
24
+
25
+ if len(msgs) > 0 {
26
+ m = msgs[0]
27
+ }
28
+
29
+ return name, m
30
+ }
31
+
32
+ func (q *Queue) PullN(n, resendTimeframe int) (string, []*Msg) {
33
+ return q.conn.Configuration().scripts().pull(q.name, n, resendTimeframe)
22
34
  }
23
35
 
24
36
  func (q *Queue) Inflight() []string {
data/go/queue_test.go CHANGED
@@ -62,86 +62,87 @@ func QueueSpec(c gospec.Context) {
62
62
  c.Expect(message.json(), Equals, msg2.json())
63
63
  })
64
64
 
65
- c.Specify("skips over facets in invalid state", func() {
66
- config.Facet = func(msg *Msg) string {
67
- str, _ := msg.Get("facet").String()
68
- return str
69
- }
70
-
71
- msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
72
- msg2, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage2"})
73
- msg3, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage3"})
74
-
75
- conn.Deliver(msg1)
76
- conn.Deliver(msg2)
77
- conn.Deliver(msg3)
78
-
79
- r := config.Pool.Get()
80
- defer r.Close()
81
-
82
- count, _ := redis.Int(r.Do("llen", "fairway:myqueue:1"))
83
- c.Expect(count, Equals, 2)
84
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
85
- c.Expect(count, Equals, 1)
86
- count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
87
- c.Expect(count, Equals, 2)
88
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
89
- c.Expect(count, Equals, 2)
90
- count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
91
- c.Expect(count, Equals, 1)
92
- count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
93
- c.Expect(count, Equals, 1)
94
-
95
- queueName, message := queue.Pull(-1)
96
- c.Expect(queueName, Equals, "myqueue")
97
- c.Expect(message.json(), Equals, msg1.json())
98
-
99
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
100
- c.Expect(count, Equals, 1)
101
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
102
- c.Expect(count, Equals, 1)
103
- count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
104
- c.Expect(count, Equals, 2)
105
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
106
- c.Expect(count, Equals, 2)
107
- count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
108
- c.Expect(count, Equals, 1)
109
- count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
110
- c.Expect(count, Equals, 1)
111
-
112
- // We expect a message to be in here
113
- r.Do("del", "fairway:myqueue:2")
114
-
115
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
116
- c.Expect(count, Equals, 1)
117
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
118
- c.Expect(count, Equals, 0)
119
- count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
120
- c.Expect(count, Equals, 2)
121
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
122
- c.Expect(count, Equals, 2)
123
- count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
124
- c.Expect(count, Equals, 1)
125
- count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
126
- c.Expect(count, Equals, 1)
127
-
128
- queueName, message = queue.Pull(-1)
129
- c.Expect(queueName, Equals, "myqueue")
130
- c.Expect(message.json(), Equals, msg3.json())
131
-
132
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
133
- c.Expect(count, Equals, 0)
134
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
135
- c.Expect(count, Equals, 0)
136
- count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
137
- c.Expect(count, Equals, 0)
138
- count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
139
- c.Expect(count, Equals, 0)
140
- count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
141
- c.Expect(count, Equals, 0)
142
- count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
143
- c.Expect(count, Equals, 0)
144
- })
65
+ // TODO
66
+ //c.Specify("skips over facets in invalid state", func() {
67
+ // config.Facet = func(msg *Msg) string {
68
+ // str, _ := msg.Get("facet").String()
69
+ // return str
70
+ // }
71
+
72
+ // msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
73
+ // msg2, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage2"})
74
+ // msg3, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage3"})
75
+
76
+ // conn.Deliver(msg1)
77
+ // conn.Deliver(msg2)
78
+ // conn.Deliver(msg3)
79
+
80
+ // r := config.Pool.Get()
81
+ // defer r.Close()
82
+
83
+ // count, _ := redis.Int(r.Do("llen", "fairway:myqueue:1"))
84
+ // c.Expect(count, Equals, 2)
85
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
86
+ // c.Expect(count, Equals, 1)
87
+ // count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
88
+ // c.Expect(count, Equals, 2)
89
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
90
+ // c.Expect(count, Equals, 2)
91
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
92
+ // c.Expect(count, Equals, 1)
93
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
94
+ // c.Expect(count, Equals, 1)
95
+
96
+ // queueName, message := queue.Pull(-1)
97
+ // c.Expect(queueName, Equals, "myqueue")
98
+ // c.Expect(message.json(), Equals, msg1.json())
99
+
100
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
101
+ // c.Expect(count, Equals, 1)
102
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
103
+ // c.Expect(count, Equals, 1)
104
+ // count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
105
+ // c.Expect(count, Equals, 2)
106
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
107
+ // c.Expect(count, Equals, 2)
108
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
109
+ // c.Expect(count, Equals, 1)
110
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
111
+ // c.Expect(count, Equals, 1)
112
+
113
+ // // We expect a message to be in here
114
+ // r.Do("del", "fairway:myqueue:2")
115
+
116
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
117
+ // c.Expect(count, Equals, 1)
118
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
119
+ // c.Expect(count, Equals, 0)
120
+ // count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
121
+ // c.Expect(count, Equals, 2)
122
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
123
+ // c.Expect(count, Equals, 2)
124
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
125
+ // c.Expect(count, Equals, 1)
126
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
127
+ // c.Expect(count, Equals, 1)
128
+
129
+ // queueName, message = queue.Pull(-1)
130
+ // c.Expect(queueName, Equals, "myqueue")
131
+ // c.Expect(message.json(), Equals, msg3.json())
132
+
133
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
134
+ // c.Expect(count, Equals, 0)
135
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
136
+ // c.Expect(count, Equals, 0)
137
+ // count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
138
+ // c.Expect(count, Equals, 0)
139
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
140
+ // c.Expect(count, Equals, 0)
141
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
142
+ // c.Expect(count, Equals, 0)
143
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
144
+ // c.Expect(count, Equals, 0)
145
+ //})
145
146
 
146
147
  c.Specify("places pulled message on inflight sorted set until acknowledged", func() {
147
148
  msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
@@ -620,4 +621,353 @@ func QueueSpec(c gospec.Context) {
620
621
  c.Expect(message, IsNil)
621
622
  })
622
623
  })
624
+
625
+ c.Specify("PullN", func() {
626
+ c.Specify("pulls a message off the queue using FIFO", func() {
627
+ msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
628
+ msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
629
+ msg3, _ := NewMsg(map[string]string{"name": "mymessage3"})
630
+
631
+ conn.Deliver(msg1)
632
+ conn.Deliver(msg2)
633
+ conn.Deliver(msg3)
634
+
635
+ r := config.Pool.Get()
636
+ defer r.Close()
637
+
638
+ count, _ := redis.Int(r.Do("llen", "fairway:myqueue:default"))
639
+ c.Expect(count, Equals, 3)
640
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
641
+ c.Expect(count, Equals, 1)
642
+ count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
643
+ c.Expect(count, Equals, 1)
644
+ count, _ = redis.Int(r.Do("get", "fairway:myqueue:limit"))
645
+ c.Expect(count, Equals, 0)
646
+ count, _ = redis.Int(r.Do("get", "fairway:myqueue:inflight"))
647
+ c.Expect(count, Equals, 0)
648
+ count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "default"))
649
+ c.Expect(count, Equals, 1)
650
+
651
+ queueName, messages := queue.PullN(2, -1)
652
+
653
+ count, _ = redis.Int(r.Do("llen", "fairway:myqueue:default"))
654
+ c.Expect(count, Equals, 1)
655
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
656
+ c.Expect(count, Equals, 1)
657
+ count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
658
+ c.Expect(count, Equals, 1)
659
+ count, _ = redis.Int(r.Do("get", "fairway:myqueue:limit"))
660
+ c.Expect(count, Equals, 0)
661
+ count, _ = redis.Int(r.Do("get", "fairway:myqueue:inflight"))
662
+ c.Expect(count, Equals, 0)
663
+ count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "default"))
664
+ c.Expect(count, Equals, 1)
665
+
666
+ c.Expect(queueName, Equals, "myqueue")
667
+ c.Expect(messages[0].json(), Equals, msg1.json())
668
+ c.Expect(messages[1].json(), Equals, msg2.json())
669
+
670
+ queueName, messages = queue.PullN(2, -1)
671
+ c.Expect(queueName, Equals, "myqueue")
672
+ c.Expect(messages[0].json(), Equals, msg3.json())
673
+
674
+ queueName, messages = queue.PullN(2, -1)
675
+ c.Expect(queueName, Equals, "")
676
+ c.Expect(len(messages), Equals, 0)
677
+ })
678
+
679
+ // TODO
680
+ //c.Specify("skips over facets in invalid state", func() {
681
+ // config.Facet = func(msg *Msg) string {
682
+ // str, _ := msg.Get("facet").String()
683
+ // return str
684
+ // }
685
+
686
+ // msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
687
+ // msg2, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage2"})
688
+ // msg3, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage3"})
689
+
690
+ // conn.Deliver(msg1)
691
+ // conn.Deliver(msg2)
692
+ // conn.Deliver(msg3)
693
+
694
+ // r := config.Pool.Get()
695
+ // defer r.Close()
696
+
697
+ // count, _ := redis.Int(r.Do("llen", "fairway:myqueue:1"))
698
+ // c.Expect(count, Equals, 2)
699
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
700
+ // c.Expect(count, Equals, 1)
701
+ // count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
702
+ // c.Expect(count, Equals, 2)
703
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
704
+ // c.Expect(count, Equals, 2)
705
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
706
+ // c.Expect(count, Equals, 1)
707
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
708
+ // c.Expect(count, Equals, 1)
709
+
710
+ // queueName, message := queue.Pull(-1)
711
+ // c.Expect(queueName, Equals, "myqueue")
712
+ // c.Expect(message.json(), Equals, msg1.json())
713
+
714
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
715
+ // c.Expect(count, Equals, 1)
716
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
717
+ // c.Expect(count, Equals, 1)
718
+ // count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
719
+ // c.Expect(count, Equals, 2)
720
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
721
+ // c.Expect(count, Equals, 2)
722
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
723
+ // c.Expect(count, Equals, 1)
724
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
725
+ // c.Expect(count, Equals, 1)
726
+
727
+ // // We expect a message to be in here
728
+ // r.Do("del", "fairway:myqueue:2")
729
+
730
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
731
+ // c.Expect(count, Equals, 1)
732
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
733
+ // c.Expect(count, Equals, 0)
734
+ // count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
735
+ // c.Expect(count, Equals, 2)
736
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
737
+ // c.Expect(count, Equals, 2)
738
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
739
+ // c.Expect(count, Equals, 1)
740
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
741
+ // c.Expect(count, Equals, 1)
742
+
743
+ // queueName, message = queue.Pull(-1)
744
+ // c.Expect(queueName, Equals, "myqueue")
745
+ // c.Expect(message.json(), Equals, msg3.json())
746
+
747
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:1"))
748
+ // c.Expect(count, Equals, 0)
749
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:2"))
750
+ // c.Expect(count, Equals, 0)
751
+ // count, _ = redis.Int(r.Do("scard", "fairway:myqueue:active_facets"))
752
+ // c.Expect(count, Equals, 0)
753
+ // count, _ = redis.Int(r.Do("llen", "fairway:myqueue:facet_queue"))
754
+ // c.Expect(count, Equals, 0)
755
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "1"))
756
+ // c.Expect(count, Equals, 0)
757
+ // count, _ = redis.Int(r.Do("hget", "fairway:myqueue:facet_pool", "2"))
758
+ // c.Expect(count, Equals, 0)
759
+ //})
760
+
761
+ c.Specify("places pulled message on inflight sorted set until acknowledged", func() {
762
+ msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
763
+ msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
764
+ msg3, _ := NewMsg(map[string]string{"name": "mymessage3"})
765
+
766
+ conn.Deliver(msg1)
767
+ conn.Deliver(msg2)
768
+ conn.Deliver(msg3)
769
+
770
+ c.Expect(len(queue.Inflight()), Equals, 0)
771
+
772
+ queueName, messages := queue.PullN(2, 100)
773
+ c.Expect(queueName, Equals, "myqueue")
774
+ c.Expect(len(messages), Equals, 2)
775
+ c.Expect(messages[0].json(), Equals, msg1.json())
776
+ c.Expect(messages[1].json(), Equals, msg2.json())
777
+
778
+ c.Expect(len(queue.Inflight()), Equals, 2)
779
+ c.Expect(queue.Inflight()[0], Equals, msg1.json())
780
+ c.Expect(queue.Inflight()[1], Equals, msg2.json())
781
+
782
+ queue.Ack(msg1)
783
+
784
+ c.Expect(len(queue.Inflight()), Equals, 1)
785
+ c.Expect(queue.Inflight()[0], Equals, msg2.json())
786
+
787
+ queueName, messages = queue.PullN(2, 100)
788
+ c.Expect(queueName, Equals, "myqueue")
789
+ c.Expect(len(messages), Equals, 1)
790
+ c.Expect(messages[0].json(), Equals, msg3.json())
791
+
792
+ c.Expect(len(queue.Inflight()), Equals, 2)
793
+ c.Expect(queue.Inflight()[0], Equals, msg2.json())
794
+ c.Expect(queue.Inflight()[1], Equals, msg3.json())
795
+ })
796
+
797
+ c.Specify("pulls from inflight message set if messages are unacknowledged", func() {
798
+ msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
799
+ msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
800
+ msg3, _ := NewMsg(map[string]string{"name": "mymessage3"})
801
+
802
+ conn.Deliver(msg1)
803
+ conn.Deliver(msg2)
804
+ conn.Deliver(msg3)
805
+
806
+ queueName, messages := queue.PullN(2, 0)
807
+ c.Expect(len(messages), Equals, 2)
808
+ c.Expect(queueName, Equals, "myqueue")
809
+ c.Expect(messages[0].json(), Equals, msg1.json())
810
+ c.Expect(messages[1].json(), Equals, msg2.json())
811
+
812
+ queueName, messages = queue.PullN(1, 0)
813
+ c.Expect(len(messages), Equals, 1)
814
+ c.Expect(queueName, Equals, "myqueue")
815
+ c.Expect(messages[0].json(), Equals, msg1.json())
816
+
817
+ queueName, messages = queue.PullN(2, 0)
818
+ c.Expect(len(messages), Equals, 2)
819
+ c.Expect(queueName, Equals, "myqueue")
820
+ c.Expect(messages[0].json(), Equals, msg1.json())
821
+ c.Expect(messages[1].json(), Equals, msg2.json())
822
+
823
+ queueName, messages = queue.PullN(2, 10)
824
+ c.Expect(len(messages), Equals, 2)
825
+ c.Expect(queueName, Equals, "myqueue")
826
+ c.Expect(messages[0].json(), Equals, msg1.json())
827
+ c.Expect(messages[1].json(), Equals, msg2.json())
828
+
829
+ queueName, messages = queue.PullN(2, 10)
830
+ c.Expect(queueName, Equals, "myqueue")
831
+ c.Expect(messages[0].json(), Equals, msg3.json())
832
+ })
833
+
834
+ c.Specify("allows puller to ping to keep message inflight", func() {
835
+ msg1, _ := NewMsg(map[string]string{"name": "mymessage1"})
836
+ msg2, _ := NewMsg(map[string]string{"name": "mymessage2"})
837
+ msg3, _ := NewMsg(map[string]string{"name": "mymessage3"})
838
+
839
+ conn.Deliver(msg1)
840
+ conn.Deliver(msg2)
841
+ conn.Deliver(msg3)
842
+
843
+ queueName, messages := queue.PullN(2, 0)
844
+ c.Expect(len(messages), Equals, 2)
845
+ c.Expect(queueName, Equals, "myqueue")
846
+ c.Expect(messages[0].json(), Equals, msg1.json())
847
+ c.Expect(messages[1].json(), Equals, msg2.json())
848
+
849
+ // Extends time before message is resent
850
+ queue.Ping(msg1, 10)
851
+
852
+ queueName, messages = queue.PullN(2, 10)
853
+ c.Expect(queueName, Equals, "myqueue")
854
+ c.Expect(len(messages), Equals, 1)
855
+ c.Expect(messages[0].json(), Equals, msg2.json())
856
+
857
+ // Sets time for message to resend to now
858
+ queue.Ping(msg1, 0)
859
+
860
+ queueName, messages = queue.PullN(2, 10)
861
+ c.Expect(queueName, Equals, "myqueue")
862
+ c.Expect(len(messages), Equals, 1)
863
+ c.Expect(messages[0].json(), Equals, msg1.json())
864
+
865
+ queueName, messages = queue.PullN(2, 10)
866
+ c.Expect(queueName, Equals, "myqueue")
867
+ c.Expect(len(messages), Equals, 1)
868
+ c.Expect(messages[0].json(), Equals, msg3.json())
869
+ })
870
+
871
+ c.Specify("limits messages inflight", func() {
872
+ r := config.Pool.Get()
873
+ defer r.Close()
874
+
875
+ config.Facet = func(msg *Msg) string {
876
+ str, _ := msg.Get("facet").String()
877
+ return str
878
+ }
879
+
880
+ msg1, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage1"})
881
+ msg2, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage2"})
882
+ msg3, _ := NewMsg(map[string]string{"facet": "2", "name": "mymessage3"})
883
+ msg4, _ := NewMsg(map[string]string{"facet": "1", "name": "mymessage4"})
884
+
885
+ conn.Deliver(msg1)
886
+ conn.Deliver(msg2)
887
+ conn.Deliver(msg3)
888
+ conn.Deliver(msg4)
889
+
890
+ queue.SetInflightLimit(1)
891
+
892
+ _, messages := queue.PullN(2, 2)
893
+ c.Expect(len(messages), Equals, 2)
894
+ c.Expect(messages[0].json(), Equals, msg1.json())
895
+ c.Expect(messages[1].json(), Equals, msg2.json())
896
+
897
+ count, _ := redis.Int(r.Do("scard", "fairway:myqueue:1:inflight"))
898
+ c.Expect(count, Equals, 2)
899
+
900
+ _, messages = queue.PullN(2, 2)
901
+ c.Expect(len(messages), Equals, 1)
902
+ c.Expect(messages[0].json(), Equals, msg3.json())
903
+
904
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:1:inflight"))
905
+ c.Expect(count, Equals, 2)
906
+
907
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:2:inflight"))
908
+ c.Expect(count, Equals, 1)
909
+
910
+ _, messages = queue.PullN(2, 2)
911
+ c.Expect(len(messages), Equals, 0)
912
+ _, messages = queue.PullN(2, 2)
913
+ c.Expect(len(messages), Equals, 0)
914
+
915
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:1:inflight"))
916
+ c.Expect(count, Equals, 2)
917
+
918
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:2:inflight"))
919
+ c.Expect(count, Equals, 1)
920
+
921
+ queue.Ack(msg1)
922
+ queue.Ack(msg1)
923
+ queue.Ack(msg1)
924
+ queue.Ack(msg1)
925
+ queue.Ack(msg1)
926
+
927
+ _, messages = queue.PullN(2, 2)
928
+ c.Expect(len(messages), Equals, 0)
929
+ _, messages = queue.PullN(2, 2)
930
+ c.Expect(len(messages), Equals, 0)
931
+
932
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:1:inflight"))
933
+ c.Expect(count, Equals, 1)
934
+
935
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:2:inflight"))
936
+ c.Expect(count, Equals, 1)
937
+
938
+ queue.Ack(msg2)
939
+ queue.Ack(msg2)
940
+ queue.Ack(msg2)
941
+ queue.Ack(msg2)
942
+ queue.Ack(msg2)
943
+
944
+ count, err := redis.Int(r.Do("scard", "fairway:myqueue:1:inflight"))
945
+ c.Expect(count, Equals, 0)
946
+ c.Expect(err, IsNil)
947
+
948
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:2:inflight"))
949
+ c.Expect(count, Equals, 1)
950
+
951
+ _, messages = queue.PullN(2, 2)
952
+ c.Expect(len(messages), Equals, 1)
953
+ c.Expect(messages[0].json(), Equals, msg4.json())
954
+
955
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:1:inflight"))
956
+ c.Expect(count, Equals, 1)
957
+
958
+ count, _ = redis.Int(r.Do("scard", "fairway:myqueue:2:inflight"))
959
+ c.Expect(count, Equals, 1)
960
+ })
961
+
962
+ c.Specify("returns empty array if there are no messages to receive", func() {
963
+ msg, _ := NewMsg(map[string]string{})
964
+ conn.Deliver(msg)
965
+
966
+ queueName, messages := queue.PullN(1, -1)
967
+ c.Expect(queueName, Equals, "myqueue")
968
+ queueName, messages = queue.PullN(1, -1)
969
+ c.Expect(queueName, Equals, "")
970
+ c.Expect(len(messages), Equals, 0)
971
+ })
972
+ })
623
973
  }
data/go/scripts.go CHANGED
@@ -9,11 +9,23 @@ import (
9
9
 
10
10
  type scripts struct {
11
11
  config *Config
12
- data map[string]*redis.Script
12
+
13
+ deliverScript *redis.Script
14
+ pullScript *redis.Script
15
+ inflightScript *redis.Script
16
+ pingScript *redis.Script
17
+ ackScript *redis.Script
13
18
  }
14
19
 
15
20
  func newScripts(config *Config) *scripts {
16
- return &scripts{config, make(map[string]*redis.Script)}
21
+ return &scripts{
22
+ config: config,
23
+ deliverScript: redis.NewScript(1, FairwayDeliver()),
24
+ pullScript: redis.NewScript(4, FairwayPull()),
25
+ inflightScript: redis.NewScript(1, FairwayInflight()),
26
+ pingScript: redis.NewScript(3, FairwayPing()),
27
+ ackScript: redis.NewScript(1, FairwayAck()),
28
+ }
17
29
  }
18
30
 
19
31
  func (s *scripts) namespace() string {
@@ -51,9 +63,7 @@ func (s *scripts) deliver(channel, facet string, msg *Msg) error {
51
63
  conn := s.config.Pool.Get()
52
64
  defer conn.Close()
53
65
 
54
- script := s.findScript(FairwayDeliver, 1)
55
-
56
- _, err := script.Do(conn, s.namespace(), channel, facet, msg.json())
66
+ _, err := s.deliverScript.Do(conn, s.namespace(), channel, facet, msg.json())
57
67
 
58
68
  return err
59
69
  }
@@ -62,9 +72,7 @@ func (s *scripts) deliverBytes(channel, facet string, msg []byte) error {
62
72
  conn := s.config.Pool.Get()
63
73
  defer conn.Close()
64
74
 
65
- script := s.findScript(FairwayDeliver, 1)
66
-
67
- _, err := script.Do(conn, s.namespace(), channel, facet, string(msg))
75
+ _, err := s.deliverScript.Do(conn, s.namespace(), channel, facet, string(msg))
68
76
 
69
77
  return err
70
78
  }
@@ -75,31 +83,40 @@ func (s *scripts) length(queue string) (int, error) {
75
83
  return redis.Int(conn.Do("get", s.namespace()+queue+":length"))
76
84
  }
77
85
 
78
- func (s *scripts) pull(queueName string, wait int) (string, *Msg) {
86
+ func (s *scripts) pull(queueName string, n, wait int) (string, []*Msg) {
79
87
  conn := s.config.Pool.Get()
80
88
  defer conn.Close()
81
89
 
82
- script := s.findScript(FairwayPull, 3)
83
-
84
- result, err := redis.Strings(script.Do(conn, s.namespace(), int(time.Now().Unix()), wait, queueName))
90
+ if n <= 0 {
91
+ n = 1
92
+ }
85
93
 
86
- if err != nil {
94
+ r, err := s.pullScript.Do(conn, s.namespace(), int(time.Now().Unix()), n, wait, queueName)
95
+ if err != nil || r == nil {
87
96
  return "", nil
88
97
  }
89
98
 
90
- queue := result[0]
91
- message, _ := NewMsgFromString(result[1])
99
+ result := r.([]interface{})
100
+
101
+ queue := string(result[0].([]byte))
92
102
 
93
- return queue, message
103
+ messages := make([]*Msg, 0, len(result[1].([]interface{})))
104
+
105
+ for _, m := range result[1].([]interface{}) {
106
+ if m != nil {
107
+ msg, _ := NewMsgFromString(string(m.([]byte)))
108
+ messages = append(messages, msg)
109
+ }
110
+ }
111
+
112
+ return queue, messages
94
113
  }
95
114
 
96
115
  func (s *scripts) inflight(queueName string) []string {
97
116
  conn := s.config.Pool.Get()
98
117
  defer conn.Close()
99
118
 
100
- script := s.findScript(FairwayInflight, 1)
101
-
102
- result, err := redis.Strings(script.Do(conn, s.namespace(), queueName))
119
+ result, err := redis.Strings(s.inflightScript.Do(conn, s.namespace(), queueName))
103
120
 
104
121
  if err != nil {
105
122
  return []string{}
@@ -134,9 +151,8 @@ func (s *scripts) ping(queueName string, message *Msg, wait int) error {
134
151
  conn := s.config.Pool.Get()
135
152
  defer conn.Close()
136
153
 
137
- script := s.findScript(FairwayPing, 3)
138
-
139
- _, err := redis.Strings(script.Do(conn, s.namespace(), int(time.Now().Unix()), wait, queueName, message.Original))
154
+ _, err := redis.Strings(s.pingScript.Do(conn, s.namespace(), int(time.Now().Unix()),
155
+ wait, queueName, message.Original))
140
156
 
141
157
  return err
142
158
  }
@@ -145,19 +161,7 @@ func (s *scripts) ack(queueName string, facet string, message *Msg) error {
145
161
  conn := s.config.Pool.Get()
146
162
  defer conn.Close()
147
163
 
148
- script := s.findScript(FairwayAck, 1)
149
-
150
- _, err := redis.Strings(script.Do(conn, s.namespace(), queueName, facet, message.Original))
164
+ _, err := redis.Strings(s.ackScript.Do(conn, s.namespace(), queueName, facet, message.Original))
151
165
 
152
166
  return err
153
167
  }
154
-
155
- func (s *scripts) findScript(script func() string, keyCount int) *redis.Script {
156
- content := script()
157
-
158
- if s.data[content] == nil {
159
- s.data[content] = redis.NewScript(keyCount, content)
160
- }
161
-
162
- return s.data[content]
163
- }
data/lib/fairway/config.rb CHANGED
@@ -23,7 +23,8 @@ module Fairway
23
23
  pool.with do |conn|
24
24
  begin
25
25
  return yield(conn)
26
- rescue *EXCEPTIONS
26
+ rescue *EXCEPTIONS => e
27
+ puts "FAIRWAY WITH EXCEPTION: #{e}"
27
28
  valid_pools -= [pool]
28
29
  end
29
30
  end
@@ -32,15 +33,25 @@ module Fairway
32
33
  raise CannotConnect.new
33
34
  end
34
35
 
35
- def with_each(&block)
36
+ def with_each_running(&block)
36
37
  @pools.shuffle.each do |pool|
37
38
  pool.with do |conn|
38
39
  begin
39
40
  yield(conn)
41
+ rescue *EXCEPTIONS => e
42
+ puts "FAIRWAY WITH EACH RUNNING EXCEPTION: #{e}"
40
43
  end
41
44
  end
42
45
  end
43
46
  end
47
+
48
+ def with_each(&block)
49
+ @pools.shuffle.each do |pool|
50
+ pool.with do |conn|
51
+ yield(conn)
52
+ end
53
+ end
54
+ end
44
55
  end
45
56
 
46
57
  class Config
data/lib/fairway/scripts.rb CHANGED
@@ -92,7 +92,7 @@ module Fairway
92
92
  end
93
93
 
94
94
  def first_pool(&block)
95
- redis.with_each do |conn|
95
+ redis.with_each_running do |conn|
96
96
  val = yield(conn)
97
97
  return val if val
98
98
  end
data/lib/fairway/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Fairway
2
- VERSION = "0.3.5"
2
+ VERSION = "0.3.6"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fairway
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 0.3.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - John Allison
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-02-27 00:00:00.000000000 Z
11
+ date: 2016-07-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport