fairway 0.2.7 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,39 +19,45 @@ for i = 1, #registered_queues, 2 do
19
19
  -- If the message topic matches the queue topic,
20
20
  -- we deliver the message to the queue.
21
21
  if string.find(topic, queue_topic) then
22
- local priorities = k(queue, 'priorities');
23
- local active_facets = k(queue, 'active_facets');
24
- local round_robin = k(queue, 'facet_queue');
25
- local facet_pool = k(queue, 'facet_pool');
22
+ local priorities = k(queue, 'priorities');
23
+ local active_facets = k(queue, 'active_facets');
24
+ local round_robin = k(queue, 'facet_queue');
25
+ local facet_pool = k(queue, 'facet_pool');
26
+ local inflight_total = k(queue, facet .. ':inflight');
27
+ local inflight_limit = k(queue, 'limit');
26
28
 
27
29
  -- Delivering the message to a queue is as simple as
28
30
  -- pushing it onto the facet's message list, and
29
31
  -- incrementing the length of the queue itself.
30
- redis.call('lpush', k(queue, facet), message)
32
+ local length = redis.call('lpush', k(queue, facet), message)
31
33
  redis.call('incr', k(queue, 'length'));
32
34
 
33
- -- If the facet just became active, we need to add
34
- -- the facet to the round-robin queue, so it's
35
- -- messages will be processed.
36
- if redis.call('sadd', active_facets, facet) == 1 then
37
- local priority = tonumber(redis.call('hget', priorities, facet)) or 1
38
-
39
- -- If the facet currently has a priority
40
- -- we need to jump start the facet by adding
41
- -- it to the round-robin queue and updating
42
- -- the current priority.
43
- if priority > 0 then
44
- redis.call('lpush', round_robin, facet);
45
- redis.call('hset', facet_pool, facet, 1);
46
-
47
- -- If the facet has no set priority, just set the
48
- -- current priority to zero. Since the facet just
49
- -- became active, we can be sure it's already zero
50
- -- or undefined.
51
- else
52
- redis.call('hset', facet_pool, facet, 0);
53
- end
35
+
36
+ -- Manage facet queue and active facets
37
+ local current = tonumber(redis.call('hget', facet_pool, facet)) or 0;
38
+ local priority = tonumber(redis.call('hget', priorities, facet)) or 1;
39
+ local inflight_cur = tonumber(redis.call('get', inflight_total)) or 0;
40
+ local inflight_max = tonumber(redis.call('get', inflight_limit)) or 0;
41
+
42
+ local n = 0
43
+
44
+ -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
45
+
46
+ if inflight_max > 0 then
47
+ n = math.min(length, priority, inflight_max - inflight_cur);
48
+ else
49
+ n = math.min(length, priority);
54
50
  end
51
+
52
+ -- redis.log(redis.LOG_WARNING, "PUSH: "..current.."/"..n);
53
+
54
+ if n > current then
55
+ -- redis.log(redis.LOG_WARNING, "growing");
56
+ redis.call('lpush', round_robin, facet);
57
+ redis.call('hset', facet_pool, facet, current + 1);
58
+ end
59
+
60
+ redis.call('sadd', active_facets, facet)
55
61
  end
56
62
  end
57
63
 
@@ -13,7 +13,7 @@ local facet_pool = k(queue, 'facet_pool');
13
13
 
14
14
  -- Find the current state of the facet for the queue
15
15
  local priority = tonumber(redis.call('hget', priorities, facet)) or 1;
16
- local current = tonumber(redis.call('hget', facet_pool, facet));
16
+ local current = tonumber(redis.call('hget', facet_pool, facet)) or 0;
17
17
 
18
18
  -- If priority is currently zero, we need to jump
19
19
  -- start the facet by adding it to the round-robin
@@ -1,4 +1,6 @@
1
1
  local namespace = KEYS[1];
2
+ local timestamp = tonumber(KEYS[2]);
3
+ local wait = tonumber(KEYS[3]);
2
4
 
3
5
  local k = function (queue, subkey)
4
6
  return namespace .. queue .. ':' .. subkey;
@@ -9,10 +11,28 @@ end
9
11
  -- provided queues, and return a message
10
12
  -- from the first one that isn't empty.
11
13
  for i, queue in ipairs(ARGV) do
12
- local priorities = k(queue, 'priorities');
13
- local active_facets = k(queue, 'active_facets');
14
- local round_robin = k(queue, 'facet_queue');
15
- local facet_pool = k(queue, 'facet_pool');
14
+ local active_facets = k(queue, 'active_facets');
15
+ local round_robin = k(queue, 'facet_queue');
16
+ local inflight = k(queue, 'inflight');
17
+ local inflight_limit = k(queue, 'limit');
18
+ local priorities = k(queue, 'priorities');
19
+ local facet_pool = k(queue, 'facet_pool');
20
+
21
+ if wait ~= -1 then
22
+ -- Check if any current inflight messages
23
+ -- have been inflight for a long time.
24
+ local inflightmessage = redis.call('zrange', inflight, 0, 0, 'WITHSCORES');
25
+
26
+ -- If we have an inflight message and it's score
27
+ -- is less than the current pull timestamp, reset
28
+ -- the inflight score for the the message and resend.
29
+ if #inflightmessage > 0 then
30
+ if tonumber(inflightmessage[2]) <= timestamp then
31
+ redis.call('zadd', inflight, timestamp + wait, inflightmessage[1]);
32
+ return {queue, inflightmessage[1]}
33
+ end
34
+ end
35
+ end
16
36
 
17
37
  -- Pull a facet from the round-robin list.
18
38
  -- This list guarantees each active facet will have a
@@ -23,68 +43,55 @@ for i, queue in ipairs(ARGV) do
23
43
  -- If we found an active facet, we know the facet
24
44
  -- has at least one message available to be pulled
25
45
  -- from it's message queue.
26
- local messages = k(queue, facet);
27
- local message = redis.call('rpop', messages);
46
+ local messages = k(queue, facet);
47
+ local inflight_total = k(queue, facet .. ':inflight');
48
+
49
+ local message = redis.call('rpop', messages);
28
50
 
29
51
  if message then
52
+ if wait ~= -1 then
53
+ redis.call('zadd', inflight, timestamp + wait, message);
54
+ redis.call('incr', inflight_total);
55
+ end
56
+
30
57
  redis.call('decr', k(queue, 'length'));
31
58
  end
32
59
 
33
- local length = redis.call('llen', messages);
60
+ -- Manage facet queue and active facets
61
+ local current = tonumber(redis.call('hget', facet_pool, facet)) or 0;
62
+ local priority = tonumber(redis.call('hget', priorities, facet)) or 1;
63
+ local length = redis.call('llen', messages);
64
+ local inflight_cur = tonumber(redis.call('get', inflight_total)) or 0;
65
+ local inflight_max = tonumber(redis.call('get', inflight_limit)) or 0;
34
66
 
35
- -- If the length of the facet's message queue
36
- -- is empty, then it is no longer active as
37
- -- it no longer has any messages.
38
- if length == 0 then
39
- -- We remove the facet from the set of active
40
- -- facets and don't push the facet back on the
41
- -- round-robin queue.
42
- redis.call('srem', active_facets, facet);
43
-
44
- -- If the facet still has messages to process,
45
- -- it remains in the active facet set, and is
46
- -- pushed back on the round-robin queue.
47
- --
48
- -- Additionally, the priority of the facet may
49
- -- have changed, so we'll check and update the
50
- -- current facet's priority if needed.
67
+ local n = 0
68
+
69
+ -- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
70
+
71
+ if inflight_max > 0 then
72
+ n = math.min(length, priority, inflight_max - inflight_cur);
51
73
  else
52
- local priority = tonumber(redis.call('hget', priorities, facet)) or 1
53
- local current = tonumber(redis.call('hget', facet_pool, facet)) or 1
54
-
55
- -- If the current priority is less than the
56
- -- desired priority, let's increase the priority
57
- -- by pushing the current facet on the round-robin
58
- -- queue twice, and incrementing the current
59
- -- priority.
60
- --
61
- -- Note: If there aren't enough messages left
62
- -- on the facet, we don't increase priority.
63
- if current < priority and length > current then
64
- redis.call('lpush', round_robin, facet);
65
- redis.call('lpush', round_robin, facet);
66
- redis.call('hset', facet_pool, facet, current + 1);
67
-
68
- -- If the current priority is greater than the
69
- -- desired priority, let's decrease the priority
70
- -- by not pushing the current facet on the round-robin
71
- -- queue, and decrementing the current priority.
72
- --
73
- -- Note: Also decrement priority if there aren't
74
- -- enough messages for the current priority. This
75
- -- ensures priority (entries in the round-robin queue)
76
- -- never exceeds the number of messages for a given
77
- -- facet.
78
- elseif current > priority or current > length then
79
- redis.call('hset', facet_pool, facet, current - 1);
80
-
81
- -- If the current priority is equals the
82
- -- desired priority, let's maintain the current priority
83
- -- by pushing the current facet on the round-robin
84
- -- queue once.
85
- else
86
- redis.call('lpush', round_robin, facet);
87
- end
74
+ n = math.min(length, priority);
75
+ end
76
+
77
+ -- redis.log(redis.LOG_WARNING, "PULL: "..current.."/"..n);
78
+
79
+ if n < current then
80
+ -- redis.log(redis.LOG_WARNING, "shrinking");
81
+ redis.call('hset', facet_pool, facet, current - 1);
82
+ elseif n > current then
83
+ -- redis.log(redis.LOG_WARNING, "growing");
84
+ redis.call('lpush', round_robin, facet);
85
+ redis.call('lpush', round_robin, facet);
86
+ redis.call('hset', facet_pool, facet, current + 1);
87
+ else
88
+ -- redis.log(redis.LOG_WARNING, "maintaining");
89
+ redis.call('lpush', round_robin, facet);
90
+ end
91
+
92
+ if (current == 1 and length == 0 and inflight_cur == 0 and n == 0) then
93
+ redis.call('del', inflight_total);
94
+ redis.call('srem', active_facets, facet);
88
95
  end
89
96
 
90
97
  return {queue, message};
@@ -75,6 +75,7 @@ module Fairway
75
75
  it "doesn't push onto to facet queue if currently active" do
76
76
  redis.with do |conn|
77
77
  conn.sadd("myqueue:active_facets", "1")
78
+ conn.hset("myqueue:facet_pool", "1", "1")
78
79
  connection.deliver(message)
79
80
  conn.llen("myqueue:facet_queue").should == 0
80
81
  end
@@ -88,6 +88,7 @@ module Fairway
88
88
  it "doesn't push onto to facet queue if currently active" do
89
89
  redis.with do |conn|
90
90
  conn.sadd("myqueue:active_facets", "1")
91
+ conn.hset("myqueue:facet_pool", "1", "1")
91
92
  connection.deliver(message)
92
93
  conn.llen("myqueue:facet_queue").should == 0
93
94
  end
@@ -139,7 +139,7 @@ module Fairway
139
139
  it "randomized order of queues attempted to reduce starvation" do
140
140
  order = {}
141
141
 
142
- queue.connection.scripts.stub(:fairway_pull) do |queues|
142
+ queue.connection.scripts.stub(:fairway_pull) do |timstamp, wait, queues|
143
143
  order[queues.join(":")] ||= 0
144
144
  order[queues.join(":")] += 1
145
145
  end
@@ -158,7 +158,7 @@ module Fairway
158
158
 
159
159
  order = {}
160
160
 
161
- queue.connection.scripts.stub(:fairway_pull) do |queues|
161
+ queue.connection.scripts.stub(:fairway_pull) do |timestamp, wait, queues|
162
162
  order[queues.join(":")] ||= 0
163
163
  order[queues.join(":")] += 1
164
164
  end
@@ -286,7 +286,7 @@ module Fairway
286
286
  queue.pull.should == ["myqueue", message1.to_json]
287
287
  end
288
288
 
289
- it "defaults current priority to 1 if currently nil (migration from 0.0.x to 0.1.x scenario)" do
289
+ it "defaults current priority to 1 if currently nil" do
290
290
  connection.deliver(message1)
291
291
  connection.deliver(message1)
292
292
  connection.deliver(message2)
@@ -294,10 +294,6 @@ module Fairway
294
294
  connection.deliver(message2)
295
295
  connection.deliver(message1)
296
296
 
297
- connection.redis.with do |conn|
298
- conn.del("myqueue:facet_pool")
299
- end
300
-
301
297
  queue.pull.should == ["myqueue", message1.to_json]
302
298
  queue.pull.should == ["myqueue", message2.to_json]
303
299
  queue.pull.should == ["myqueue", message1.to_json]
@@ -54,7 +54,7 @@ module Fairway
54
54
  let(:scripts) { Scripts.new(redis, "foo") }
55
55
 
56
56
  it "runs the script" do
57
- scripts.fairway_pull("namespace", "name")
57
+ scripts.fairway_pull(Time.now.to_i, -1, "name")
58
58
  end
59
59
 
60
60
  context "when the script does not exist" do
@@ -63,7 +63,7 @@ module Fairway
63
63
  conn.script(:flush)
64
64
  end
65
65
 
66
- scripts.fairway_pull("namespace", "name")
66
+ scripts.fairway_pull(Time.now.to_i, -1, "name")
67
67
  end
68
68
  end
69
69
  end
metadata CHANGED
@@ -1,69 +1,69 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fairway
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.7
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - John Allison
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-01-14 00:00:00.000000000 Z
11
+ date: 2016-02-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - '>='
17
+ - - ! '>='
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - '>='
24
+ - - ! '>='
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: redis
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '>='
31
+ - - ! '>='
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ! '>='
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: redis-namespace
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - '>='
45
+ - - ! '>='
46
46
  - !ruby/object:Gem::Version
47
47
  version: 1.3.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - ! '>='
53
53
  - !ruby/object:Gem::Version
54
54
  version: 1.3.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: connection_pool
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - '>='
59
+ - - ! '>='
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - '>='
66
+ - - ! '>='
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  description: A fair way to queue work in multi-user systems.
@@ -88,9 +88,12 @@ files:
88
88
  - go/config_test.go
89
89
  - go/connection.go
90
90
  - go/connection_test.go
91
+ - go/fairway_ack.go
91
92
  - go/fairway_deliver.go
92
93
  - go/fairway_destroy.go
94
+ - go/fairway_inflight.go
93
95
  - go/fairway_peek.go
96
+ - go/fairway_ping.go
94
97
  - go/fairway_priority.go
95
98
  - go/fairway_pull.go
96
99
  - go/message.go
@@ -135,17 +138,17 @@ require_paths:
135
138
  - lib
136
139
  required_ruby_version: !ruby/object:Gem::Requirement
137
140
  requirements:
138
- - - '>='
141
+ - - ! '>='
139
142
  - !ruby/object:Gem::Version
140
143
  version: '0'
141
144
  required_rubygems_version: !ruby/object:Gem::Requirement
142
145
  requirements:
143
- - - '>='
146
+ - - ! '>='
144
147
  - !ruby/object:Gem::Version
145
148
  version: '0'
146
149
  requirements: []
147
150
  rubyforge_project:
148
- rubygems_version: 2.0.3
151
+ rubygems_version: 2.4.8
149
152
  signing_key:
150
153
  specification_version: 4
151
154
  summary: A fair way to queue work in multi-user systems.