fairway 0.2.7 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +13 -5
- data/Gemfile.lock +8 -7
- data/go/channeled_connection_test.go +22 -0
- data/go/config.go +1 -1
- data/go/config_test.go +2 -2
- data/go/connection.go +5 -0
- data/go/connection_test.go +1 -0
- data/go/fairway_ack.go +60 -0
- data/go/fairway_deliver.go +32 -27
- data/go/fairway_destroy.go +11 -6
- data/go/fairway_inflight.go +15 -0
- data/go/fairway_peek.go +1 -3
- data/go/fairway_ping.go +20 -0
- data/go/fairway_priority.go +2 -3
- data/go/fairway_pull.go +67 -61
- data/go/message.go +3 -2
- data/go/queue.go +31 -2
- data/go/queue_test.go +467 -9
- data/go/scripts.go +82 -4
- data/lib/fairway/queue.rb +13 -1
- data/lib/fairway/scripts.rb +20 -0
- data/lib/fairway/version.rb +1 -1
- data/redis/fairway_deliver.lua +32 -26
- data/redis/fairway_priority.lua +1 -1
- data/redis/fairway_pull.lua +66 -59
- data/spec/lib/fairway/channeled_connection_spec.rb +1 -0
- data/spec/lib/fairway/connection_spec.rb +1 -0
- data/spec/lib/fairway/queue_spec.rb +3 -7
- data/spec/lib/fairway/scripts_spec.rb +2 -2
- metadata +16 -13
data/redis/fairway_deliver.lua
CHANGED
@@ -19,39 +19,45 @@ for i = 1, #registered_queues, 2 do
|
|
19
19
|
-- If the message topic matches the queue topic,
|
20
20
|
-- we deliver the message to the queue.
|
21
21
|
if string.find(topic, queue_topic) then
|
22
|
-
local priorities
|
23
|
-
local active_facets
|
24
|
-
local round_robin
|
25
|
-
local facet_pool
|
22
|
+
local priorities = k(queue, 'priorities');
|
23
|
+
local active_facets = k(queue, 'active_facets');
|
24
|
+
local round_robin = k(queue, 'facet_queue');
|
25
|
+
local facet_pool = k(queue, 'facet_pool');
|
26
|
+
local inflight_total = k(queue, facet .. ':inflight');
|
27
|
+
local inflight_limit = k(queue, 'limit');
|
26
28
|
|
27
29
|
-- Delivering the message to a queue is as simple as
|
28
30
|
-- pushing it onto the facet's message list, and
|
29
31
|
-- incrementing the length of the queue itself.
|
30
|
-
redis.call('lpush', k(queue, facet), message)
|
32
|
+
local length = redis.call('lpush', k(queue, facet), message)
|
31
33
|
redis.call('incr', k(queue, 'length'));
|
32
34
|
|
33
|
-
|
34
|
-
--
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
-- current priority to zero. Since the facet just
|
49
|
-
-- became active, we can be sure it's already zero
|
50
|
-
-- or undefined.
|
51
|
-
else
|
52
|
-
redis.call('hset', facet_pool, facet, 0);
|
53
|
-
end
|
35
|
+
|
36
|
+
-- Manage facet queue and active facets
|
37
|
+
local current = tonumber(redis.call('hget', facet_pool, facet)) or 0;
|
38
|
+
local priority = tonumber(redis.call('hget', priorities, facet)) or 1;
|
39
|
+
local inflight_cur = tonumber(redis.call('get', inflight_total)) or 0;
|
40
|
+
local inflight_max = tonumber(redis.call('get', inflight_limit)) or 0;
|
41
|
+
|
42
|
+
local n = 0
|
43
|
+
|
44
|
+
-- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
|
45
|
+
|
46
|
+
if inflight_max > 0 then
|
47
|
+
n = math.min(length, priority, inflight_max - inflight_cur);
|
48
|
+
else
|
49
|
+
n = math.min(length, priority);
|
54
50
|
end
|
51
|
+
|
52
|
+
-- redis.log(redis.LOG_WARNING, "PUSH: "..current.."/"..n);
|
53
|
+
|
54
|
+
if n > current then
|
55
|
+
-- redis.log(redis.LOG_WARNING, "growing");
|
56
|
+
redis.call('lpush', round_robin, facet);
|
57
|
+
redis.call('hset', facet_pool, facet, current + 1);
|
58
|
+
end
|
59
|
+
|
60
|
+
redis.call('sadd', active_facets, facet)
|
55
61
|
end
|
56
62
|
end
|
57
63
|
|
data/redis/fairway_priority.lua
CHANGED
@@ -13,7 +13,7 @@ local facet_pool = k(queue, 'facet_pool');
|
|
13
13
|
|
14
14
|
-- Find the current state of the facet for the queue
|
15
15
|
local priority = tonumber(redis.call('hget', priorities, facet)) or 1;
|
16
|
-
local current = tonumber(redis.call('hget', facet_pool, facet));
|
16
|
+
local current = tonumber(redis.call('hget', facet_pool, facet)) or 0;
|
17
17
|
|
18
18
|
-- If priority is currently zero, we need to jump
|
19
19
|
-- start the facet by adding it to the round-robin
|
data/redis/fairway_pull.lua
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
local namespace = KEYS[1];
|
2
|
+
local timestamp = tonumber(KEYS[2]);
|
3
|
+
local wait = tonumber(KEYS[3]);
|
2
4
|
|
3
5
|
local k = function (queue, subkey)
|
4
6
|
return namespace .. queue .. ':' .. subkey;
|
@@ -9,10 +11,28 @@ end
|
|
9
11
|
-- provided queues, and return a message
|
10
12
|
-- from the first one that isn't empty.
|
11
13
|
for i, queue in ipairs(ARGV) do
|
12
|
-
local
|
13
|
-
local
|
14
|
-
local
|
15
|
-
local
|
14
|
+
local active_facets = k(queue, 'active_facets');
|
15
|
+
local round_robin = k(queue, 'facet_queue');
|
16
|
+
local inflight = k(queue, 'inflight');
|
17
|
+
local inflight_limit = k(queue, 'limit');
|
18
|
+
local priorities = k(queue, 'priorities');
|
19
|
+
local facet_pool = k(queue, 'facet_pool');
|
20
|
+
|
21
|
+
if wait ~= -1 then
|
22
|
+
-- Check if any current inflight messages
|
23
|
+
-- have been inflight for a long time.
|
24
|
+
local inflightmessage = redis.call('zrange', inflight, 0, 0, 'WITHSCORES');
|
25
|
+
|
26
|
+
-- If we have an inflight message and it's score
|
27
|
+
-- is less than the current pull timestamp, reset
|
28
|
+
-- the inflight score for the the message and resend.
|
29
|
+
if #inflightmessage > 0 then
|
30
|
+
if tonumber(inflightmessage[2]) <= timestamp then
|
31
|
+
redis.call('zadd', inflight, timestamp + wait, inflightmessage[1]);
|
32
|
+
return {queue, inflightmessage[1]}
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
16
36
|
|
17
37
|
-- Pull a facet from the round-robin list.
|
18
38
|
-- This list guarantees each active facet will have a
|
@@ -23,68 +43,55 @@ for i, queue in ipairs(ARGV) do
|
|
23
43
|
-- If we found an active facet, we know the facet
|
24
44
|
-- has at least one message available to be pulled
|
25
45
|
-- from it's message queue.
|
26
|
-
local messages
|
27
|
-
local
|
46
|
+
local messages = k(queue, facet);
|
47
|
+
local inflight_total = k(queue, facet .. ':inflight');
|
48
|
+
|
49
|
+
local message = redis.call('rpop', messages);
|
28
50
|
|
29
51
|
if message then
|
52
|
+
if wait ~= -1 then
|
53
|
+
redis.call('zadd', inflight, timestamp + wait, message);
|
54
|
+
redis.call('incr', inflight_total);
|
55
|
+
end
|
56
|
+
|
30
57
|
redis.call('decr', k(queue, 'length'));
|
31
58
|
end
|
32
59
|
|
33
|
-
|
60
|
+
-- Manage facet queue and active facets
|
61
|
+
local current = tonumber(redis.call('hget', facet_pool, facet)) or 0;
|
62
|
+
local priority = tonumber(redis.call('hget', priorities, facet)) or 1;
|
63
|
+
local length = redis.call('llen', messages);
|
64
|
+
local inflight_cur = tonumber(redis.call('get', inflight_total)) or 0;
|
65
|
+
local inflight_max = tonumber(redis.call('get', inflight_limit)) or 0;
|
34
66
|
|
35
|
-
|
36
|
-
|
37
|
-
--
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
-- round-robin queue.
|
42
|
-
redis.call('srem', active_facets, facet);
|
43
|
-
|
44
|
-
-- If the facet still has messages to process,
|
45
|
-
-- it remains in the active facet set, and is
|
46
|
-
-- pushed back on the round-robin queue.
|
47
|
-
--
|
48
|
-
-- Additionally, the priority of the facet may
|
49
|
-
-- have changed, so we'll check and update the
|
50
|
-
-- current facet's priority if needed.
|
67
|
+
local n = 0
|
68
|
+
|
69
|
+
-- redis.log(redis.LOG_WARNING, current.."/"..length.."/"..priority.."/"..inflight_max.."/"..inflight_cur);
|
70
|
+
|
71
|
+
if inflight_max > 0 then
|
72
|
+
n = math.min(length, priority, inflight_max - inflight_cur);
|
51
73
|
else
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
--
|
59
|
-
|
60
|
-
|
61
|
-
--
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
-- Note: Also decrement priority if there aren't
|
74
|
-
-- enough messages for the current priority. This
|
75
|
-
-- ensures priority (entries in the round-robin queue)
|
76
|
-
-- never exceeds the number of messages for a given
|
77
|
-
-- facet.
|
78
|
-
elseif current > priority or current > length then
|
79
|
-
redis.call('hset', facet_pool, facet, current - 1);
|
80
|
-
|
81
|
-
-- If the current priority is equals the
|
82
|
-
-- desired priority, let's maintain the current priority
|
83
|
-
-- by pushing the current facet on the round-robin
|
84
|
-
-- queue once.
|
85
|
-
else
|
86
|
-
redis.call('lpush', round_robin, facet);
|
87
|
-
end
|
74
|
+
n = math.min(length, priority);
|
75
|
+
end
|
76
|
+
|
77
|
+
-- redis.log(redis.LOG_WARNING, "PULL: "..current.."/"..n);
|
78
|
+
|
79
|
+
if n < current then
|
80
|
+
-- redis.log(redis.LOG_WARNING, "shrinking");
|
81
|
+
redis.call('hset', facet_pool, facet, current - 1);
|
82
|
+
elseif n > current then
|
83
|
+
-- redis.log(redis.LOG_WARNING, "growing");
|
84
|
+
redis.call('lpush', round_robin, facet);
|
85
|
+
redis.call('lpush', round_robin, facet);
|
86
|
+
redis.call('hset', facet_pool, facet, current + 1);
|
87
|
+
else
|
88
|
+
-- redis.log(redis.LOG_WARNING, "maintaining");
|
89
|
+
redis.call('lpush', round_robin, facet);
|
90
|
+
end
|
91
|
+
|
92
|
+
if (current == 1 and length == 0 and inflight_cur == 0 and n == 0) then
|
93
|
+
redis.call('del', inflight_total);
|
94
|
+
redis.call('srem', active_facets, facet);
|
88
95
|
end
|
89
96
|
|
90
97
|
return {queue, message};
|
@@ -75,6 +75,7 @@ module Fairway
|
|
75
75
|
it "doesn't push onto to facet queue if currently active" do
|
76
76
|
redis.with do |conn|
|
77
77
|
conn.sadd("myqueue:active_facets", "1")
|
78
|
+
conn.hset("myqueue:facet_pool", "1", "1")
|
78
79
|
connection.deliver(message)
|
79
80
|
conn.llen("myqueue:facet_queue").should == 0
|
80
81
|
end
|
@@ -88,6 +88,7 @@ module Fairway
|
|
88
88
|
it "doesn't push onto to facet queue if currently active" do
|
89
89
|
redis.with do |conn|
|
90
90
|
conn.sadd("myqueue:active_facets", "1")
|
91
|
+
conn.hset("myqueue:facet_pool", "1", "1")
|
91
92
|
connection.deliver(message)
|
92
93
|
conn.llen("myqueue:facet_queue").should == 0
|
93
94
|
end
|
@@ -139,7 +139,7 @@ module Fairway
|
|
139
139
|
it "randomized order of queues attempted to reduce starvation" do
|
140
140
|
order = {}
|
141
141
|
|
142
|
-
queue.connection.scripts.stub(:fairway_pull) do |queues|
|
142
|
+
queue.connection.scripts.stub(:fairway_pull) do |timstamp, wait, queues|
|
143
143
|
order[queues.join(":")] ||= 0
|
144
144
|
order[queues.join(":")] += 1
|
145
145
|
end
|
@@ -158,7 +158,7 @@ module Fairway
|
|
158
158
|
|
159
159
|
order = {}
|
160
160
|
|
161
|
-
queue.connection.scripts.stub(:fairway_pull) do |queues|
|
161
|
+
queue.connection.scripts.stub(:fairway_pull) do |timestamp, wait, queues|
|
162
162
|
order[queues.join(":")] ||= 0
|
163
163
|
order[queues.join(":")] += 1
|
164
164
|
end
|
@@ -286,7 +286,7 @@ module Fairway
|
|
286
286
|
queue.pull.should == ["myqueue", message1.to_json]
|
287
287
|
end
|
288
288
|
|
289
|
-
it "defaults current priority to 1 if currently nil
|
289
|
+
it "defaults current priority to 1 if currently nil" do
|
290
290
|
connection.deliver(message1)
|
291
291
|
connection.deliver(message1)
|
292
292
|
connection.deliver(message2)
|
@@ -294,10 +294,6 @@ module Fairway
|
|
294
294
|
connection.deliver(message2)
|
295
295
|
connection.deliver(message1)
|
296
296
|
|
297
|
-
connection.redis.with do |conn|
|
298
|
-
conn.del("myqueue:facet_pool")
|
299
|
-
end
|
300
|
-
|
301
297
|
queue.pull.should == ["myqueue", message1.to_json]
|
302
298
|
queue.pull.should == ["myqueue", message2.to_json]
|
303
299
|
queue.pull.should == ["myqueue", message1.to_json]
|
@@ -54,7 +54,7 @@ module Fairway
|
|
54
54
|
let(:scripts) { Scripts.new(redis, "foo") }
|
55
55
|
|
56
56
|
it "runs the script" do
|
57
|
-
scripts.fairway_pull(
|
57
|
+
scripts.fairway_pull(Time.now.to_i, -1, "name")
|
58
58
|
end
|
59
59
|
|
60
60
|
context "when the script does not exist" do
|
@@ -63,7 +63,7 @@ module Fairway
|
|
63
63
|
conn.script(:flush)
|
64
64
|
end
|
65
65
|
|
66
|
-
scripts.fairway_pull(
|
66
|
+
scripts.fairway_pull(Time.now.to_i, -1, "name")
|
67
67
|
end
|
68
68
|
end
|
69
69
|
end
|
metadata
CHANGED
@@ -1,69 +1,69 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fairway
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- John Allison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-02-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - '>='
|
17
|
+
- - ! '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - '>='
|
24
|
+
- - ! '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: redis
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - '>='
|
31
|
+
- - ! '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - '>='
|
38
|
+
- - ! '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: redis-namespace
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - '>='
|
45
|
+
- - ! '>='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: 1.3.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - '>='
|
52
|
+
- - ! '>='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: 1.3.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: connection_pool
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - '>='
|
59
|
+
- - ! '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - '>='
|
66
|
+
- - ! '>='
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
description: A fair way to queue work in multi-user systems.
|
@@ -88,9 +88,12 @@ files:
|
|
88
88
|
- go/config_test.go
|
89
89
|
- go/connection.go
|
90
90
|
- go/connection_test.go
|
91
|
+
- go/fairway_ack.go
|
91
92
|
- go/fairway_deliver.go
|
92
93
|
- go/fairway_destroy.go
|
94
|
+
- go/fairway_inflight.go
|
93
95
|
- go/fairway_peek.go
|
96
|
+
- go/fairway_ping.go
|
94
97
|
- go/fairway_priority.go
|
95
98
|
- go/fairway_pull.go
|
96
99
|
- go/message.go
|
@@ -135,17 +138,17 @@ require_paths:
|
|
135
138
|
- lib
|
136
139
|
required_ruby_version: !ruby/object:Gem::Requirement
|
137
140
|
requirements:
|
138
|
-
- - '>='
|
141
|
+
- - ! '>='
|
139
142
|
- !ruby/object:Gem::Version
|
140
143
|
version: '0'
|
141
144
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
142
145
|
requirements:
|
143
|
-
- - '>='
|
146
|
+
- - ! '>='
|
144
147
|
- !ruby/object:Gem::Version
|
145
148
|
version: '0'
|
146
149
|
requirements: []
|
147
150
|
rubyforge_project:
|
148
|
-
rubygems_version: 2.
|
151
|
+
rubygems_version: 2.4.8
|
149
152
|
signing_key:
|
150
153
|
specification_version: 4
|
151
154
|
summary: A fair way to queue work in multi-user systems.
|