collective 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.autotest +13 -0
- data/.gitignore +5 -0
- data/.rspec +1 -0
- data/Gemfile +16 -0
- data/Guardfile +6 -0
- data/README +7 -0
- data/Rakefile +11 -0
- data/bin/collective +37 -0
- data/collective.gemspec +23 -0
- data/demo/demo +36 -0
- data/demo/demo.rb +30 -0
- data/demo/demo3 +36 -0
- data/demo/job1.rb +31 -0
- data/demo/job2.rb +42 -0
- data/demo/job3.rb +44 -0
- data/demo/populate.rb +22 -0
- data/lib/collective.rb +52 -0
- data/lib/collective/checker.rb +51 -0
- data/lib/collective/configuration.rb +219 -0
- data/lib/collective/idler.rb +81 -0
- data/lib/collective/key.rb +48 -0
- data/lib/collective/lifecycle_observer.rb +25 -0
- data/lib/collective/log.rb +29 -0
- data/lib/collective/messager.rb +218 -0
- data/lib/collective/mocks/storage.rb +108 -0
- data/lib/collective/monitor.rb +58 -0
- data/lib/collective/policy.rb +60 -0
- data/lib/collective/pool.rb +180 -0
- data/lib/collective/redis/storage.rb +142 -0
- data/lib/collective/registry.rb +123 -0
- data/lib/collective/squiggly.rb +20 -0
- data/lib/collective/utilities/airbrake_observer.rb +26 -0
- data/lib/collective/utilities/hoptoad_observer.rb +26 -0
- data/lib/collective/utilities/log_observer.rb +40 -0
- data/lib/collective/utilities/observeable.rb +18 -0
- data/lib/collective/utilities/observer_base.rb +59 -0
- data/lib/collective/utilities/process.rb +82 -0
- data/lib/collective/utilities/signal_hook.rb +47 -0
- data/lib/collective/utilities/storage_base.rb +41 -0
- data/lib/collective/version.rb +3 -0
- data/lib/collective/worker.rb +161 -0
- data/spec/checker_spec.rb +20 -0
- data/spec/configuration_spec.rb +24 -0
- data/spec/helper.rb +33 -0
- data/spec/idler_spec.rb +58 -0
- data/spec/key_spec.rb +41 -0
- data/spec/messager_spec.rb +131 -0
- data/spec/mocks/storage_spec.rb +108 -0
- data/spec/monitor_spec.rb +15 -0
- data/spec/policy_spec.rb +43 -0
- data/spec/pool_spec.rb +119 -0
- data/spec/redis/storage_spec.rb +133 -0
- data/spec/registry_spec.rb +52 -0
- data/spec/support/jobs.rb +58 -0
- data/spec/support/redis.rb +22 -0
- data/spec/support/timing.rb +32 -0
- data/spec/utilities/observer_base_spec.rb +50 -0
- data/spec/utilities/process_spec.rb +17 -0
- data/spec/worker_spec.rb +121 -0
- data/unused/times.rb +45 -0
- metadata +148 -0
@@ -0,0 +1,108 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
|
5
|
+
describe Collective::Mocks::Storage do
|
6
|
+
|
7
|
+
before do
|
8
|
+
@it = Collective::Mocks::Storage.new
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should be concrete" do
|
12
|
+
@it.should_not be_nil
|
13
|
+
end
|
14
|
+
|
15
|
+
describe "simple values" do
|
16
|
+
|
17
|
+
before do
|
18
|
+
@it.put("foo","bar")
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should be settable" do
|
22
|
+
@it.get("foo").should eq("bar")
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should be replaceable" do
|
26
|
+
@it.put("foo","goo")
|
27
|
+
@it.get("foo").should eq("goo")
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should be deleteable" do
|
31
|
+
@it.get("foo").should_not be_nil
|
32
|
+
@it.del("foo")
|
33
|
+
@it.get("foo").should be_nil
|
34
|
+
end
|
35
|
+
|
36
|
+
end # simple values
|
37
|
+
|
38
|
+
describe "lists" do
|
39
|
+
|
40
|
+
before do
|
41
|
+
@it.set_add("foos","A")
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should not add to a list twice" do
|
45
|
+
@it.set_add("foos","A")
|
46
|
+
@it.set_size("foos").should eq 1
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should be able to add to, enumerate, and remove from a list" do
|
50
|
+
@it.set_add("foos","B")
|
51
|
+
@it.set_size("foos").should eq 2
|
52
|
+
@it.set_get_all("foos").should be_include("B")
|
53
|
+
@it.set_remove("foos","A")
|
54
|
+
@it.set_size("foos").should eq 1
|
55
|
+
end
|
56
|
+
|
57
|
+
it "can detect membership in a list" do
|
58
|
+
@it.set_add("foos","A")
|
59
|
+
@it.set_member?("foos","A").should be_true
|
60
|
+
end
|
61
|
+
|
62
|
+
end # lists
|
63
|
+
|
64
|
+
describe "maps" do
|
65
|
+
|
66
|
+
before do
|
67
|
+
@it.map_set("good","A","E")
|
68
|
+
@it.map_set("food","A","B")
|
69
|
+
@it.map_set("food","C","D")
|
70
|
+
end
|
71
|
+
|
72
|
+
it "should count a set in a key as one item" do
|
73
|
+
@it.map_size("food").should eq 2
|
74
|
+
@it.map_size("good").should eq 1
|
75
|
+
end
|
76
|
+
|
77
|
+
it "should retain the set" do
|
78
|
+
@it.map_get("food","A").should eq "B"
|
79
|
+
@it.map_get("food","C").should eq "D"
|
80
|
+
@it.map_get("good","A").should eq "E"
|
81
|
+
end
|
82
|
+
|
83
|
+
it "should add to the set" do
|
84
|
+
@it.map_get_all_keys("food").size.should eq 2
|
85
|
+
end
|
86
|
+
|
87
|
+
it "should delete the set" do
|
88
|
+
@it.del("food")
|
89
|
+
@it.map_size("food").should eq 0
|
90
|
+
end
|
91
|
+
|
92
|
+
end # maps
|
93
|
+
|
94
|
+
describe "priority queues" do
|
95
|
+
|
96
|
+
it "can add items and remove them in order" do
|
97
|
+
@it.queue_add "foo", "A", 1
|
98
|
+
@it.queue_add "foo", "C", 3
|
99
|
+
@it.queue_add "foo", "B", 2
|
100
|
+
@it.queue_pop("foo",0).should eq(nil)
|
101
|
+
@it.queue_pop("foo",9).should eq("A")
|
102
|
+
@it.queue_pop("foo",9).should eq("B")
|
103
|
+
@it.queue_pop("foo",9).should eq("C")
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
|
5
|
+
describe Collective::Monitor do
|
6
|
+
|
7
|
+
it "can monitor all pools continuously"
|
8
|
+
|
9
|
+
it "can start all workers in all pools"
|
10
|
+
|
11
|
+
it "can stop all workers in all pools"
|
12
|
+
|
13
|
+
it "can restart all workers in all pools"
|
14
|
+
|
15
|
+
end
|
data/spec/policy_spec.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
|
5
|
+
describe Collective::Policy do
|
6
|
+
|
7
|
+
it "should have defaults" do
|
8
|
+
p = Collective::Policy.resolve
|
9
|
+
p.worker_max_jobs.should eq(100)
|
10
|
+
end
|
11
|
+
|
12
|
+
it "should be changeable" do
|
13
|
+
p = Collective::Policy.resolve worker_max_jobs: 5
|
14
|
+
p.worker_max_jobs.should eq(5)
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should work with symbols" do
|
18
|
+
p = Collective::Policy.resolve worker_max_jobs: 5
|
19
|
+
p.worker_max_jobs.should eq(5)
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should support observers" do
|
23
|
+
o = Collective::Utilities::ObserverBase.new
|
24
|
+
p = Collective::Policy.resolve observers: [o]
|
25
|
+
p.observers.should eq([o])
|
26
|
+
end
|
27
|
+
|
28
|
+
it "is copied from another policy" do
|
29
|
+
p1 = Collective::Policy.resolve worker_max_jobs: 7, worker_max_lifetime: 999
|
30
|
+
p2 = Collective::Policy.resolve policy: p1, worker_max_jobs: 12
|
31
|
+
|
32
|
+
p2.worker_max_lifetime.should eq(p1.worker_max_lifetime)
|
33
|
+
p1.worker_max_jobs.should eq(7)
|
34
|
+
p2.worker_max_jobs.should eq(12)
|
35
|
+
end
|
36
|
+
|
37
|
+
it "can resolve storage" do
|
38
|
+
p = Collective::Policy.resolve storage: nil
|
39
|
+
s = p.storage
|
40
|
+
s.should be_instance_of(Collective::Mocks::Storage)
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
data/spec/pool_spec.rb
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
require "redis"
|
5
|
+
|
6
|
+
describe Collective::Pool do
|
7
|
+
|
8
|
+
before do
|
9
|
+
@name = "#{ described_class || 'Test' }::#{example.description}"
|
10
|
+
end
|
11
|
+
|
12
|
+
describe "name" do
|
13
|
+
|
14
|
+
it "needs to be specified for a proc" do
|
15
|
+
job = ->(context) { false }
|
16
|
+
expect { pool = Collective::Pool.new( job ) }.to raise_error
|
17
|
+
expect { pool = Collective::Pool.new( job, name: @name ) }.to_not raise_error
|
18
|
+
end
|
19
|
+
|
20
|
+
it "does not need to be specified for a class" do
|
21
|
+
job = TrueJob
|
22
|
+
expect { pool = Collective::Pool.new( job ) }.to_not raise_error
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
describe "when spawning proceses", redis: true do
|
28
|
+
|
29
|
+
def make_policy( options = {} )
|
30
|
+
options = { name: @name, worker_max_lifetime: 10, storage: :redis, observers: [ [ :log, "/tmp/debug.log" ] ] }.merge(options)
|
31
|
+
Collective::Policy.resolve(options)
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should spawn a worker" do
|
35
|
+
job = ->(context) {}
|
36
|
+
pool = Collective::Pool.new( job, make_policy )
|
37
|
+
|
38
|
+
pool.stub(:spawn) {} # must be called at least once
|
39
|
+
|
40
|
+
pool.synchronize
|
41
|
+
end
|
42
|
+
|
43
|
+
it "spins up an actual worker" do
|
44
|
+
pool = Collective::Pool.new( ListenerJob, make_policy )
|
45
|
+
|
46
|
+
pool.registry.workers.size.should be == 0
|
47
|
+
|
48
|
+
pool.synchronize
|
49
|
+
wait_until { pool.registry.workers.size > 0 }
|
50
|
+
pool.registry.workers.size.should be > 0
|
51
|
+
other = pool.registry.workers.first
|
52
|
+
|
53
|
+
pool.mq.expect(/State/) { |message| puts message.body }
|
54
|
+
pool.mq.send "State?", to: other
|
55
|
+
pool.mq.receive
|
56
|
+
pool.mq.send "Quit", to: other
|
57
|
+
end
|
58
|
+
|
59
|
+
it "spins up a worker only once" do
|
60
|
+
policy = make_policy pool_max_workers: 1
|
61
|
+
pool = Collective::Pool.new( ListenerJob, policy )
|
62
|
+
registry = pool.registry
|
63
|
+
|
64
|
+
registry.checked_workers( policy ).live.size.should eq(0)
|
65
|
+
|
66
|
+
pool.synchronize
|
67
|
+
registry.checked_workers( policy ).live.size.should eq(1)
|
68
|
+
|
69
|
+
pool.synchronize
|
70
|
+
registry.checked_workers( policy ).live.size.should eq(1)
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should spin up new workers as necessary" do
|
74
|
+
policy = make_policy pool_min_workers: 2, pool_max_workers: 2
|
75
|
+
pool = Collective::Pool.new( ListenerJob, policy )
|
76
|
+
registry = pool.registry
|
77
|
+
|
78
|
+
registry.checked_workers( policy ).live.size.should eq(0)
|
79
|
+
|
80
|
+
pool.synchronize
|
81
|
+
registry.checked_workers( policy ).live.size.should eq(2)
|
82
|
+
|
83
|
+
first = registry.checked_workers( policy ).live.first
|
84
|
+
pool.mq.send "Quit", to: first
|
85
|
+
wait_until { registry.checked_workers( policy ).live.size == 1 }
|
86
|
+
registry.checked_workers( policy ).live.size.should eq(1)
|
87
|
+
|
88
|
+
pool.synchronize
|
89
|
+
registry.checked_workers( policy ).live.size.should eq(2)
|
90
|
+
end
|
91
|
+
|
92
|
+
it "spins down workers when there are too many" do
|
93
|
+
policy = make_policy pool_min_workers: 2, pool_max_workers: 2
|
94
|
+
pool = Collective::Pool.new( ListenerJob, policy )
|
95
|
+
registry = pool.registry
|
96
|
+
|
97
|
+
registry.checked_workers( policy ).live.size.should eq(0)
|
98
|
+
|
99
|
+
pool.synchronize
|
100
|
+
registry.checked_workers( policy ).live.size.should eq(2)
|
101
|
+
|
102
|
+
# create a second pool with its own policy to force quitting
|
103
|
+
policy2 = make_policy pool_min_workers: 1, pool_max_workers: 1
|
104
|
+
pool2 = Collective::Pool.new( ListenerJob, policy2 )
|
105
|
+
pool2.synchronize
|
106
|
+
registry.checked_workers( policy ).live.size.should eq(1)
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
|
111
|
+
describe "when tracking how long workers run", time: true do
|
112
|
+
|
113
|
+
it "should warn when a worker is running late"
|
114
|
+
|
115
|
+
it "should kill when a worker is running too late"
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
|
5
|
+
describe Collective::Redis::Storage, redis: true do
|
6
|
+
|
7
|
+
before do
|
8
|
+
@it = Collective::Redis::Storage.new(redis)
|
9
|
+
end
|
10
|
+
|
11
|
+
it "should be concrete" do
|
12
|
+
@it.should_not be_nil
|
13
|
+
end
|
14
|
+
|
15
|
+
describe "simple values" do
|
16
|
+
|
17
|
+
before do
|
18
|
+
@it.put("foo","bar")
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should be settable" do
|
22
|
+
@it.get("foo").should eq("bar")
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should be replaceable" do
|
26
|
+
@it.put("foo","goo")
|
27
|
+
@it.get("foo").should eq("goo")
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should be deleteable" do
|
31
|
+
@it.get("foo").should_not be_nil
|
32
|
+
@it.del("foo")
|
33
|
+
@it.get("foo").should be_nil
|
34
|
+
end
|
35
|
+
|
36
|
+
end # simple values
|
37
|
+
|
38
|
+
describe "lists" do
|
39
|
+
|
40
|
+
before do
|
41
|
+
@it.del("foos")
|
42
|
+
@it.set_add("foos","A")
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should not add to a list twice" do
|
46
|
+
@it.set_add("foos","A")
|
47
|
+
@it.set_size("foos").should eq 1
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should be able to add to, enumerate, and remove from a list" do
|
51
|
+
@it.set_add("foos","B")
|
52
|
+
@it.set_size("foos").should eq 2
|
53
|
+
@it.set_get_all("foos").should be_include("B")
|
54
|
+
@it.set_remove("foos","A")
|
55
|
+
@it.set_size("foos").should eq 1
|
56
|
+
end
|
57
|
+
|
58
|
+
it "can detect membership in a list" do
|
59
|
+
@it.set_add("foos","A")
|
60
|
+
@it.set_member?("foos","A").should be_true
|
61
|
+
end
|
62
|
+
|
63
|
+
end # lists
|
64
|
+
|
65
|
+
describe "maps" do
|
66
|
+
|
67
|
+
before do
|
68
|
+
@it.map_set("good","A","E")
|
69
|
+
@it.map_set("food","A","B")
|
70
|
+
@it.map_set("food","C","D")
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should count a set in a key as one item" do
|
74
|
+
@it.map_size("food").should eq 2
|
75
|
+
@it.map_size("good").should eq 1
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should retain the set" do
|
79
|
+
@it.map_get("food","A").should eq "B"
|
80
|
+
@it.map_get("food","C").should eq "D"
|
81
|
+
@it.map_get("good","A").should eq "E"
|
82
|
+
end
|
83
|
+
|
84
|
+
it "should add to the set" do
|
85
|
+
@it.map_get_all_keys("food").size.should eq 2
|
86
|
+
end
|
87
|
+
|
88
|
+
it "should delete the set" do
|
89
|
+
@it.del("food")
|
90
|
+
@it.map_size("food").should eq 0
|
91
|
+
end
|
92
|
+
|
93
|
+
end # maps
|
94
|
+
|
95
|
+
describe "priority queues" do
|
96
|
+
|
97
|
+
it "can add items and remove them in order" do
|
98
|
+
@it.queue_add "foo", "A", 1
|
99
|
+
@it.queue_add "foo", "C", 3
|
100
|
+
@it.queue_add "foo", "B", 2
|
101
|
+
@it.queue_pop("foo",0).should eq(nil)
|
102
|
+
@it.queue_pop("foo",9).should eq("A")
|
103
|
+
@it.queue_pop("foo",9).should eq("B")
|
104
|
+
@it.queue_pop("foo",9).should eq("C")
|
105
|
+
end
|
106
|
+
|
107
|
+
it "can handle some load" do
|
108
|
+
# expect 1..1000 in the queue
|
109
|
+
# write our 1..1000 in two separate processes
|
110
|
+
# plus some extra just to mess us up
|
111
|
+
|
112
|
+
Collective::Utilities::Process.fork_and_detach do
|
113
|
+
redis.client.reconnect
|
114
|
+
q = Collective::Redis::Storage.new(redis)
|
115
|
+
(1..2000).each { |i| q.queue_add( "foo", i.to_s, i ) if i % 2 == 0 }
|
116
|
+
end
|
117
|
+
|
118
|
+
Collective::Utilities::Process.fork_and_detach do
|
119
|
+
redis.client.reconnect
|
120
|
+
q = Collective::Redis::Storage.new(redis)
|
121
|
+
(1..2000).each { |i| q.queue_add( "foo", i.to_s, i ) if i % 2 == 1 }
|
122
|
+
end
|
123
|
+
|
124
|
+
(1..1000).each do |i|
|
125
|
+
i2 = @it.queue_pop_sync( "foo", i, timeout: 10 )
|
126
|
+
i2.should eq(i.to_s)
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
require "helper"
|
4
|
+
|
5
|
+
describe Collective::Registry, redis: true do
|
6
|
+
|
7
|
+
before do
|
8
|
+
@policy = Collective::Policy.resolve
|
9
|
+
end
|
10
|
+
|
11
|
+
it "can register a worker" do
|
12
|
+
registry = Collective::Registry.new( "Test", @policy.storage )
|
13
|
+
worker = Collective::Worker.new( TrueJob, registry: registry )
|
14
|
+
|
15
|
+
registry.register( worker.key )
|
16
|
+
registry.workers.should be_include( worker.key )
|
17
|
+
end
|
18
|
+
|
19
|
+
it "can unregister a worker" do
|
20
|
+
registry = Collective::Registry.new( "Test", @policy.storage )
|
21
|
+
worker = Collective::Worker.new( TrueJob, registry: registry )
|
22
|
+
|
23
|
+
registry.register( worker.key )
|
24
|
+
registry.workers.should be_include( worker.key )
|
25
|
+
|
26
|
+
registry.unregister( worker.key )
|
27
|
+
registry.workers.should_not be_include( worker.key )
|
28
|
+
end
|
29
|
+
|
30
|
+
it "can find live workers" do
|
31
|
+
registry = Collective::Registry.new( "Test", @policy.storage )
|
32
|
+
heartbeat = @policy.worker_late / 2
|
33
|
+
key = Collective::Key.new("Test",1234)
|
34
|
+
|
35
|
+
registry.register(key)
|
36
|
+
checked = registry.checked_workers(@policy)
|
37
|
+
checked.live.should eq([key])
|
38
|
+
end
|
39
|
+
|
40
|
+
it "can find late workers" do
|
41
|
+
registry = Collective::Registry.new( "Test", @policy.storage )
|
42
|
+
heartbeat = @policy.worker_late
|
43
|
+
key = Collective::Key.new("Test",1234)
|
44
|
+
|
45
|
+
now = Time.now.to_i
|
46
|
+
registry.register(key) # should register with heartbeat = now or now+1
|
47
|
+
registry.stub(:now) { now + @policy.worker_late + 2 }
|
48
|
+
checked = registry.checked_workers(@policy)
|
49
|
+
checked.late.should eq([key])
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|