collective 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/.autotest +13 -0
  2. data/.gitignore +5 -0
  3. data/.rspec +1 -0
  4. data/Gemfile +16 -0
  5. data/Guardfile +6 -0
  6. data/README +7 -0
  7. data/Rakefile +11 -0
  8. data/bin/collective +37 -0
  9. data/collective.gemspec +23 -0
  10. data/demo/demo +36 -0
  11. data/demo/demo.rb +30 -0
  12. data/demo/demo3 +36 -0
  13. data/demo/job1.rb +31 -0
  14. data/demo/job2.rb +42 -0
  15. data/demo/job3.rb +44 -0
  16. data/demo/populate.rb +22 -0
  17. data/lib/collective.rb +52 -0
  18. data/lib/collective/checker.rb +51 -0
  19. data/lib/collective/configuration.rb +219 -0
  20. data/lib/collective/idler.rb +81 -0
  21. data/lib/collective/key.rb +48 -0
  22. data/lib/collective/lifecycle_observer.rb +25 -0
  23. data/lib/collective/log.rb +29 -0
  24. data/lib/collective/messager.rb +218 -0
  25. data/lib/collective/mocks/storage.rb +108 -0
  26. data/lib/collective/monitor.rb +58 -0
  27. data/lib/collective/policy.rb +60 -0
  28. data/lib/collective/pool.rb +180 -0
  29. data/lib/collective/redis/storage.rb +142 -0
  30. data/lib/collective/registry.rb +123 -0
  31. data/lib/collective/squiggly.rb +20 -0
  32. data/lib/collective/utilities/airbrake_observer.rb +26 -0
  33. data/lib/collective/utilities/hoptoad_observer.rb +26 -0
  34. data/lib/collective/utilities/log_observer.rb +40 -0
  35. data/lib/collective/utilities/observeable.rb +18 -0
  36. data/lib/collective/utilities/observer_base.rb +59 -0
  37. data/lib/collective/utilities/process.rb +82 -0
  38. data/lib/collective/utilities/signal_hook.rb +47 -0
  39. data/lib/collective/utilities/storage_base.rb +41 -0
  40. data/lib/collective/version.rb +3 -0
  41. data/lib/collective/worker.rb +161 -0
  42. data/spec/checker_spec.rb +20 -0
  43. data/spec/configuration_spec.rb +24 -0
  44. data/spec/helper.rb +33 -0
  45. data/spec/idler_spec.rb +58 -0
  46. data/spec/key_spec.rb +41 -0
  47. data/spec/messager_spec.rb +131 -0
  48. data/spec/mocks/storage_spec.rb +108 -0
  49. data/spec/monitor_spec.rb +15 -0
  50. data/spec/policy_spec.rb +43 -0
  51. data/spec/pool_spec.rb +119 -0
  52. data/spec/redis/storage_spec.rb +133 -0
  53. data/spec/registry_spec.rb +52 -0
  54. data/spec/support/jobs.rb +58 -0
  55. data/spec/support/redis.rb +22 -0
  56. data/spec/support/timing.rb +32 -0
  57. data/spec/utilities/observer_base_spec.rb +50 -0
  58. data/spec/utilities/process_spec.rb +17 -0
  59. data/spec/worker_spec.rb +121 -0
  60. data/unused/times.rb +45 -0
  61. metadata +148 -0
@@ -0,0 +1,58 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require "support/redis"
4
+
5
+ class QuitJob
6
+ def call( context = {} )
7
+ context[:worker].quit!
8
+ end
9
+ end
10
+
11
+ class TrueJob
12
+ def call
13
+ true
14
+ end
15
+ end
16
+
17
+ class QuitJobWithSet
18
+ include RedisClient
19
+ def call( context = {} )
20
+ redis.set("QuitJobWithSet",Process.pid)
21
+ context[:worker].quit!
22
+ end
23
+ end
24
+
25
+ class ForeverJobWithSet
26
+ include RedisClient
27
+ def call( context = {} )
28
+ redis.set("ForeverJobWithSet",Process.pid)
29
+ false
30
+ end
31
+ end
32
+
33
+ class ForeverUntilQuitJob
34
+ include RedisClient
35
+ def call( context = {} )
36
+ if redis.get("ForeverUntilQuitJob") then
37
+ context[:worker].quit!
38
+ else
39
+ false
40
+ end
41
+ end
42
+ end
43
+
44
+ class ListenerJob
45
+ include RedisClient
46
+
47
+ def initialize( context = {} )
48
+ storage = Collective::Redis::Storage.new(redis)
49
+ @mq = Collective::Messager.new( storage, my_address: context[:worker].key )
50
+ @mq.expect("Quit") { |message| context[:worker].quit! }
51
+ @mq.expect("Exit!") { |message| Kernel.exit! }
52
+ @mq.expect("State?") { |message| @mq.reply "State: #{context[:worker].state}", to: message }
53
+ end
54
+
55
+ def call( context = {} )
56
+ @mq.receive
57
+ end
58
+ end
@@ -0,0 +1,22 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require "redis"
4
+
5
+ module RedisClient
6
+
7
+ TEST_REDIS = { url: "redis://127.0.0.1:6379/1" }
8
+
9
+ def redis
10
+ @redis ||= ::Redis.connect(TEST_REDIS)
11
+ end
12
+
13
+ def with_clean_redis(&block)
14
+ redis.client.disconnect # auto connect after fork
15
+ redis.flushall
16
+ yield
17
+ ensure
18
+ redis.flushall
19
+ redis.quit # quit (close) connection, not server
20
+ end
21
+
22
+ end # RedisClient
@@ -0,0 +1,32 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ module Timing
4
+
5
+ def time(&block)
6
+ _time(&block)
7
+ elapsed
8
+ end
9
+
10
+ protected
11
+
12
+ def start
13
+ @start = Time.now.to_f
14
+ end
15
+
16
+ def finish
17
+ @finish = Time.now.to_f
18
+ end
19
+
20
+ def elapsed
21
+ @finish - @start
22
+ end
23
+
24
+ def _time(&block)
25
+ # elapsed time should be known whether or not it raises an error
26
+ start
27
+ yield
28
+ ensure
29
+ finish
30
+ end
31
+
32
+ end # Timing
@@ -0,0 +1,50 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require "helper"
4
+ require "collective"
5
+
6
+ class TestObserver < Collective::Utilities::ObserverBase
7
+ attr :alpha
8
+ attr :beta
9
+ def initialize( alpha = 1, beta = 2 )
10
+ @alpha = alpha
11
+ @beta = beta
12
+ end
13
+ end
14
+
15
+ describe Collective::Utilities::ObserverBase do
16
+
17
+ it "can instaniate from a class name" do
18
+ o = Collective::Utilities::ObserverBase.resolve TestObserver
19
+ o.should be_instance_of TestObserver
20
+ o.alpha.should eq(1)
21
+ o.beta.should eq(2)
22
+ end
23
+
24
+ it "can instaniate from a string" do
25
+ o = Collective::Utilities::ObserverBase.resolve "TestObserver"
26
+ o.should be_instance_of TestObserver
27
+ o.alpha.should eq(1)
28
+ o.beta.should eq(2)
29
+ end
30
+
31
+ it "can instaniate from a symbol" do
32
+ o = Collective::Utilities::ObserverBase.resolve :log
33
+ o.should be_instance_of Collective::Utilities::LogObserver
34
+ end
35
+
36
+ it "can instaniate from a block" do
37
+ o = Collective::Utilities::ObserverBase.resolve (->() { TestObserver.new })
38
+ o.should be_instance_of TestObserver
39
+ o.alpha.should eq(1)
40
+ o.beta.should eq(2)
41
+ end
42
+
43
+ it "can instaniate from an array" do
44
+ o = Collective::Utilities::ObserverBase.resolve [ TestObserver, 2, 4 ]
45
+ o.should be_instance_of TestObserver
46
+ o.alpha.should eq(2)
47
+ o.beta.should eq(4)
48
+ end
49
+
50
+ end
@@ -0,0 +1,17 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require "helper"
4
+ require "collective"
5
+
6
+ describe Collective::Utilities::Process do
7
+ it "does not fail like Process.wait2" do
8
+ system("false")
9
+ pid = $?.pid
10
+ expect { ::Process.wait2( pid ) }.to raise_exception
11
+ end
12
+ it "can handle dead processes" do
13
+ system("false")
14
+ pid = $?.pid
15
+ expect { Collective::Utilities::Process.wait_and_terminate(pid) }.to_not raise_exception
16
+ end
17
+ end
@@ -0,0 +1,121 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require "helper"
4
+
5
+ describe Collective::Worker do
6
+
7
+ it "should run once" do
8
+ count = 0
9
+ worker = nil
10
+ job = ->(context={}) { count += 1; worker.quit! }
11
+ worker = Collective::Worker.new( job )
12
+ worker.run
13
+ count.should eq 1
14
+ end
15
+
16
+ it "should run with a classname" do
17
+ worker = Collective::Worker.new("QuitJob")
18
+ expect { worker.run }.should_not raise_error
19
+ end
20
+
21
+ it "should run with a class" do
22
+ worker = Collective::Worker.new(QuitJob)
23
+ expect { worker.run }.should_not raise_error
24
+ end
25
+
26
+ it "should run with a lambda" do
27
+ job = ->(context) { context[:worker].quit! }
28
+ worker = Collective::Worker.new( job )
29
+ expect { worker.run }.should_not raise_error
30
+ end
31
+
32
+ it "should pass a context with a worker" do
33
+ ok = false
34
+ worker = nil
35
+ job = ->(context) { worker.should eq context[:worker]; worker.quit! }
36
+ worker = Collective::Worker.new( job )
37
+ worker.run
38
+ end
39
+
40
+ it "should use observers" do
41
+ obsrvr = Collective::Utilities::ObserverBase.new
42
+ obsrvr.should_receive(:notify).with(anything,:worker_started).ordered
43
+ obsrvr.should_receive(:notify).with(anything,:worker_heartbeat).ordered
44
+ obsrvr.should_receive(:notify).with(anything,:worker_stopped).ordered
45
+
46
+ job = ->(context) { context[:worker].quit! }
47
+ policy = Collective::Policy.resolve({ observers: [ obsrvr ] })
48
+ worker = Collective::Worker.new job, policy: policy
49
+
50
+ worker.run
51
+ end
52
+
53
+ it "should exit when the policy says to run out (of jobs)" do
54
+ count = 0
55
+ job = ->(context) { count += 1; true }
56
+ policy = Collective::Policy.resolve({ worker_max_jobs: 5 })
57
+ worker = Collective::Worker.new job, policy: policy
58
+ worker.run
59
+ count.should be <= 5
60
+ end
61
+
62
+ it "should execute after_fork blocks"
63
+
64
+ describe "when testing lifetime", time: true do
65
+ it "should exit when the policy says to run out (of time)" do
66
+ overhead = 1
67
+ worker_max_lifetime = 2
68
+ count = 0
69
+ job = ->(context) { count += 1; true }
70
+ policy = Collective::Policy.resolve worker_max_lifetime: worker_max_lifetime, worker_max_jobs: 1e9
71
+ worker = Collective::Worker.new job, policy: policy
72
+ time { worker.run }
73
+ elapsed.should be <= worker_max_lifetime + overhead
74
+ end
75
+ end
76
+
77
+ describe "when spawning a process", redis: true do
78
+
79
+ before do
80
+ @policy = Collective::Policy.resolve worker_max_lifetime: 4, worker_max_jobs: 100, storage: :redis
81
+ end
82
+
83
+ it "should spawn a new process" do
84
+ Collective::Worker.spawn( QuitJobWithSet )
85
+ wait_until { redis.get("QuitJobWithSet").to_i > 0 }
86
+ redis.get("QuitJobWithSet").to_i.should be > 0
87
+ end
88
+
89
+ it "should respond to TERM" do
90
+ Collective::Worker.spawn( ForeverJobWithSet )
91
+
92
+ wait_until { redis.get("ForeverJobWithSet").to_i != 0 }
93
+ pid = redis.get("ForeverJobWithSet").to_i
94
+ Collective::Utilities::Process.alive?(pid).should be_true
95
+
96
+ Process.kill( "TERM", pid )
97
+ wait_until { ! Collective::Utilities::Process.alive?(pid) }
98
+ Collective::Utilities::Process.alive?(pid).should be_false
99
+ end
100
+
101
+ it "uses the registry" do
102
+ job = ForeverUntilQuitJob
103
+ registry = Collective::Registry.new( job.to_s, @policy.storage )
104
+ registry.workers.size.should eq(0)
105
+
106
+ Collective::Worker.spawn ForeverUntilQuitJob, registry: registry, policy: @policy
107
+
108
+ wait_until { registry.workers.size > 0 }
109
+ registry.workers.size.should eq(1)
110
+
111
+ key = registry.workers.first
112
+ key.name.should eq(job.to_s)
113
+
114
+ redis.set("ForeverUntilQuitJob",true)
115
+ wait_until { registry.workers.size == 0 }
116
+ registry.workers.size.should eq(0)
117
+ end
118
+
119
+ end
120
+
121
+ end
@@ -0,0 +1,45 @@
1
+ start, time
2
+ stop, time
3
+ start, time
4
+ stop, time
5
+ start, time
6
+ stop, time
7
+
8
+ def generate
9
+ now = Time.now.to_i
10
+ 10.times do
11
+ puts -now
12
+ now += rand(7)
13
+ puts now
14
+ now += rand(3)
15
+ end
16
+ end
17
+
18
+ times = [-1318798764,1318798766,-1318798768,1318798768,-1318798768,1318798772,-1318798773,1318798776,-1318798777,1318798781,-1318798781,1318798781,-1318798783,1318798784,-1318798786,1318798792,-1318798793,1318798796,-1318798796,1318798800]
19
+
20
+ def analyze( start, times )
21
+ whats = []
22
+ whens = []
23
+ last = nil
24
+ ( times + [Time.now.to_i] ).each do |time|
25
+ if time < 0 then
26
+ if last && whats[last] == :start then
27
+ whats << :stop
28
+ whens << -time
29
+ end
30
+ whats << :start
31
+ whens << -time
32
+ else
33
+ if last && whats[last] == :stop then
34
+ whats << :start
35
+ whens << time
36
+ end
37
+ whats << :stop
38
+ whens << time
39
+ end
40
+ last = (last || -1) + 1
41
+ end
42
+ [ whats, whens ]
43
+ end
44
+
45
+ analyze(times)
metadata ADDED
@@ -0,0 +1,148 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: collective
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.0
5
+ prerelease: !!null
6
+ platform: ruby
7
+ authors:
8
+ - Mark Lanett
9
+ autorequire: !!null
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2011-12-26 00:00:00.000000000 -08:00
13
+ default_executable: !!null
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: redis
17
+ requirement: &70125036305720 !ruby/object:Gem::Requirement
18
+ none: false
19
+ requirements:
20
+ - - ! '>='
21
+ - !ruby/object:Gem::Version
22
+ version: '0'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: *70125036305720
26
+ - !ruby/object:Gem::Dependency
27
+ name: redis-namespace
28
+ requirement: &70125036305300 !ruby/object:Gem::Requirement
29
+ none: false
30
+ requirements:
31
+ - - ! '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: *70125036305300
37
+ description: !!null
38
+ email:
39
+ - mark.lanett@gmail.com
40
+ executables:
41
+ - collective
42
+ extensions: []
43
+ extra_rdoc_files: []
44
+ files:
45
+ - .autotest
46
+ - .gitignore
47
+ - .rspec
48
+ - Gemfile
49
+ - Guardfile
50
+ - README
51
+ - Rakefile
52
+ - bin/collective
53
+ - collective.gemspec
54
+ - demo/demo
55
+ - demo/demo.rb
56
+ - demo/demo3
57
+ - demo/job1.rb
58
+ - demo/job2.rb
59
+ - demo/job3.rb
60
+ - demo/populate.rb
61
+ - lib/collective.rb
62
+ - lib/collective/checker.rb
63
+ - lib/collective/configuration.rb
64
+ - lib/collective/idler.rb
65
+ - lib/collective/key.rb
66
+ - lib/collective/lifecycle_observer.rb
67
+ - lib/collective/log.rb
68
+ - lib/collective/messager.rb
69
+ - lib/collective/mocks/storage.rb
70
+ - lib/collective/monitor.rb
71
+ - lib/collective/policy.rb
72
+ - lib/collective/pool.rb
73
+ - lib/collective/redis/storage.rb
74
+ - lib/collective/registry.rb
75
+ - lib/collective/squiggly.rb
76
+ - lib/collective/utilities/airbrake_observer.rb
77
+ - lib/collective/utilities/hoptoad_observer.rb
78
+ - lib/collective/utilities/log_observer.rb
79
+ - lib/collective/utilities/observeable.rb
80
+ - lib/collective/utilities/observer_base.rb
81
+ - lib/collective/utilities/process.rb
82
+ - lib/collective/utilities/signal_hook.rb
83
+ - lib/collective/utilities/storage_base.rb
84
+ - lib/collective/version.rb
85
+ - lib/collective/worker.rb
86
+ - spec/checker_spec.rb
87
+ - spec/configuration_spec.rb
88
+ - spec/helper.rb
89
+ - spec/idler_spec.rb
90
+ - spec/key_spec.rb
91
+ - spec/messager_spec.rb
92
+ - spec/mocks/storage_spec.rb
93
+ - spec/monitor_spec.rb
94
+ - spec/policy_spec.rb
95
+ - spec/pool_spec.rb
96
+ - spec/redis/storage_spec.rb
97
+ - spec/registry_spec.rb
98
+ - spec/support/jobs.rb
99
+ - spec/support/redis.rb
100
+ - spec/support/timing.rb
101
+ - spec/utilities/observer_base_spec.rb
102
+ - spec/utilities/process_spec.rb
103
+ - spec/worker_spec.rb
104
+ - unused/times.rb
105
+ has_rdoc: true
106
+ homepage: ''
107
+ licenses: []
108
+ post_install_message: !!null
109
+ rdoc_options: []
110
+ require_paths:
111
+ - lib
112
+ required_ruby_version: !ruby/object:Gem::Requirement
113
+ none: false
114
+ requirements:
115
+ - - ! '>='
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ required_rubygems_version: !ruby/object:Gem::Requirement
119
+ none: false
120
+ requirements:
121
+ - - ! '>='
122
+ - !ruby/object:Gem::Version
123
+ version: '0'
124
+ requirements: []
125
+ rubyforge_project: collective
126
+ rubygems_version: 1.5.0
127
+ signing_key: !!null
128
+ specification_version: 3
129
+ summary: Manage a collection of worker processes
130
+ test_files:
131
+ - spec/checker_spec.rb
132
+ - spec/configuration_spec.rb
133
+ - spec/helper.rb
134
+ - spec/idler_spec.rb
135
+ - spec/key_spec.rb
136
+ - spec/messager_spec.rb
137
+ - spec/mocks/storage_spec.rb
138
+ - spec/monitor_spec.rb
139
+ - spec/policy_spec.rb
140
+ - spec/pool_spec.rb
141
+ - spec/redis/storage_spec.rb
142
+ - spec/registry_spec.rb
143
+ - spec/support/jobs.rb
144
+ - spec/support/redis.rb
145
+ - spec/support/timing.rb
146
+ - spec/utilities/observer_base_spec.rb
147
+ - spec/utilities/process_spec.rb
148
+ - spec/worker_spec.rb