infopark-politics 0.3.1 → 0.3.2pg55d4620
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +2 -0
- data/.rbenv-version +1 -0
- data/.rspec +2 -0
- data/Gemfile +3 -0
- data/Manifest +17 -0
- data/Rakefile +25 -0
- data/VERSION +1 -0
- data/dcc_config.rb +5 -0
- data/examples/queue_worker_example.rb +3 -2
- data/examples/token_worker_example.rb +4 -3
- data/infopark-politics.gemspec +38 -0
- data/lib/init.rb +1 -0
- data/lib/politics/discoverable_node.rb +18 -17
- data/lib/politics/static_queue_worker.rb +6 -1
- data/lib/politics/token_worker.rb +15 -14
- data/lib/politics.rb +2 -1
- data/spec/spec_helper.rb +17 -0
- data/spec/static_queue_worker_spec.rb +224 -199
- data/test/static_queue_worker_test.rb +6 -5
- data/test/test_helper.rb +1 -0
- data/test/token_worker_test.rb +6 -5
- metadata +136 -29
- data/lib/politics/version.rb +0 -5
data/.gitignore
ADDED
data/.rbenv-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.8.7-p374
|
data/.rspec
ADDED
data/Gemfile
ADDED
data/Manifest
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
examples/queue_worker_example.rb
|
2
|
+
examples/token_worker_example.rb
|
3
|
+
History.rdoc
|
4
|
+
lib/init.rb
|
5
|
+
lib/politics/discoverable_node.rb
|
6
|
+
lib/politics/static_queue_worker.rb
|
7
|
+
lib/politics/token_worker.rb
|
8
|
+
lib/politics/version.rb
|
9
|
+
lib/politics.rb
|
10
|
+
LICENSE
|
11
|
+
Manifest
|
12
|
+
politics.gemspec
|
13
|
+
Rakefile
|
14
|
+
README.rdoc
|
15
|
+
test/static_queue_worker_test.rb
|
16
|
+
test/test_helper.rb
|
17
|
+
test/token_worker_test.rb
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'rdoc/task'
|
3
|
+
require 'rake/testtask'
|
4
|
+
require 'rspec/core/rake_task'
|
5
|
+
require 'rubygems/tasks'
|
6
|
+
|
7
|
+
Gem::Tasks.new
|
8
|
+
|
9
|
+
desc "Run tests"
|
10
|
+
Rake::TestTask.new do |t|
|
11
|
+
t.libs << 'test' << 'lib'
|
12
|
+
t.test_files = FileList['test/*_test.rb']
|
13
|
+
end
|
14
|
+
|
15
|
+
desc "Run specs"
|
16
|
+
RSpec::Core::RakeTask.new
|
17
|
+
|
18
|
+
desc "Create rdoc"
|
19
|
+
Rake::RDocTask.new do |rd|
|
20
|
+
rd.main = "README.rdoc"
|
21
|
+
rd.rdoc_files.include("README.rdoc", "History.rdoc", "lib/**/*.rb")
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
task :default => :test
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.3.2
|
data/dcc_config.rb
ADDED
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
#gem 'mperham-politics'
|
2
3
|
require 'politics'
|
3
4
|
require 'politics/static_queue_worker'
|
@@ -21,14 +22,14 @@ module Politics
|
|
21
22
|
def initialize
|
22
23
|
register_worker 'queue-example', TOTAL_BUCKETS, :iteration_length => 60, :servers => memcached_servers
|
23
24
|
end
|
24
|
-
|
25
|
+
|
25
26
|
def start
|
26
27
|
process_bucket do |bucket|
|
27
28
|
puts "PID #{$$} processing bucket #{bucket}/#{TOTAL_BUCKETS} at #{Time.now}..."
|
28
29
|
sleep 1.5
|
29
30
|
end
|
30
31
|
end
|
31
|
-
|
32
|
+
|
32
33
|
def memcached_servers
|
33
34
|
['127.0.0.1:11211']
|
34
35
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
#gem 'mperham-politics'
|
2
3
|
require 'politics'
|
3
4
|
require 'politics/token_worker'
|
@@ -16,17 +17,17 @@ require 'politics/token_worker'
|
|
16
17
|
module Politics
|
17
18
|
class TokenWorkerExample
|
18
19
|
include Politics::TokenWorker
|
19
|
-
|
20
|
+
|
20
21
|
def initialize
|
21
22
|
register_worker 'token-example', :iteration_length => 10, :servers => memcached_servers
|
22
23
|
end
|
23
|
-
|
24
|
+
|
24
25
|
def start
|
25
26
|
process do
|
26
27
|
puts "PID #{$$} processing at #{Time.now}..."
|
27
28
|
end
|
28
29
|
end
|
29
|
-
|
30
|
+
|
30
31
|
def memcached_servers
|
31
32
|
['localhost:11211']
|
32
33
|
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
requirement = Gem::Requirement.new(">= 1.6.2")
|
3
|
+
unless requirement.satisfied_by?(Gem::Version.new(Gem::VERSION))
|
4
|
+
raise "You need at RubyGems in Version #{requirement} to build this gem."
|
5
|
+
end
|
6
|
+
|
7
|
+
version = `cd #{File.dirname(__FILE__)} && git describe --tags`.strip.gsub(/^v/, "").gsub(/-.*-/, "p")
|
8
|
+
|
9
|
+
Gem::Specification.new do |gem|
|
10
|
+
gem.name = "infopark-politics"
|
11
|
+
gem.version = version
|
12
|
+
gem.summary = "Algorithms and Tools for Distributed Computing in Ruby."
|
13
|
+
gem.description = ""
|
14
|
+
gem.authors = ["Mike Perham", "Tilo Prütz"]
|
15
|
+
gem.email = "tilo@infopark.de"
|
16
|
+
gem.homepage = "http://github.com/infopark/politics"
|
17
|
+
|
18
|
+
gem.required_rubygems_version = Gem::Requirement.new(">= 1.8")
|
19
|
+
gem.extra_rdoc_files = [
|
20
|
+
"History.rdoc",
|
21
|
+
"LICENSE",
|
22
|
+
"README.rdoc"
|
23
|
+
]
|
24
|
+
gem.files = `git ls-files`.split("\n")
|
25
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
26
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
27
|
+
gem.require_paths = ["lib"]
|
28
|
+
|
29
|
+
gem.add_development_dependency "rspec", ">= 2.14"
|
30
|
+
gem.add_development_dependency "rake", ">= 10"
|
31
|
+
gem.add_development_dependency "rdoc", ">= 3.9"
|
32
|
+
gem.add_development_dependency "rubygems-tasks", ">=0.2"
|
33
|
+
|
34
|
+
gem.add_runtime_dependency "memcache-client", ">= 1.5.0"
|
35
|
+
gem.add_runtime_dependency "starling-starling", ">= 0.9.8"
|
36
|
+
gem.add_runtime_dependency "net-mdns", ">= 0.4"
|
37
|
+
end
|
38
|
+
|
data/lib/init.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
require 'socket'
|
2
3
|
require 'ipaddr'
|
3
4
|
require 'uri'
|
@@ -34,10 +35,10 @@ module Politics
|
|
34
35
|
# replica set. The algorithm is robust in the face of crash failures, but not
|
35
36
|
# Byzantine failures.
|
36
37
|
module DiscoverableNode
|
37
|
-
|
38
|
+
|
38
39
|
attr_accessor :group
|
39
40
|
attr_accessor :coordinator
|
40
|
-
|
41
|
+
|
41
42
|
def register(group='foo')
|
42
43
|
self.group = group
|
43
44
|
start_drb
|
@@ -46,11 +47,11 @@ module Politics
|
|
46
47
|
sleep 0.5
|
47
48
|
find_replicas(0)
|
48
49
|
end
|
49
|
-
|
50
|
+
|
50
51
|
def replicas
|
51
52
|
@replicas ||= {}
|
52
53
|
end
|
53
|
-
|
54
|
+
|
54
55
|
def find_replicas(count)
|
55
56
|
replicas.clear if count % 5 == 0
|
56
57
|
return if count > 10 # Guaranteed to terminate, but not successfully :-(
|
@@ -76,14 +77,14 @@ module Politics
|
|
76
77
|
Politics::log.info { "Found #{replicas.size} peers: #{replicas.keys.sort.inspect}" } if count == 0
|
77
78
|
replicas
|
78
79
|
end
|
79
|
-
|
80
|
+
|
80
81
|
# Called for one peer to introduce itself to another peer. The caller
|
81
82
|
# sends his RID, the responder sends his RID and his list of current peer
|
82
83
|
# RIDs.
|
83
84
|
def hello(remote_rid)
|
84
85
|
[rid, replicas.keys]
|
85
86
|
end
|
86
|
-
|
87
|
+
|
87
88
|
# A process's Replica ID is its PID + a random 16-bit value. We don't want
|
88
89
|
# weigh solely based on PID or IP as that may unduly load one machine.
|
89
90
|
def rid
|
@@ -93,25 +94,25 @@ module Politics
|
|
93
94
|
end
|
94
95
|
|
95
96
|
private
|
96
|
-
|
97
|
+
|
97
98
|
def register_with_bonjour(group)
|
98
99
|
# Register our DRb server with Bonjour.
|
99
|
-
handle = Net::DNS::MDNSSD.register("#{self.group}-#{local_ip}-#{$$}",
|
100
|
+
handle = Net::DNS::MDNSSD.register("#{self.group}-#{local_ip}-#{$$}",
|
100
101
|
"_#{self.group}._tcp", 'local', @port)
|
101
|
-
|
102
|
-
['INT', 'TERM'].each { |signal|
|
102
|
+
|
103
|
+
['INT', 'TERM'].each { |signal|
|
103
104
|
trap(signal) { handle.stop }
|
104
105
|
}
|
105
106
|
end
|
106
|
-
|
107
|
+
|
107
108
|
def start_drb
|
108
109
|
server = DRb.start_service(nil, self)
|
109
110
|
@port = URI.parse(DRb.uri).port
|
110
|
-
['INT', 'TERM'].each { |signal|
|
111
|
+
['INT', 'TERM'].each { |signal|
|
111
112
|
trap(signal) { server.stop_service }
|
112
113
|
}
|
113
114
|
end
|
114
|
-
|
115
|
+
|
115
116
|
def bonjour_scan
|
116
117
|
Net::DNS::MDNSSD.browse("_#{@group}._tcp") do |b|
|
117
118
|
Net::DNS::MDNSSD.resolve(b.name, b.type) do |r|
|
@@ -121,17 +122,17 @@ module Politics
|
|
121
122
|
end
|
122
123
|
end
|
123
124
|
end
|
124
|
-
|
125
|
+
|
125
126
|
# http://coderrr.wordpress.com/2008/05/28/get-your-local-ip-address/
|
126
127
|
def local_ip
|
127
128
|
orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true # turn off reverse DNS resolution temporarily
|
128
|
-
|
129
|
+
|
129
130
|
UDPSocket.open do |s|
|
130
131
|
s.connect '64.233.187.99', 1
|
131
132
|
IPAddr.new(s.addr.last).to_i
|
132
133
|
end
|
133
134
|
ensure
|
134
135
|
Socket.do_not_reverse_lookup = orig
|
135
|
-
end
|
136
|
+
end
|
136
137
|
end
|
137
|
-
end
|
138
|
+
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
require 'socket'
|
2
3
|
require 'ipaddr'
|
3
4
|
require 'uri'
|
@@ -222,6 +223,10 @@ module Politics
|
|
222
223
|
workers
|
223
224
|
end
|
224
225
|
|
226
|
+
def hostname
|
227
|
+
nil
|
228
|
+
end
|
229
|
+
|
225
230
|
private
|
226
231
|
|
227
232
|
def restart_wanted?
|
@@ -328,7 +333,7 @@ module Politics
|
|
328
333
|
end
|
329
334
|
|
330
335
|
def register_with_bonjour
|
331
|
-
server = DRb.start_service(
|
336
|
+
server = DRb.start_service("druby://#{hostname || ""}:0", self)
|
332
337
|
@uri = DRb.uri
|
333
338
|
@port = URI.parse(DRb.uri).port
|
334
339
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
begin
|
2
3
|
require 'memcache'
|
3
4
|
rescue LoadError => e
|
@@ -13,8 +14,8 @@ module Politics
|
|
13
14
|
# The worker instance must obtain the leader token before performing some task.
|
14
15
|
# We use a memcached server as a central token authority to provide a shared,
|
15
16
|
# network-wide view for all processors. This reliance on a single resource means
|
16
|
-
# if your memcached server goes down, so do the processors. Oftentimes,
|
17
|
-
# this is an acceptable trade-off since many high-traffic web sites would
|
17
|
+
# if your memcached server goes down, so do the processors. Oftentimes,
|
18
|
+
# this is an acceptable trade-off since many high-traffic web sites would
|
18
19
|
# not be useable without memcached running anyhow.
|
19
20
|
#
|
20
21
|
# Essentially each TokenWorker attempts to elect itself every +:iteration_length+
|
@@ -46,7 +47,7 @@ module Politics
|
|
46
47
|
# This can often times be quite beneficial (e.g. leveraging a warm cache from the last iteration)
|
47
48
|
# for performance but is left to the reader to implement.
|
48
49
|
module TokenWorker
|
49
|
-
|
50
|
+
|
50
51
|
def self.included(model) #:nodoc:
|
51
52
|
model.class_eval do
|
52
53
|
attr_accessor :memcache_client, :token, :iteration_length, :worker_name
|
@@ -59,7 +60,7 @@ module Politics
|
|
59
60
|
# Register this instance as a worker.
|
60
61
|
#
|
61
62
|
# Options:
|
62
|
-
# +:iteration_length+:: The length of a processing iteration, in seconds. The
|
63
|
+
# +:iteration_length+:: The length of a processing iteration, in seconds. The
|
63
64
|
# leader's 'reign' lasts for this length of time.
|
64
65
|
# +:servers+:: An array of memcached server strings
|
65
66
|
def register_worker(name, config={})
|
@@ -77,7 +78,7 @@ module Politics
|
|
77
78
|
|
78
79
|
cleanup
|
79
80
|
end
|
80
|
-
|
81
|
+
|
81
82
|
def process(*args, &block)
|
82
83
|
verify_registration
|
83
84
|
|
@@ -103,18 +104,18 @@ module Politics
|
|
103
104
|
Politics::log.error(me.backtrace.join("\n"))
|
104
105
|
self.memcache_client.reset
|
105
106
|
end
|
106
|
-
|
107
|
+
|
107
108
|
pause_until_expiry(time)
|
108
109
|
reset_state
|
109
110
|
end while loop?
|
110
111
|
end
|
111
112
|
|
112
113
|
private
|
113
|
-
|
114
|
+
|
114
115
|
def reset_state
|
115
116
|
@leader = nil
|
116
117
|
end
|
117
|
-
|
118
|
+
|
118
119
|
def verify_registration
|
119
120
|
unless self.class.worker_instance
|
120
121
|
raise ArgumentError, "Cannot call process without first calling register_worker"
|
@@ -123,30 +124,30 @@ module Politics
|
|
123
124
|
raise SecurityError, "Only one instance of #{self.class} per process. Another instance was created after this one."
|
124
125
|
end
|
125
126
|
end
|
126
|
-
|
127
|
+
|
127
128
|
def loop?
|
128
129
|
true
|
129
130
|
end
|
130
|
-
|
131
|
+
|
131
132
|
def cleanup
|
132
133
|
at_exit do
|
133
134
|
memcache_client.delete(token) if leader?
|
134
135
|
end
|
135
136
|
end
|
136
|
-
|
137
|
+
|
137
138
|
def pause_until_expiry(elapsed)
|
138
139
|
pause_time = (iteration_length - elapsed).to_f
|
139
140
|
if pause_time > 0
|
140
|
-
relax(pause_time)
|
141
|
+
relax(pause_time)
|
141
142
|
else
|
142
143
|
raise ArgumentError, "Negative iteration time left. Assuming the worst and exiting... #{iteration_length}/#{elapsed}"
|
143
144
|
end
|
144
145
|
end
|
145
|
-
|
146
|
+
|
146
147
|
def relax(time)
|
147
148
|
sleep time
|
148
149
|
end
|
149
|
-
|
150
|
+
|
150
151
|
# Nominate ourself as leader by contacting the memcached server
|
151
152
|
# and attempting to add the token with our name attached.
|
152
153
|
# The result will tell us if memcached stored our value and therefore
|
data/lib/politics.rb
CHANGED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# Require this file using `require "spec_helper"` to ensure that it is only
|
4
|
+
# loaded once.
|
5
|
+
#
|
6
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
7
|
+
RSpec.configure do |config|
|
8
|
+
config.treat_symbols_as_metadata_keys_with_true_values = true
|
9
|
+
config.run_all_when_everything_filtered = true
|
10
|
+
config.filter_run :focus
|
11
|
+
|
12
|
+
# Run specs in random order to surface order dependencies. If you find an
|
13
|
+
# order dependency and want to debug it, you can fix the order by providing
|
14
|
+
# the seed, which is printed after each run.
|
15
|
+
# --seed 1234
|
16
|
+
config.order = 'random'
|
17
|
+
end
|