infopark-politics 0.3.1 → 0.3.2pg55d4620
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -0
- data/.rbenv-version +1 -0
- data/.rspec +2 -0
- data/Gemfile +3 -0
- data/Manifest +17 -0
- data/Rakefile +25 -0
- data/VERSION +1 -0
- data/dcc_config.rb +5 -0
- data/examples/queue_worker_example.rb +3 -2
- data/examples/token_worker_example.rb +4 -3
- data/infopark-politics.gemspec +38 -0
- data/lib/init.rb +1 -0
- data/lib/politics/discoverable_node.rb +18 -17
- data/lib/politics/static_queue_worker.rb +6 -1
- data/lib/politics/token_worker.rb +15 -14
- data/lib/politics.rb +2 -1
- data/spec/spec_helper.rb +17 -0
- data/spec/static_queue_worker_spec.rb +224 -199
- data/test/static_queue_worker_test.rb +6 -5
- data/test/test_helper.rb +1 -0
- data/test/token_worker_test.rb +6 -5
- metadata +136 -29
- data/lib/politics/version.rb +0 -5
data/.gitignore
ADDED
data/.rbenv-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.8.7-p374
|
data/.rspec
ADDED
data/Gemfile
ADDED
data/Manifest
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
examples/queue_worker_example.rb
|
2
|
+
examples/token_worker_example.rb
|
3
|
+
History.rdoc
|
4
|
+
lib/init.rb
|
5
|
+
lib/politics/discoverable_node.rb
|
6
|
+
lib/politics/static_queue_worker.rb
|
7
|
+
lib/politics/token_worker.rb
|
8
|
+
lib/politics/version.rb
|
9
|
+
lib/politics.rb
|
10
|
+
LICENSE
|
11
|
+
Manifest
|
12
|
+
politics.gemspec
|
13
|
+
Rakefile
|
14
|
+
README.rdoc
|
15
|
+
test/static_queue_worker_test.rb
|
16
|
+
test/test_helper.rb
|
17
|
+
test/token_worker_test.rb
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'rdoc/task'
|
3
|
+
require 'rake/testtask'
|
4
|
+
require 'rspec/core/rake_task'
|
5
|
+
require 'rubygems/tasks'
|
6
|
+
|
7
|
+
Gem::Tasks.new
|
8
|
+
|
9
|
+
desc "Run tests"
|
10
|
+
Rake::TestTask.new do |t|
|
11
|
+
t.libs << 'test' << 'lib'
|
12
|
+
t.test_files = FileList['test/*_test.rb']
|
13
|
+
end
|
14
|
+
|
15
|
+
desc "Run specs"
|
16
|
+
RSpec::Core::RakeTask.new
|
17
|
+
|
18
|
+
desc "Create rdoc"
|
19
|
+
Rake::RDocTask.new do |rd|
|
20
|
+
rd.main = "README.rdoc"
|
21
|
+
rd.rdoc_files.include("README.rdoc", "History.rdoc", "lib/**/*.rb")
|
22
|
+
end
|
23
|
+
|
24
|
+
|
25
|
+
task :default => :test
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.3.2
|
data/dcc_config.rb
ADDED
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
#gem 'mperham-politics'
|
2
3
|
require 'politics'
|
3
4
|
require 'politics/static_queue_worker'
|
@@ -21,14 +22,14 @@ module Politics
|
|
21
22
|
def initialize
|
22
23
|
register_worker 'queue-example', TOTAL_BUCKETS, :iteration_length => 60, :servers => memcached_servers
|
23
24
|
end
|
24
|
-
|
25
|
+
|
25
26
|
def start
|
26
27
|
process_bucket do |bucket|
|
27
28
|
puts "PID #{$$} processing bucket #{bucket}/#{TOTAL_BUCKETS} at #{Time.now}..."
|
28
29
|
sleep 1.5
|
29
30
|
end
|
30
31
|
end
|
31
|
-
|
32
|
+
|
32
33
|
def memcached_servers
|
33
34
|
['127.0.0.1:11211']
|
34
35
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
#gem 'mperham-politics'
|
2
3
|
require 'politics'
|
3
4
|
require 'politics/token_worker'
|
@@ -16,17 +17,17 @@ require 'politics/token_worker'
|
|
16
17
|
module Politics
|
17
18
|
class TokenWorkerExample
|
18
19
|
include Politics::TokenWorker
|
19
|
-
|
20
|
+
|
20
21
|
def initialize
|
21
22
|
register_worker 'token-example', :iteration_length => 10, :servers => memcached_servers
|
22
23
|
end
|
23
|
-
|
24
|
+
|
24
25
|
def start
|
25
26
|
process do
|
26
27
|
puts "PID #{$$} processing at #{Time.now}..."
|
27
28
|
end
|
28
29
|
end
|
29
|
-
|
30
|
+
|
30
31
|
def memcached_servers
|
31
32
|
['localhost:11211']
|
32
33
|
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
requirement = Gem::Requirement.new(">= 1.6.2")
|
3
|
+
unless requirement.satisfied_by?(Gem::Version.new(Gem::VERSION))
|
4
|
+
raise "You need at RubyGems in Version #{requirement} to build this gem."
|
5
|
+
end
|
6
|
+
|
7
|
+
version = `cd #{File.dirname(__FILE__)} && git describe --tags`.strip.gsub(/^v/, "").gsub(/-.*-/, "p")
|
8
|
+
|
9
|
+
Gem::Specification.new do |gem|
|
10
|
+
gem.name = "infopark-politics"
|
11
|
+
gem.version = version
|
12
|
+
gem.summary = "Algorithms and Tools for Distributed Computing in Ruby."
|
13
|
+
gem.description = ""
|
14
|
+
gem.authors = ["Mike Perham", "Tilo Prütz"]
|
15
|
+
gem.email = "tilo@infopark.de"
|
16
|
+
gem.homepage = "http://github.com/infopark/politics"
|
17
|
+
|
18
|
+
gem.required_rubygems_version = Gem::Requirement.new(">= 1.8")
|
19
|
+
gem.extra_rdoc_files = [
|
20
|
+
"History.rdoc",
|
21
|
+
"LICENSE",
|
22
|
+
"README.rdoc"
|
23
|
+
]
|
24
|
+
gem.files = `git ls-files`.split("\n")
|
25
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
26
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
27
|
+
gem.require_paths = ["lib"]
|
28
|
+
|
29
|
+
gem.add_development_dependency "rspec", ">= 2.14"
|
30
|
+
gem.add_development_dependency "rake", ">= 10"
|
31
|
+
gem.add_development_dependency "rdoc", ">= 3.9"
|
32
|
+
gem.add_development_dependency "rubygems-tasks", ">=0.2"
|
33
|
+
|
34
|
+
gem.add_runtime_dependency "memcache-client", ">= 1.5.0"
|
35
|
+
gem.add_runtime_dependency "starling-starling", ">= 0.9.8"
|
36
|
+
gem.add_runtime_dependency "net-mdns", ">= 0.4"
|
37
|
+
end
|
38
|
+
|
data/lib/init.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
require 'socket'
|
2
3
|
require 'ipaddr'
|
3
4
|
require 'uri'
|
@@ -34,10 +35,10 @@ module Politics
|
|
34
35
|
# replica set. The algorithm is robust in the face of crash failures, but not
|
35
36
|
# Byzantine failures.
|
36
37
|
module DiscoverableNode
|
37
|
-
|
38
|
+
|
38
39
|
attr_accessor :group
|
39
40
|
attr_accessor :coordinator
|
40
|
-
|
41
|
+
|
41
42
|
def register(group='foo')
|
42
43
|
self.group = group
|
43
44
|
start_drb
|
@@ -46,11 +47,11 @@ module Politics
|
|
46
47
|
sleep 0.5
|
47
48
|
find_replicas(0)
|
48
49
|
end
|
49
|
-
|
50
|
+
|
50
51
|
def replicas
|
51
52
|
@replicas ||= {}
|
52
53
|
end
|
53
|
-
|
54
|
+
|
54
55
|
def find_replicas(count)
|
55
56
|
replicas.clear if count % 5 == 0
|
56
57
|
return if count > 10 # Guaranteed to terminate, but not successfully :-(
|
@@ -76,14 +77,14 @@ module Politics
|
|
76
77
|
Politics::log.info { "Found #{replicas.size} peers: #{replicas.keys.sort.inspect}" } if count == 0
|
77
78
|
replicas
|
78
79
|
end
|
79
|
-
|
80
|
+
|
80
81
|
# Called for one peer to introduce itself to another peer. The caller
|
81
82
|
# sends his RID, the responder sends his RID and his list of current peer
|
82
83
|
# RIDs.
|
83
84
|
def hello(remote_rid)
|
84
85
|
[rid, replicas.keys]
|
85
86
|
end
|
86
|
-
|
87
|
+
|
87
88
|
# A process's Replica ID is its PID + a random 16-bit value. We don't want
|
88
89
|
# weigh solely based on PID or IP as that may unduly load one machine.
|
89
90
|
def rid
|
@@ -93,25 +94,25 @@ module Politics
|
|
93
94
|
end
|
94
95
|
|
95
96
|
private
|
96
|
-
|
97
|
+
|
97
98
|
def register_with_bonjour(group)
|
98
99
|
# Register our DRb server with Bonjour.
|
99
|
-
handle = Net::DNS::MDNSSD.register("#{self.group}-#{local_ip}-#{$$}",
|
100
|
+
handle = Net::DNS::MDNSSD.register("#{self.group}-#{local_ip}-#{$$}",
|
100
101
|
"_#{self.group}._tcp", 'local', @port)
|
101
|
-
|
102
|
-
['INT', 'TERM'].each { |signal|
|
102
|
+
|
103
|
+
['INT', 'TERM'].each { |signal|
|
103
104
|
trap(signal) { handle.stop }
|
104
105
|
}
|
105
106
|
end
|
106
|
-
|
107
|
+
|
107
108
|
def start_drb
|
108
109
|
server = DRb.start_service(nil, self)
|
109
110
|
@port = URI.parse(DRb.uri).port
|
110
|
-
['INT', 'TERM'].each { |signal|
|
111
|
+
['INT', 'TERM'].each { |signal|
|
111
112
|
trap(signal) { server.stop_service }
|
112
113
|
}
|
113
114
|
end
|
114
|
-
|
115
|
+
|
115
116
|
def bonjour_scan
|
116
117
|
Net::DNS::MDNSSD.browse("_#{@group}._tcp") do |b|
|
117
118
|
Net::DNS::MDNSSD.resolve(b.name, b.type) do |r|
|
@@ -121,17 +122,17 @@ module Politics
|
|
121
122
|
end
|
122
123
|
end
|
123
124
|
end
|
124
|
-
|
125
|
+
|
125
126
|
# http://coderrr.wordpress.com/2008/05/28/get-your-local-ip-address/
|
126
127
|
def local_ip
|
127
128
|
orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true # turn off reverse DNS resolution temporarily
|
128
|
-
|
129
|
+
|
129
130
|
UDPSocket.open do |s|
|
130
131
|
s.connect '64.233.187.99', 1
|
131
132
|
IPAddr.new(s.addr.last).to_i
|
132
133
|
end
|
133
134
|
ensure
|
134
135
|
Socket.do_not_reverse_lookup = orig
|
135
|
-
end
|
136
|
+
end
|
136
137
|
end
|
137
|
-
end
|
138
|
+
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
require 'socket'
|
2
3
|
require 'ipaddr'
|
3
4
|
require 'uri'
|
@@ -222,6 +223,10 @@ module Politics
|
|
222
223
|
workers
|
223
224
|
end
|
224
225
|
|
226
|
+
def hostname
|
227
|
+
nil
|
228
|
+
end
|
229
|
+
|
225
230
|
private
|
226
231
|
|
227
232
|
def restart_wanted?
|
@@ -328,7 +333,7 @@ module Politics
|
|
328
333
|
end
|
329
334
|
|
330
335
|
def register_with_bonjour
|
331
|
-
server = DRb.start_service(
|
336
|
+
server = DRb.start_service("druby://#{hostname || ""}:0", self)
|
332
337
|
@uri = DRb.uri
|
333
338
|
@port = URI.parse(DRb.uri).port
|
334
339
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
begin
|
2
3
|
require 'memcache'
|
3
4
|
rescue LoadError => e
|
@@ -13,8 +14,8 @@ module Politics
|
|
13
14
|
# The worker instance must obtain the leader token before performing some task.
|
14
15
|
# We use a memcached server as a central token authority to provide a shared,
|
15
16
|
# network-wide view for all processors. This reliance on a single resource means
|
16
|
-
# if your memcached server goes down, so do the processors. Oftentimes,
|
17
|
-
# this is an acceptable trade-off since many high-traffic web sites would
|
17
|
+
# if your memcached server goes down, so do the processors. Oftentimes,
|
18
|
+
# this is an acceptable trade-off since many high-traffic web sites would
|
18
19
|
# not be useable without memcached running anyhow.
|
19
20
|
#
|
20
21
|
# Essentially each TokenWorker attempts to elect itself every +:iteration_length+
|
@@ -46,7 +47,7 @@ module Politics
|
|
46
47
|
# This can often times be quite beneficial (e.g. leveraging a warm cache from the last iteration)
|
47
48
|
# for performance but is left to the reader to implement.
|
48
49
|
module TokenWorker
|
49
|
-
|
50
|
+
|
50
51
|
def self.included(model) #:nodoc:
|
51
52
|
model.class_eval do
|
52
53
|
attr_accessor :memcache_client, :token, :iteration_length, :worker_name
|
@@ -59,7 +60,7 @@ module Politics
|
|
59
60
|
# Register this instance as a worker.
|
60
61
|
#
|
61
62
|
# Options:
|
62
|
-
# +:iteration_length+:: The length of a processing iteration, in seconds. The
|
63
|
+
# +:iteration_length+:: The length of a processing iteration, in seconds. The
|
63
64
|
# leader's 'reign' lasts for this length of time.
|
64
65
|
# +:servers+:: An array of memcached server strings
|
65
66
|
def register_worker(name, config={})
|
@@ -77,7 +78,7 @@ module Politics
|
|
77
78
|
|
78
79
|
cleanup
|
79
80
|
end
|
80
|
-
|
81
|
+
|
81
82
|
def process(*args, &block)
|
82
83
|
verify_registration
|
83
84
|
|
@@ -103,18 +104,18 @@ module Politics
|
|
103
104
|
Politics::log.error(me.backtrace.join("\n"))
|
104
105
|
self.memcache_client.reset
|
105
106
|
end
|
106
|
-
|
107
|
+
|
107
108
|
pause_until_expiry(time)
|
108
109
|
reset_state
|
109
110
|
end while loop?
|
110
111
|
end
|
111
112
|
|
112
113
|
private
|
113
|
-
|
114
|
+
|
114
115
|
def reset_state
|
115
116
|
@leader = nil
|
116
117
|
end
|
117
|
-
|
118
|
+
|
118
119
|
def verify_registration
|
119
120
|
unless self.class.worker_instance
|
120
121
|
raise ArgumentError, "Cannot call process without first calling register_worker"
|
@@ -123,30 +124,30 @@ module Politics
|
|
123
124
|
raise SecurityError, "Only one instance of #{self.class} per process. Another instance was created after this one."
|
124
125
|
end
|
125
126
|
end
|
126
|
-
|
127
|
+
|
127
128
|
def loop?
|
128
129
|
true
|
129
130
|
end
|
130
|
-
|
131
|
+
|
131
132
|
def cleanup
|
132
133
|
at_exit do
|
133
134
|
memcache_client.delete(token) if leader?
|
134
135
|
end
|
135
136
|
end
|
136
|
-
|
137
|
+
|
137
138
|
def pause_until_expiry(elapsed)
|
138
139
|
pause_time = (iteration_length - elapsed).to_f
|
139
140
|
if pause_time > 0
|
140
|
-
relax(pause_time)
|
141
|
+
relax(pause_time)
|
141
142
|
else
|
142
143
|
raise ArgumentError, "Negative iteration time left. Assuming the worst and exiting... #{iteration_length}/#{elapsed}"
|
143
144
|
end
|
144
145
|
end
|
145
|
-
|
146
|
+
|
146
147
|
def relax(time)
|
147
148
|
sleep time
|
148
149
|
end
|
149
|
-
|
150
|
+
|
150
151
|
# Nominate ourself as leader by contacting the memcached server
|
151
152
|
# and attempting to add the token with our name attached.
|
152
153
|
# The result will tell us if memcached stored our value and therefore
|
data/lib/politics.rb
CHANGED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# Require this file using `require "spec_helper"` to ensure that it is only
|
4
|
+
# loaded once.
|
5
|
+
#
|
6
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
7
|
+
RSpec.configure do |config|
|
8
|
+
config.treat_symbols_as_metadata_keys_with_true_values = true
|
9
|
+
config.run_all_when_everything_filtered = true
|
10
|
+
config.filter_run :focus
|
11
|
+
|
12
|
+
# Run specs in random order to surface order dependencies. If you find an
|
13
|
+
# order dependency and want to debug it, you can fix the order by providing
|
14
|
+
# the seed, which is printed after each run.
|
15
|
+
# --seed 1234
|
16
|
+
config.order = 'random'
|
17
|
+
end
|