zeevex_cluster 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +5 -0
- data/Gemfile +22 -0
- data/Rakefile +44 -0
- data/doc/BUGS-zookeeper.txt +60 -0
- data/doc/TODO.txt +85 -0
- data/lib/zeevex_cluster/base.rb +95 -0
- data/lib/zeevex_cluster/coordinator/base_key_val_store.rb +85 -0
- data/lib/zeevex_cluster/coordinator/memcached.rb +118 -0
- data/lib/zeevex_cluster/coordinator/mysql.rb +396 -0
- data/lib/zeevex_cluster/coordinator/redis.rb +101 -0
- data/lib/zeevex_cluster/coordinator.rb +29 -0
- data/lib/zeevex_cluster/election.rb +102 -0
- data/lib/zeevex_cluster/message.rb +52 -0
- data/lib/zeevex_cluster/nil_logger.rb +7 -0
- data/lib/zeevex_cluster/serializer/json_hash.rb +67 -0
- data/lib/zeevex_cluster/serializer.rb +27 -0
- data/lib/zeevex_cluster/static.rb +67 -0
- data/lib/zeevex_cluster/strategy/base.rb +92 -0
- data/lib/zeevex_cluster/strategy/cas.rb +403 -0
- data/lib/zeevex_cluster/strategy/static.rb +55 -0
- data/lib/zeevex_cluster/strategy/unclustered.rb +9 -0
- data/lib/zeevex_cluster/strategy/zookeeper.rb +163 -0
- data/lib/zeevex_cluster/strategy.rb +12 -0
- data/lib/zeevex_cluster/synchronized.rb +46 -0
- data/lib/zeevex_cluster/unclustered.rb +11 -0
- data/lib/zeevex_cluster/util/logging.rb +7 -0
- data/lib/zeevex_cluster/util.rb +15 -0
- data/lib/zeevex_cluster/version.rb +3 -0
- data/lib/zeevex_cluster.rb +29 -0
- data/script/election.rb +46 -0
- data/script/memc.rb +13 -0
- data/script/mysql.rb +25 -0
- data/script/redis.rb +14 -0
- data/script/repl +10 -0
- data/script/repl.rb +8 -0
- data/script/ser.rb +11 -0
- data/script/static.rb +34 -0
- data/script/testall +2 -0
- data/spec/cluster_static_spec.rb +49 -0
- data/spec/cluster_unclustered_spec.rb +32 -0
- data/spec/coordinator/coordinator_memcached_spec.rb +102 -0
- data/spec/message_spec.rb +38 -0
- data/spec/serializer/json_hash_spec.rb +68 -0
- data/spec/shared_master_examples.rb +20 -0
- data/spec/shared_member_examples.rb +39 -0
- data/spec/shared_non_master_examples.rb +8 -0
- data/spec/spec_helper.rb +14 -0
- data/zeevex_cluster.gemspec +43 -0
- metadata +298 -0
data/Gemfile
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
group :development, :test do
|
4
|
+
gem 'pry'
|
5
|
+
gem 'pry-remote'
|
6
|
+
gem 'pry-doc'
|
7
|
+
gem 'pry-nav'
|
8
|
+
gem 'pry-buffers'
|
9
|
+
gem 'pry-syntax-hacks'
|
10
|
+
gem 'pry-git', :platform => :mri
|
11
|
+
gem 'jist'
|
12
|
+
gem 'ruby18_source_location', :platform => :mri_18
|
13
|
+
gem 'mysql2', :platform => :mri
|
14
|
+
end
|
15
|
+
|
16
|
+
group :development, :test do
|
17
|
+
gem 'zk', :path => '/Users/Shared/squid/src/github/cluster/zk'
|
18
|
+
gem 'zk-group', :path => '/Users/Shared/squid/src/github/cluster/zk-group'
|
19
|
+
end
|
20
|
+
|
21
|
+
# Specify your gem's dependencies in zeevex_cluster.gemspec
|
22
|
+
gemspec
|
data/Rakefile
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'bundler'
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
|
+
|
4
|
+
|
5
|
+
require 'rspec/core/rake_task'
|
6
|
+
|
7
|
+
RSpec::Core::RakeTask.new(:spec)
|
8
|
+
|
9
|
+
namespace :spec do
|
10
|
+
SPEC_PLATFORMS = ENV.has_key?('SPEC_PLATFORMS') ?
|
11
|
+
ENV['SPEC_PLATFORMS'].split(/ +/) :
|
12
|
+
%w{1.9.3-p448 2.0.0-p247 1.8.7-p374}
|
13
|
+
|
14
|
+
desc "Run on three Rubies"
|
15
|
+
task :platforms do
|
16
|
+
# current = %x[rbenv version | awk '{print $1}']
|
17
|
+
|
18
|
+
fail = false
|
19
|
+
SPEC_PLATFORMS.each do |version|
|
20
|
+
puts "Switching to #{version}"
|
21
|
+
Bundler.with_clean_env do
|
22
|
+
system %{bash -c 'eval "$(rbenv init -)" && rbenv use #{version} && rbenv rehash && ruby -v && bundle exec rake spec'}
|
23
|
+
end
|
24
|
+
if $?.exitstatus != 0
|
25
|
+
fail = true
|
26
|
+
break
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
exit (fail ? 1 : 0)
|
31
|
+
end
|
32
|
+
|
33
|
+
desc 'Install gems for all tested rubies'
|
34
|
+
task :platform_setup do
|
35
|
+
SPEC_PLATFORMS.each do |version|
|
36
|
+
puts "Setting up platform #{version}"
|
37
|
+
Bundler.with_clean_env do
|
38
|
+
system %{bash -c 'eval "$(rbenv init -)" && rbenv use #{version} && rbenv rehash && gem install bundler && bundle install'}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
task :default => 'spec'
|
@@ -0,0 +1,60 @@
|
|
1
|
+
When suspending master with Ctrl-Z for a while, the other 2 nodes in a
|
2
|
+
3 node cluster elect a new master and consider the suspended node to
|
3
|
+
have exited.
|
4
|
+
|
5
|
+
Resuming the suspended node shows that it is not fully aware of its altered
|
6
|
+
status, though note that zk-group shows it's not a member anymore.
|
7
|
+
|
8
|
+
[17] MASTER[2] pry(main)>
|
9
|
+
$c.@strategy._get_synchronized_object.@elector.leader?
|
10
|
+
=> true
|
11
|
+
[18] MASTER[2] pry(main)>
|
12
|
+
[19] MASTER[2] pry(main)> $c.members
|
13
|
+
=> ["blacktip.esquimaux.org:/dev/ttys005",
|
14
|
+
"blacktip.esquimaux.org:/dev/ttys006"]
|
15
|
+
[20] MASTER[2] pry(main)> $c.nodename
|
16
|
+
=> "blacktip.esquimaux.org:/dev/ttys003"
|
17
|
+
|
18
|
+
Do we need to act on ZK session/connection changes?
|
19
|
+
|
20
|
+
ZK::Client::Base#methods: assert_we_are_not_on_the_event_dispatch_thread! children delete event_dispatch_thread? event_handler exists? get get_acl inspect register session_id session_passwd set set_acl set_debug_level stat wait_until_connected watcher
|
21
|
+
ZK::Client::StateMixin#methods: on_connected on_connecting on_expired_session on_state_change
|
22
|
+
Zookeeper::Constants#methods: event_by_value state_by_value
|
23
|
+
ZK::Client::Unixisms#methods: block_until_node_deleted find mkdir_p rm_rf
|
24
|
+
ZK::Client::Conveniences#methods: defer election_candidate election_observer exclusive_locker locker ping? queue shared_locker with_lock
|
25
|
+
ZK::Logging#methods: logger
|
26
|
+
ZK::Client::Threaded#methods: associating? client_state close close! closed? connect connected? connecting? create expired_session? on_exception on_threadpool? pause_before_fork_in_parent raw_event_handler reopen resume_after_fork_in_parent state wait_until_closed wait_until_connected_or_dying
|
27
|
+
instance variables: @client_state @cnx @cond @connection_timeout @event_handler @fork_subs @host @last_cnx_state @mutex @pid @reconnect @reconnect_thread @retry_duration @threadpool
|
28
|
+
|
29
|
+
----
|
30
|
+
log after resume:
|
31
|
+
|
32
|
+
[1] + 28262 continued bundle exec ./script/election.rb zookeeper
|
33
|
+
[8] MASTER[3] pry(main)>
|
34
|
+
[9] MASTER[3] pry(main)> D, [2012-12-24T14:08:51.765368 #28262] DEBUG -- : EventHandler#process dispatching event: #<Zookeeper::Callbacks::WatcherCallback:0x109b0a1d0 @path="/_zk/groups/foobs", @type=4, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @completed=false, @context=nil>
|
35
|
+
D, [2012-12-24T14:08:51.766028 #28262] DEBUG -- : EventHandler#process dispatching event: #<Zookeeper::Callbacks::WatcherCallback:0x10993b278 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=1, @completed=false, @context=nil>
|
36
|
+
D, [2012-12-24T14:08:51.766576 #28262] DEBUG -- : called #<ZK::EventHandlerSubscription::Base:0x109d6d5d0 @mutex=#<ZK::Monitor:0x109d6d530 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @path="/_zk/groups/foobs", @parent=#<ZK::EventHandler:0x109d7b0e0 @thread_opt=:single, @mutex=#<ZK::Monitor:0x109d7abb8 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @default_watcher_block=#<Proc:0x0000000109f905d8@/Users/Shared/squid/src/github/cluster/zk/lib/zk/event_handler.rb:251>, @orig_pid=28262, @state=:running, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd0078 ...>, @outstanding_watches={:child=>#<Set: {}>, :data=>#<Set: {}>}, @callbacks={"/_zkelection/foobs/leader_ack"=>[], :all_node_events=>[], "/_zk/groups/foobs"=>[#<ZK::EventHandlerSubscription::Base:0x109d6d5d0 ...>], :all_state_events=>[], "state_1"=>[], "state_3"=>[#<ZK::EventHandlerSubscription::Base:0x109d6cf40 @mutex=#<ZK::Monitor:0x109d6cea0 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @path="state_3", @parent=#<ZK::EventHandler:0x109d7b0e0 ...>, @interests=#<Set: {:deleted, :changed, :child, :created}>, @callable=#<Proc:0x0000000109d96520@/Users/Shared/squid/src/github/cluster/zk-group/lib/zk-group/group.rb:90>>]}>, @interests=#<Set: {:child}>, @callable=#<Proc:0x0000000109d96868@/Users/Shared/squid/src/github/cluster/zk-group/lib/zk-group/group.rb:86>> with [#<Zookeeper::Callbacks::WatcherCallback:0x109b0a1d0 @path="/_zk/groups/foobs", @type=4, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd0078 ...>, @completed=true, @context=nil>] on threadpool
|
37
|
+
D, [2012-12-24T14:08:51.766643 #28262] DEBUG -- : session state was invalid, calling reopen
|
38
|
+
D, [2012-12-24T14:08:51.766687 #28262] DEBUG -- : EventHandler#process dispatching event: #<Zookeeper::Callbacks::WatcherCallback:0x1099259a0 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=-112, @completed=false, @context=nil>
|
39
|
+
D, [2012-12-24T14:08:51.766743 #28262] DEBUG -- : reopening, no fork detected
|
40
|
+
D, [2012-12-24T14:08:51.771478 #28262] DEBUG -- : EventHandler#process dispatching event: #<Zookeeper::Callbacks::WatcherCallback:0x1098fa188 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @completed=false, @context=nil>
|
41
|
+
D, [2012-12-24T14:08:51.771547 #28262] DEBUG -- : wait_until_connected_or_dying @last_cnx_state: 3, time_left? true, @client_state: :running
|
42
|
+
D, [2012-12-24T14:08:51.771710 #28262] DEBUG -- : reopen returned: :connected
|
43
|
+
D, [2012-12-24T14:08:51.771908 #28262] DEBUG -- : broadcast_membership_change! received event #<Zookeeper::Callbacks::WatcherCallback:0x109b0a1d0 @path="/_zk/groups/foobs", @type=4, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd007b ...>, @completed=true, @context=nil>
|
44
|
+
D, [2012-12-24T14:08:51.772264 #28262] DEBUG -- : called #<ZK::EventHandlerSubscription::Base:0x109d6cf40 @mutex=#<ZK::Monitor:0x109d6cea0 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @path="state_3", @parent=#<ZK::EventHandler:0x109d7b0e0 @thread_opt=:single, @mutex=#<ZK::Monitor:0x109d7abb8 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @default_watcher_block=#<Proc:0x0000000109f905d8@/Users/Shared/squid/src/github/cluster/zk/lib/zk/event_handler.rb:251>, @orig_pid=28262, @state=:running, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd007b ...>, @outstanding_watches={:child=>#<Set: {}>, :data=>#<Set: {}>}, @callbacks={"/_zkelection/foobs/leader_ack"=>[], :all_node_events=>[], "/_zk/groups/foobs"=>[#<ZK::EventHandlerSubscription::Base:0x109d6d5d0 @mutex=#<ZK::Monitor:0x109d6d530 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @path="/_zk/groups/foobs", @parent=#<ZK::EventHandler:0x109d7b0e0 ...>, @interests=#<Set: {:child}>, @callable=#<Proc:0x0000000109d96868@/Users/Shared/squid/src/github/cluster/zk-group/lib/zk-group/group.rb:86>>], :all_state_events=>[], "state_1"=>[], "state_3"=>[#<ZK::EventHandlerSubscription::Base:0x109d6cf40 ...>], "state_-112"=>[]}>, @interests=#<Set: {:deleted, :changed, :child, :created}>, @callable=#<Proc:0x0000000109d96520@/Users/Shared/squid/src/github/cluster/zk-group/lib/zk-group/group.rb:90>> with [#<Zookeeper::Callbacks::WatcherCallback:0x1098fa188 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd007b ...>, @completed=true, @context=nil>] on threadpool
|
45
|
+
D, [2012-12-24T14:08:51.772987 #28262] DEBUG -- : last_members: ["m0000000045", "m0000000046", "m0000000047"]
|
46
|
+
D, [2012-12-24T14:08:51.773031 #28262] DEBUG -- : @known_members: ["m0000000046", "m0000000047"]
|
47
|
+
D, [2012-12-24T14:08:51.774129 #28262] DEBUG -- : ZK: membership change from ZK::Group: from ["blacktip.esquimaux.org:/dev/ttys003", "blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"] to ["blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"]
|
48
|
+
D, [2012-12-24T14:08:51.774173 #28262] DEBUG -- : <running hook membership_change([["blacktip.esquimaux.org:/dev/ttys003", "blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"], ["blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"]])>
|
49
|
+
D, [2012-12-24T14:08:51.774221 #28262] DEBUG -- : ZeevexCluster::Election observed hook: membership_change [["blacktip.esquimaux.org:/dev/ttys003", "blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"], ["blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"]]
|
50
|
+
D, [2012-12-24T14:08:51.774276 #28262] DEBUG -- : <running hook strategy_membership_change([["blacktip.esquimaux.org:/dev/ttys003", "blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"], ["blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"]])>
|
51
|
+
D, [2012-12-24T14:08:51.774410 #28262] DEBUG -- : broadcast_membership_change! received event #<Zookeeper::Callbacks::WatcherCallback:0x1098fa188 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd007b ...>, @completed=true, @context=nil>
|
52
|
+
D, [2012-12-24T14:08:51.775273 #28262] DEBUG -- : last_members: ["m0000000046", "m0000000047"]
|
53
|
+
D, [2012-12-24T14:08:51.775313 #28262] DEBUG -- : @known_members: ["m0000000046", "m0000000047"]
|
54
|
+
D, [2012-12-24T14:08:51.775342 #28262] DEBUG -- : membership data did not actually change, not notifying
|
55
|
+
[9] MASTER[3] pry(main)>
|
56
|
+
[10] MASTER[2] pry(main)>
|
57
|
+
|
58
|
+
|
59
|
+
-----------------------------
|
60
|
+
|
data/doc/TODO.txt
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
Goals:
|
2
|
+
|
3
|
+
- Stable leader election
|
4
|
+
- Membership w/node info
|
5
|
+
- Simple configuration management
|
6
|
+
- Leases and Locks
|
7
|
+
- Message queue (low performance target)
|
8
|
+
|
9
|
+
----
|
10
|
+
|
11
|
+
* Mutex around every method that might have cross-thread access,
|
12
|
+
including:
|
13
|
+
* coordinator objects
|
14
|
+
|
15
|
+
* Sort out the strategy thing - is it really useful? Is CAS really the kind of strategy? Can coordinators
|
16
|
+
be used for more than one kind of strategy?
|
17
|
+
|
18
|
+
** TESTS!
|
19
|
+
* CAS should be run against a mock backend
|
20
|
+
* Cluster should be run against a mock strategy
|
21
|
+
|
22
|
+
* Other user code integration, including event runloops perhaps?
|
23
|
+
|
24
|
+
** Backends
|
25
|
+
* All
|
26
|
+
* In membership or cluster info, check for (sem)version compatibility
|
27
|
+
* Redis
|
28
|
+
* Use queues and PUB/SUB for messaging
|
29
|
+
* Use e.g. atomic hashes instead of CAS writes for group membership to scale better?
|
30
|
+
* MySQL backend
|
31
|
+
* auto-create memory table if it doesn't exist
|
32
|
+
* Use MySQL locks somehow to implement non-poll messaging?
|
33
|
+
* memcached
|
34
|
+
* implement queue-like structure for messaging (using append/prepend, CAS for reads)
|
35
|
+
* Easy way to notify a node it should check for various conditions
|
36
|
+
* switch to dalli as client
|
37
|
+
* disable memcache-client's backoff feature (waits N seconds after
|
38
|
+
conn error before allowing next contact)
|
39
|
+
* Moneto (ruby key/value store abstraction - needs CAS, though)
|
40
|
+
* AR backend
|
41
|
+
* MongoDB backend
|
42
|
+
- replica sets and/or write concerns for replication / HA?
|
43
|
+
- tailable cursors for queues
|
44
|
+
* ZK (ruby Zookeeper wrapper) backend - would be very thin, as ZK provides all that this
|
45
|
+
lib plans to
|
46
|
+
* Local/in-process/filesystem backend for testing
|
47
|
+
|
48
|
+
* More core functionality / cluster primitives
|
49
|
+
* Global Cluster info struct, and version checking
|
50
|
+
* Group and single node messaging
|
51
|
+
* Leases - can reuse most of leader election code for this; in fact leader election
|
52
|
+
can be a wrapper around a lease
|
53
|
+
* Locks (auto-released when member exits / times out?)
|
54
|
+
|
55
|
+
** General user code API
|
56
|
+
* Health check callback into user code - when failed, notify members, resign, and leave
|
57
|
+
* Clearer policy for cross-thread callbacks
|
58
|
+
* Separate out membership from leader election
|
59
|
+
* Cluster configuration file format
|
60
|
+
|
61
|
+
** Leader election
|
62
|
+
* Replace fixed @stale_time with lease duration specified by candidate
|
63
|
+
* Different members will notice master state changes at different times, depending on their polling.
|
64
|
+
Should they coordinate or predict when scheduled changes are happening?
|
65
|
+
* Callback for no master, and suspect master
|
66
|
+
|
67
|
+
* Implement the state machines in terms of actual state machine gems instead of spaghetti
|
68
|
+
|
69
|
+
** Membership
|
70
|
+
* Callbacks into user code for group membership changes - member joined, suspect, left
|
71
|
+
* Fast failure of member - attempt to query when suspect
|
72
|
+
* Kick live member?
|
73
|
+
* After a long partition (including ^Z in which the client isn't running at all), the lib and
|
74
|
+
user code should (locally) leave the cluster and rejoin from scratch - otherwise other nodes
|
75
|
+
will see it as having left, but it may think it never did
|
76
|
+
- Similarly, if it goes to update its membership record and finds it missing, then it should
|
77
|
+
consider itself kicked
|
78
|
+
|
79
|
+
|
80
|
+
|
81
|
+
|
82
|
+
Dreamlist:
|
83
|
+
|
84
|
+
* Consistent causal ordering of all callbacks / messages on all members?
|
85
|
+
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'socket'
|
2
|
+
require 'hookem'
|
3
|
+
|
4
|
+
module ZeevexCluster
|
5
|
+
class Base
|
6
|
+
include ZeevexCluster::Util::Logging
|
7
|
+
include Hookem
|
8
|
+
|
9
|
+
attr_accessor :nodename, :options
|
10
|
+
|
11
|
+
def initialize(options = {})
|
12
|
+
@options = {:nodename => Socket.gethostname,
|
13
|
+
:autojoin => true}.merge(options)
|
14
|
+
@logger = @options[:logger]
|
15
|
+
|
16
|
+
_initialize_hook_module
|
17
|
+
|
18
|
+
if @options[:hooks]
|
19
|
+
add_hooks @options[:hooks]
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def join
|
24
|
+
raise NotImplementedError
|
25
|
+
end
|
26
|
+
|
27
|
+
def leave
|
28
|
+
raise NotImplementedError
|
29
|
+
end
|
30
|
+
|
31
|
+
def master?
|
32
|
+
raise NotImplementedError
|
33
|
+
end
|
34
|
+
|
35
|
+
def member?
|
36
|
+
raise NotImplementedError
|
37
|
+
end
|
38
|
+
|
39
|
+
##
|
40
|
+
## Make this node the master, returning true if successful.
|
41
|
+
##
|
42
|
+
def make_master!
|
43
|
+
raise NotImplementedError
|
44
|
+
end
|
45
|
+
|
46
|
+
##
|
47
|
+
## Make this node the master if not already the master
|
48
|
+
## if provided a block, run that IFF we are the master
|
49
|
+
##
|
50
|
+
def ensure_master(&block)
|
51
|
+
make_master! unless master?
|
52
|
+
if block
|
53
|
+
run_if_master &block
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
##
|
58
|
+
## Run the code block only if this node is the master
|
59
|
+
##
|
60
|
+
def run_if_master(&block)
|
61
|
+
if master?
|
62
|
+
block.call
|
63
|
+
else
|
64
|
+
false
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
##
|
69
|
+
## Resign from mastership; returns false if this is the only node.
|
70
|
+
##
|
71
|
+
def resign!
|
72
|
+
raise NotImplementedError
|
73
|
+
end
|
74
|
+
|
75
|
+
##
|
76
|
+
## Return name of master node
|
77
|
+
##
|
78
|
+
def master
|
79
|
+
raise NotImplementedError
|
80
|
+
end
|
81
|
+
|
82
|
+
##
|
83
|
+
## Return this node's name
|
84
|
+
##
|
85
|
+
def nodename
|
86
|
+
options[:nodename]
|
87
|
+
end
|
88
|
+
|
89
|
+
protected
|
90
|
+
|
91
|
+
def after_initialize
|
92
|
+
join if options.fetch(:autojoin, true)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'zeevex_cluster/coordinator'
|
2
|
+
|
3
|
+
class ZeevexCluster::Coordinator::BaseKeyValStore
|
4
|
+
include ZeevexCluster::Util::Logging
|
5
|
+
|
6
|
+
def self.setup
|
7
|
+
unless @setup
|
8
|
+
require 'memcache'
|
9
|
+
require 'zeevex_cluster/serializer/json_hash'
|
10
|
+
@setup = true
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(options = {})
|
15
|
+
self.class.setup
|
16
|
+
@options = options
|
17
|
+
if (!options[:server] && !options[:client]) || !options[:expiration]
|
18
|
+
raise ArgumentError, "Must supply [:server or :client] and :expiration"
|
19
|
+
end
|
20
|
+
if options[:client]
|
21
|
+
@client = options[:client]
|
22
|
+
else
|
23
|
+
@server = options[:server]
|
24
|
+
@port = options[:port] || 11211
|
25
|
+
end
|
26
|
+
@expiration = options[:expiration] || 60
|
27
|
+
|
28
|
+
@logger = options[:logger]
|
29
|
+
|
30
|
+
@serializer = options[:serializer] || ZeevexCluster::Serializer::JsonHash.new
|
31
|
+
|
32
|
+
@retries = options.fetch(:retries, 20)
|
33
|
+
@retry_wait = options.fetch(:retry_wait, 2)
|
34
|
+
@retry_bo = options.fetch(:retry_bo, 1.5)
|
35
|
+
end
|
36
|
+
|
37
|
+
[:add, :set, :cas, :get].each do |name|
|
38
|
+
define_method "#{name}_with_retry", lambda { |*args, &block|
|
39
|
+
with_connection_retry name, {}, *args, &block
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
def with_connection_retry(method, options = {}, *args, &block)
|
44
|
+
retry_left = options.fetch(:retries, @retries)
|
45
|
+
retry_wait = options.fetch(:retry_wait, @retry_wait)
|
46
|
+
begin
|
47
|
+
send "do_#{method}", *args, &block
|
48
|
+
rescue ZeevexCluster::Coordinator::ConnectionError
|
49
|
+
if retry_left > 0
|
50
|
+
logger.debug "retrying after #{retry_wait} seconds"
|
51
|
+
retry_left -= 1
|
52
|
+
sleep retry_wait
|
53
|
+
retry_wait = retry_wait * options.fetch('retry_bo', @retry_bo)
|
54
|
+
retry
|
55
|
+
else
|
56
|
+
logger.error 'Ran out of connection retries, re-raising'
|
57
|
+
raise
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
protected
|
63
|
+
|
64
|
+
def serialize_value(obj, raw = false)
|
65
|
+
raw ? obj : @serializer.serialize(obj)
|
66
|
+
end
|
67
|
+
|
68
|
+
def deserialize_value(str, raw = false)
|
69
|
+
raw ? str : @serializer.deserialize(str)
|
70
|
+
end
|
71
|
+
|
72
|
+
def is_raw?(options)
|
73
|
+
(@options && @options[:raw]) || (options && options[:raw])
|
74
|
+
end
|
75
|
+
|
76
|
+
def to_key(key)
|
77
|
+
if @options[:namespace]
|
78
|
+
"#{@options[:namespace]}:#{key}"
|
79
|
+
elsif @options[:to_key_proc]
|
80
|
+
@options[:to_key_proc].call(key)
|
81
|
+
else
|
82
|
+
key.to_s
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'zeevex_cluster/coordinator/base_key_val_store'
|
2
|
+
|
3
|
+
module ZeevexCluster::Coordinator
|
4
|
+
class Memcached < BaseKeyValStore
|
5
|
+
def self.setup
|
6
|
+
unless @setup
|
7
|
+
require 'memcache'
|
8
|
+
BaseKeyValStore.setup
|
9
|
+
|
10
|
+
@setup = true
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(options = {})
|
15
|
+
super
|
16
|
+
@client ||= MemCache.new "#@server:#@port"
|
17
|
+
end
|
18
|
+
|
19
|
+
def add(key, value, options = {})
|
20
|
+
status( @client.add(to_key(key), serialize_value(value, options[:raw]),
|
21
|
+
options.fetch(:expiration, @expiration), raw?) ) == STORED
|
22
|
+
rescue MemCache::MemCacheError
|
23
|
+
raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
|
24
|
+
end
|
25
|
+
|
26
|
+
def set(key, value, options = {})
|
27
|
+
status( @client.set(to_key(key), serialize_value(value, options[:raw]),
|
28
|
+
options.fetch(:expiration, @expiration), raw?) ) == STORED
|
29
|
+
rescue MemCache::MemCacheError
|
30
|
+
raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
|
31
|
+
end
|
32
|
+
|
33
|
+
def delete(key, options = {})
|
34
|
+
status( @client.delete(to_key(key)) ) == DELETED
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Block is passed the current value, and returns the updated value.
|
39
|
+
#
|
40
|
+
# Block can raise DontChange to simply exit the block without updating.
|
41
|
+
#
|
42
|
+
# returns nil for no value
|
43
|
+
# returns false for failure (somebody else set)
|
44
|
+
# returns true for success
|
45
|
+
#
|
46
|
+
def cas(key, options = {}, &block)
|
47
|
+
res = @client.cas(to_key(key), options.fetch(:expiration, @expiration), raw?) do |inval|
|
48
|
+
serialize_value(yield(deserialize_value(inval, options[:raw])), options[:raw])
|
49
|
+
end
|
50
|
+
case status(res)
|
51
|
+
when nil then nil
|
52
|
+
when EXISTS then false
|
53
|
+
when STORED then true
|
54
|
+
else raise "Unhandled status code: #{res}"
|
55
|
+
end
|
56
|
+
rescue ZeevexCluster::Coordinator::DontChange
|
57
|
+
false
|
58
|
+
rescue MemCache::MemCacheError
|
59
|
+
raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
|
60
|
+
end
|
61
|
+
|
62
|
+
def get(key, options = {})
|
63
|
+
val = @client.get(to_key(key), raw?)
|
64
|
+
if val && !options[:raw]
|
65
|
+
deserialize_value(val)
|
66
|
+
else
|
67
|
+
val
|
68
|
+
end
|
69
|
+
rescue MemCache::MemCacheError
|
70
|
+
raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
|
71
|
+
end
|
72
|
+
|
73
|
+
def append(key, val, options = {})
|
74
|
+
val = serialize_value(val, options[:raw])
|
75
|
+
key = to_key(key)
|
76
|
+
status( @client.append(key, val) ) == STORED ||
|
77
|
+
status( @client.add(key, val, options.fetch(:expiration, @expiration), true) ) == STORED ||
|
78
|
+
status( @client.append(key, val) ) == STORED
|
79
|
+
rescue MemCache::MemCacheError
|
80
|
+
raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
|
81
|
+
end
|
82
|
+
|
83
|
+
def prepend(key, val, options = {})
|
84
|
+
val = serialize_value(val, options[:raw])
|
85
|
+
key = to_key(key)
|
86
|
+
status( @client.prepend(key, val) ) == STORED ||
|
87
|
+
status( @client.add(key, val, options.fetch(:expiration, @expiration), true) ) == STORED ||
|
88
|
+
status( @client.prepend(key, val) ) == STORED
|
89
|
+
rescue MemCache::MemCacheError
|
90
|
+
raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
|
91
|
+
end
|
92
|
+
|
93
|
+
def push_to_queue(key, object, options = {})
|
94
|
+
|
95
|
+
end
|
96
|
+
|
97
|
+
protected
|
98
|
+
|
99
|
+
STORED = 'STORED'
|
100
|
+
EXISTS = 'EXISTS'
|
101
|
+
NOT_STORED = 'NOT_STORED'
|
102
|
+
NOT_FOUND = 'NOT_FOUND'
|
103
|
+
DELETED = 'DELETED'
|
104
|
+
|
105
|
+
def status(response)
|
106
|
+
case response
|
107
|
+
when nil, true, false then response
|
108
|
+
when String then response.chomp
|
109
|
+
else
|
110
|
+
raise ArgumentError, "This should only be called on results from cas, add, set, etc. - got result #{response.inspect}"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def raw?
|
115
|
+
true
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|