corosync-commander 0.0.2 → 0.1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f9f90af1b3fac9c2164e54e664cae15be76ef65
-  data.tar.gz: ccddc4d241a87b80def6ce1ad6dae3073a520e5e
+  metadata.gz: 9ffd9a76d120685a5cc4ca5b46d63f416c25f2ac
+  data.tar.gz: 8c9f81c68f0ac85a2bc053f634e0f61a8f86257a
 SHA512:
-  metadata.gz: 47de186e7ba2e4bfc304761b6e704f776ded20ab55d3428fdb5dbd5d5e886511fc80ee438003d357a269c5f66703f5c68142502507c38a6b0d55004ccccdf3c1
-  data.tar.gz: 47136a7dbb32771007e34f76a4ae86341b14d360b6cbfa7988e34c05533451961991c2564cce2bcf6da891190d3cbba17129c9f1017649066c472902223764ad
+  metadata.gz: 5e6ca5cd16bca2f7e87392357cc65de13e6e9f53608d8b3f87f1aef52332f1df86a027bc56abed06a42263ac52627612730bb371664837f0c42d82e263a2375b
+  data.tar.gz: e001b910753074d6633c0704f7c10099119d6d6dcd006e20851e5c4b46a443a1edfe5f19aac1cd37aad39cd59c86ff856db545a7ea8195f9cdff48ef7c61f81a

data/Gemfile

@@ -1,7 +1,8 @@
 # A sample Gemfile
 source "https://rubygems.org"
 
-gem 'corosync', '~>0.0.3'
+gem 'corosync', '~>0.1.0'
+#gem 'corosync', :path => '/home/phemmer/git/ruby-corosync'
 
 group :development do
 	gem 'rake'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    corosync (0.0.3)
+    corosync (0.1.0.0)
       ffi (~> 1.9)
     diff-lcs (1.2.4)
     ffi (1.9.3)
@@ -23,7 +23,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  corosync (~> 0.0.3)
+  corosync (~> 0.1.0)
   rake
   rdoc
   rspec

data/README.md

@@ -0,0 +1,30 @@
+# corosync-commander
+
+## Description
+corosync-commander is a simplified interface into cluster communication via the Corosync library, based on the [corosync gem](http://github.com/phemmer/ruby-corosync/).
+
+It allows you to build apps which communicate with each other in a reliable fashion. You can send messages to the apps and ensure that every app receives the message in the exact same order. This lets you synchronize data between the apps by only sending deltas.  
+The key to this is that when you send a message to the cluster, you receive your own message. Thus corosync-commander becomes a communication bus between the frontend and backend of your application.
+
+## Examples
+
+There is a fully working example in the `examples` directory, but here's a brief condensed version:
+
+require 'corosync_commander'
+cc = CorosyncCommander.new
+cc.commands.register('shell command') do |sender, shellcmd|
+  %x{#{shellcmd}}
+end
+cc.join('remote commands')
+
+exe = cc.execute([], 'shell command', 'hostname')
+  enum = exe.to_enum
+  begin
+    enum.each do |sender, response|
+      $stdout.write "#{sender.nodeid}:#{sender.pid}: #{response.chomp}\n"
+    end
+  rescue CorosyncCommander::RemoteException => e
+    puts "Caught remote exception: #{e}"
+    retry
+  end
+end

data/Rakefile

@@ -17,10 +17,7 @@ end
 
 desc 'Bump version'
 task 'version' do
-	file = 'lib/version.rb'
-	const = 'CorosyncCommander::GEM_VERSION'
-
-	current_version = %x{ruby -e "require './#{file}'; puts #{const}"}.chomp
+	current_version = File.read('VERSION').chomp
 	current_version_commit = %x{git rev-parse --verify #{current_version} 2>/dev/null}.chomp
 	current_head_commit = %x{git rev-parse HEAD}.chomp
 	if current_version_commit != '' and current_version_commit != current_head_commit then
@@ -32,23 +29,17 @@ task 'version' do
 		print "Next version? (#{next_version}): "
 		response = STDIN.gets.chomp
 		if response != '' then
-			raise StandardError, "Not a valid version" unless response.match(/^[0-9\.]$/)
+			raise StandardError, "Not a valid version" unless response.match(/^[0-9\.]+$/)
 			next_version = response
 		end
 
-		const_name = const.sub(/^.+:/, '')
-		new_file_content = ''
-		File.open(file, 'r') do |file|
-			file.each_line do |line|
-				new_file_content += line.sub(/(#{const_name}\s*=\s*['"])#{current_version}(['"])/, "\\1#{next_version}\\2")
-			end
-		end
-		File.open(file, 'w') do |file|
-			file.write new_file_content
+		File.open('VERSION', 'w') do |file|
+			file.puts next_version
 		end
 		message = %x{git log #{current_version_commit}..HEAD --pretty=format:'* %s%n  %an (%ai) - @%h%n'}.gsub(/'/, "'\\\\''")
 
-		sh "git commit -m 'Version: #{next_version}\n\n#{message}' #{file}"
+		sh "git commit -m 'Version: #{next_version}\n\n#{message}' VERSION"
+		sh "git tag #{next_version}"
 
 		@spec = nil
 	end
@@ -62,7 +53,6 @@ end
 desc 'Publish gem file'
 task 'publish' do
 	gem_file = "#{spec.name}-#{spec.version}.gem"
-	sh "git tag #{spec.version}"
 	sh "git push"
 	sh "gem push #{gem_file}"
 end

data/VERSION

@@ -0,0 +1 @@
+0.1.0.0

data/corosync-commander.gemspec

@@ -1,6 +1,4 @@
-require File.expand_path('../lib/version.rb', __FILE__)
-
-Gem::Specification.new 'corosync-commander', CorosyncCommander::GEM_VERSION do |s|
+Gem::Specification.new 'corosync-commander', File.read('VERSION').chomp do |s|
 	s.description = 'Provides a simplified interface for issuing commands to nodes in a Corosync closed process group.'
 	s.summary = 'Sends/receives Corosync CPG commands'
 	s.homepage = 'http://github.com/phemmer/ruby-corosync-commander/'
@@ -9,5 +7,5 @@ Gem::Specification.new 'corosync-commander', CorosyncCommander::GEM_VERSION do |
 	s.license = 'MIT'
 	s.files = %x{git ls-files}.split("\n")
 
-	s.add_runtime_dependency 'corosync', '~> 0.0.3'
+	s.add_runtime_dependency 'corosync', '~> 0.1.0'
 end

data/examples/ccsh.rb

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+# This is a more heavyweight example of the remote_commands example.
+# It utilizes additional features such as quorum. It follows the same principles though, and you can launch it as many times as you want to play with it.
+
+if RUBY_ENGINE == 'ruby' and ENV['RUBY_THREAD_MACHINE_STACK_SIZE'].to_i < 1572864 then
+	ENV['RUBY_THREAD_MACHINE_STACK_SIZE'] = '1572864'
+	exec('ruby', $0, *ARGV)
+end
+
+require 'rubygems'
+require 'bundler/setup'
+
+$:.unshift(File.expand_path('../../lib', File.realpath(__FILE__)))
+
+require 'corosync_commander'
+require 'timeout'
+require 'readline'
+
+class CCSH
+	def initialize
+		@cc = CorosyncCommander.new
+		@cc.commands.register 'sh', &self.method(:cc_sh)
+		@cc.on_confchg &self.method(:cc_confchg)
+		@cc.on_quorumchg &self.method(:cc_quorumchg)
+
+		@cc.join('ccsh')
+	end
+
+	def cc_sh(sender, command)
+		output = nil
+		status = nil
+		begin
+			Timeout::timeout(10) do
+				output = %x{#{command}}
+				status = $?.exitstatus
+			end
+		rescue Timeout::Error => e
+			output = 'Command timed out'
+			status = 255
+		end
+
+		[status, output]
+	end
+
+	def cc_confchg(members, left, joined)
+		$stderr.puts "Group membership changed"
+		$stderr.puts "Members: #{members.map{|m| m.to_s}.join(' ')}" if members.size > 0
+		$stderr.puts "Lost: #{left.map{|m| m.to_s}.join(' ')}" if left.size > 0
+		$stderr.puts "Joined: #{joined.map{|m| m.to_s}.join(' ')}" if joined.size > 0
+		$stderr.puts ""
+	end
+
+	def cc_quorumchg(quorate, members)
+		msg = quorate ? "gained" : "lost"
+		$stderr.puts "Quorum #{msg}"
+		$stderr.puts ""
+	end
+
+	def run
+		while line = Readline.readline('> ', true) do
+			line.chomp!
+
+			if line.match(/^!\s*(.*)/) then
+				# internal command
+				command = $1
+				if command == 'leaders' then
+					leader_pool = @cc.instance_variable_get(:@leader_pool)
+					$stdout.write(leader_pool.map{|m| m.to_s}.join("\n") + "\n")
+				elsif command == 'whoami' then
+					$stdout.write(@cc.cpg.member.to_s + "\n")
+				elsif command == 'exit' then
+					exit
+				end
+			else
+				# remote command
+				exe = @cc.execute([], 'sh', line)
+				exe.to_enum.each do |sender, result|
+					status, output = result
+					output.split("\n").each do |line|
+						$stdout.write("#{sender}: #{line}\n")
+					end
+				end
+			end
+
+			$stdout.write("\n")
+		end
+	end
+end
+
+ccsh = CCSH.new
+ccsh.run

data/examples/remote_commands.rb

@@ -0,0 +1,38 @@
+# This example will send shell commands to all the running instances in the cluster.
+# It connects to the 'remote commands' CPG group, executes any commands received, and sends the result back to the app that sent the command.
+# The local app will read commands from STDIN and display the response from all the nodes.
+#
+# Launch this as many times as you want and play around with it.
+#
+
+
+if RUBY_ENGINE == 'ruby' and !ENV['RUBY_THREAD_MACHINE_STACK_SIZE'] then
+  ENV['RUBY_THREAD_MACHINE_STACK_SIZE'] = '1572864'
+  exec('ruby', $0, *ARGV)
+end
+
+$:.unshift(File.expand_path('../../lib', __FILE__))
+require 'corosync_commander'
+cc = CorosyncCommander.new
+cc.commands.register('shell command') do |sender, shellcmd|
+  %x{#{shellcmd}}
+end
+cc.join('remote commands')
+
+while line = STDIN.gets do
+  exe = cc.execute([], 'shell command', line.chomp)
+  enum = exe.to_enum
+  begin
+    enum.each do |sender, response|
+      $stdout.write "
+#{sender.nodeid}:#{sender.pid}:
+#{response.chomp}
+========================================
+
+"
+    end
+  rescue CorosyncCommander::RemoteException => e
+    puts "Caught remote exception: #{e}"
+    retry
+  end
+end

data/lib/corosync_commander/execution.rb

@@ -47,18 +47,21 @@ class CorosyncCommander::Execution
 	# Provides an enumerator that can be looped through.
 	# Will raise an exception if the remote node generated an exception. Can be verified by calling `is_a?(CorosyncCommander::RemoteException)`.
 	# Restarting the enumerator will not restart from the first response, it will continue on to the next.
+	# @param ignore [Exception, Boolean] List of exception classes to suppress. `true` for all
 	# @yieldparam response [Object] The response generated by the remote command.
 	# @yieldparam sender [Corosync::CPG::Member] The node which generated the response.
 	# @return [Enumerator]
-	def to_enum(ignore_exception = false)
+	def to_enum(*ignore)
+		ignore = [Exception] if ignore.size == 1 and ignore[0] == true
+
 		Enumerator.new do |block|
 			begin
 				while response = self.response do
-					block.yield response.content, response.sender
+					block.yield response.sender, response.content
 				end
 			rescue CorosyncCommander::RemoteException => e
-				raise e unless ignore_exception
-				retry
+				retry if ignore.find {|extype| e.is_a?(extype)}
+				raise e
 			end
 		end
 	end
@@ -104,6 +107,7 @@ class CorosyncCommander::Execution
 
 			raise RuntimeError, "Received unexpected response while waiting for echo" if message.type != 'echo'
 
+			# CorosyncCommander#cpg_message sets the content to the list of cpg members at the time the echo was received
 			@pending_members = @recipients.size == 0 ? message.content.dup : message.content & @recipients
 		end
 

data/lib/corosync_commander.rb

@@ -1,4 +1,5 @@
 require 'corosync/cpg'
+require 'corosync/quorum'
 require File.expand_path('../corosync_commander/execution', __FILE__)
 require File.expand_path('../corosync_commander/execution/message', __FILE__)
 require File.expand_path('../corosync_commander/callback_list', __FILE__)
@@ -16,7 +17,7 @@ require File.expand_path('../corosync_commander/callback_list', __FILE__)
 #
 # @example
 #   cc = CorosyncCommander.new
-#   cc.commands.register('shell command') do |shellcmd|
+#   cc.commands.register('shell command') do |sender, shellcmd|
 #     %x{#{shellcmd}}
 #   end
 #   cc.join('my group')
@@ -26,7 +27,7 @@ require File.expand_path('../corosync_commander/callback_list', __FILE__)
 #   enum = exe.to_enum
 #   hostnames = []
 #   begin
-#     enum.each do |response, node|
+#     enum.each do |sender, response|
 #       hostnames << response
 #     end
 #   rescue CorosyncCommander::RemoteException => e
@@ -64,6 +65,11 @@ class CorosyncCommander
 		@cpg.connect
 		@cpg.fd.close_on_exec = true
 
+		@quorum = Corosync::Quorum.new
+		@quorum.on_notify {|*args| quorum_notify(*args)}
+		@quorum.connect
+		@quorum.fd.close_on_exec = true
+
 		@cpg_members = nil
 
 		@leader_pool = []
@@ -78,16 +84,30 @@ class CorosyncCommander
 
 		@command_callbacks = CorosyncCommander::CallbackList.new
 
+		if RUBY_ENGINE == 'ruby' and (Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.0.0') or ENV['RUBY_THREAD_MACHINE_STACK_SIZE'].to_i < 1572864) then
+			abort "MRI Ruby must be >= 2.0 and RUBY_THREAD_MACHINE_STACK_SIZE must be > 1572864"
+		end
+
+		join(group_name) if group_name
+	end
+
+	# Starts watching for notifications
+	# @return [void]
+	def start
+		@quorum.start(true)
+
 		@dispatch_thread = Thread.new do
 			Thread.current.abort_on_exception = true
 			loop do
-				@cpg.dispatch
+				select_ready = select([@cpg.fd, @quorum.fd], [], [])
+				if select_ready[0].include?(@quorum.fd) then
+					@quorum.dispatch
+				end
+				if select_ready[0].include?(@cpg.fd) then
+					@cpg.dispatch
+				end
 			end
 		end
-
-		if group_name then
-			join(group_name)
-		end
 	end
 
 	# Joins the specified group.
@@ -95,17 +115,30 @@ class CorosyncCommander
 	# @param group_name [String] Name of group to join
 	# @return [void]
 	def join(group_name)
+		start unless @dispatch_thread
+
 		@cpg.join(group_name)
 	end
 
+	# Leave the active CPG group.
+	# Will not stop quorum notifications. If you wish to stop quorum as well you should use {#stop} instead.
+	# @return [void]
+	def leave
+		@cpg.leave
+	end
+
 	# Shuts down the dispatch thread and disconnects CPG
 	# @return [void]
 	def stop
-		@dispatch_thread.kill
+		@dispatch_thread.kill if !@dispatch_thread.nil?
 		@dispatch_thread = nil
-		@cpg.disconnect
+
+		@cpg.close if !@cpg.nil?
 		@cpg = nil
 		@cpg_members = nil
+
+		@quorum.finalize if !@quorum.nil?
+		@quorum = nil
 	end
 
 	def next_execution_id()
@@ -154,6 +187,19 @@ class CorosyncCommander
 				message_echo.content = @cpg_members
 				execution_queue << message_echo
 			end
+		elsif message.type == 'leader reset' then
+			# The sender is requesting we reset their leader position
+			# For remote node, act as if the node left.
+			# For the local node, act as if we just joined.
+			if sender != @cpg.member then
+				@leader_pool.sync_synchronize(:EX) do
+					@leader_pool.delete(sender)
+				end
+			else
+				@leader_pool.sync_synchronize(:EX) do
+					@leader_pool.replace(@cpg_members.to_a)
+				end
+			end
 		elsif message.type != 'command' and message.recipients.include?(@cpg.member) then
 			# It's a response to us
 			execution_queue = nil
@@ -183,6 +229,7 @@ class CorosyncCommander
 				message_reply = message.reply(reply_value)
 				@cpg.send(message_reply)
 			rescue => e
+				$stderr.puts "Exception: #{e} (#{e.class})\n#{e.backtrace.join("\n")}"
 				message_reply = message.reply([e.class, e.to_s, e.backtrace])
 				message_reply.type = 'exception'
 				@cpg.send(message_reply)
@@ -214,7 +261,7 @@ class CorosyncCommander
 		end
 
 		@execution_queues.sync_synchronize(:SH) do
-			@execution_queues.each do |queue|
+			@execution_queues.values.each do |queue|
 				messages.each do |message|
 					queue << message
 				end
@@ -222,10 +269,34 @@ class CorosyncCommander
 		end
 	end
 
+	# Callback to execute when the CPG configuration changes
+	# @yieldparam member_list [Array<Corosync::CPG::Member>] List of members in group after change
+	# @yieldparam left_list [Array<Corosync::CPG::Member>] List of members which left the group
+	# @yieldparam join_list [Array<Corosync::CPG::Member>] List of members which joined the group
 	def on_confchg(&block)
 		@confchg_callback = block
 	end
 
+	# @!visibility private
+	def quorum_notify(quorate, node_list)
+		return if @quorate == quorate # didn't change
+		@quorate = quorate
+
+		if quorate then
+			# we just became quorate
+			@cpg.send(CorosyncCommander::Execution::Message.new(:recipients => [], :type => 'leader reset')) # 'leader reset' simulates a leave and then a join of this node
+		end
+
+		@quorumchg_callback.call(quorate, node_list) if @quorumchg_callback
+	end
+
+	# Callback to execute when the quorum state changes
+	# @yieldparam quorate [Boolean] Whether cluster is quorate
+	# @yieldparam member_list [Array] List of node IDs in the cluster after change
+	def on_quorumchg(&block)
+		@quorumchg_callback = block
+	end
+
 	# @!attribute [r] commands
 	# @return [CorosyncCommander::CallbackList] List of command callbacks
 	def commands
@@ -277,6 +348,12 @@ class CorosyncCommander
 	# This is slightly different than just calling `leader_position == 0` in that if it is -1 (meaning we havent received the CPG confchg callback yet), we wait for the CPG join to complete.
 	# @return [Boolean]
 	def leader?
+
+		# The way leadership works is that we record the members that were present when we joined the group in @leader_pool. Each time a node leaves the group, we remove them from @leader_pool. Once we become the only member in @leader_pool, we are the leader.
+		# Now in the event that the cluster splits, this becomes complicated. Each side will see the members of the other side leaving. So each side will end up with their own leader. Since they can't talk to eachother, having a leader in each group is perfectly fine. However when the 2 sides re-join, each side will see the members of the other side joining as new nodes, and both leaders will remain as leaders.
+		# We solve this by using the quorum status. When we go from inquorate to quorate, we give up our position. We send a 'leader reset' command to the cluster which tells everyone to remove us from their @leader_pool. When we receive the message ourself, we set @leader_pool to the group members at that moment.
+		# It doesn't matter if multiple members end up doing a 'leader reset' at the same time. It basically simulates the node leaving and then joining. Whoever performs the action first will move to the front. It will capture @leader_pool as the current members when it receives it's own message, and as the other resets come in, it will remove those members. Leaving itself in front of the ones that just joined (and reset after). But it will still remain after all the members that didn't do a reset.
+
 		position = nil
 		loop do
 			position = leader_position
@@ -287,4 +364,16 @@ class CorosyncCommander
 		end
 		position == 0
 	end
+
+	# Indicates whether cluster is quorate.
+	# @return [Boolean]
+	def quorate?
+		@quorate
+	end
+
+	# List of current members
+	# @return [Array<Corosync::CPG::Member>] List of members currently in the group
+	def members
+		@cpg_members
+	end
 end

data/spec/corosync_commander.rb

@@ -44,7 +44,7 @@ describe CorosyncCommander do
 
 	it 'calls the callback' do
 		exe = @cc.execute([], 'summation', 123, 456)
-		results = exe.to_enum.collect do |response,node|
+		results = exe.to_enum.collect do |node,response|
 			response
 		end
 		
@@ -101,6 +101,31 @@ describe CorosyncCommander do
 		end
 	end
 
+	it 'handles pending responses when a node leaves' do
+		complete = false
+		Timeout.timeout(5) do
+			forkpid = fork do
+				cc = CorosyncCommander.new(@cc.cpg.group)
+				cc.commands.register('exit') do |sender|
+					sleep 2
+					exit!
+				end
+				cc.dispatch_thread.join
+				exit 1
+			end
+
+			sleep 1 # wait for the fork to connect
+
+			@cc.commands.register('exit') {} # ignore it
+			exe = @cc.execute([], 'exit')
+			exe.wait
+			complete = true
+			Process.kill('TERM', forkpid)
+		end
+
+		expect(complete).to eq(true)
+	end
+
 	it 'captures remote exceptions' do
 		Timeout.timeout(5) do
 			@cc.commands.register('make exception') do
@@ -141,7 +166,7 @@ describe CorosyncCommander do
 			responses = []
 			exceptions = 0
 			begin
-				enum.each do |response,node|
+				enum.each do |node,response|
 					responses << response
 				end
 			rescue CorosyncCommander::RemoteException

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: corosync-commander
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0.0
 platform: ruby
 authors:
 - Patrick Hemmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-10 00:00:00.000000000 Z
+date: 2014-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: corosync
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: 0.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.3
+        version: 0.1.0
 description: Provides a simplified interface for issuing commands to nodes in a Corosync
   closed process group.
 email: patrick.hemmer@gmail.com
@@ -35,13 +35,16 @@ files:
 - Gemfile
 - Gemfile.lock
 - LICENSE
+- README.md
 - Rakefile
+- VERSION
 - corosync-commander.gemspec
+- examples/ccsh.rb
+- examples/remote_commands.rb
 - lib/corosync_commander.rb
 - lib/corosync_commander/callback_list.rb
 - lib/corosync_commander/execution.rb
 - lib/corosync_commander/execution/message.rb
-- lib/version.rb
 - spec/corosync_commander.rb
 - spec/spec_helper.rb
 homepage: http://github.com/phemmer/ruby-corosync-commander/

data/lib/version.rb

@@ -1,3 +0,0 @@
-class CorosyncCommander
-	GEM_VERSION = '0.0.2'
-end