corosync-commander 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 90dec889b8284d9eacfcf3ad9fc2e30156790428
+  data.tar.gz: 8beb44d2adfb7f708b48e0f986f8e1e8c68300c5
+SHA512:
+  metadata.gz: a65aa251126d8f329c7f9d125b9c4d7508270a5fedcf49e2b147612073ceadaddae4afc13683a4355ef6dd0a86830861f346a70c5f5df78f78c3505cec9a5671
+  data.tar.gz: 5ff4ac01de166776a97c30fa7a8d7f5338383bb927e89c2ea68964b06609a9969f1942f01ec044f2597473a67a0195154df7e618d783ffca5be34e70acaf4595

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/Gemfile

@@ -0,0 +1,11 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+gem 'corosync', '~>0.0.3'
+
+group :development do
+	gem 'rake'
+	gem 'yard'
+	gem 'rdoc'
+	gem 'rspec'
+end

data/Gemfile.lock

@@ -0,0 +1,30 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    corosync (0.0.3)
+      ffi (~> 1.9)
+    diff-lcs (1.2.4)
+    ffi (1.9.3)
+    json (1.8.1)
+    rake (10.1.0)
+    rdoc (4.0.1)
+      json (~> 1.4)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.6)
+    rspec-expectations (2.14.3)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    yard (0.8.7.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  corosync (~> 0.0.3)
+  rake
+  rdoc
+  rspec
+  yard

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Patrick Hemmer <patrick.hemmer@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,7 @@
+########################################
+desc 'Run tests'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:test) do |t|
+	t.pattern = 'spec/**/*.rb'
+	t.rspec_opts = '-c -f d --fail-fast'
+end

data/corosync-commander.gemspec

@@ -0,0 +1,13 @@
+require File.expand_path('../lib/version.rb', __FILE__)
+
+Gem::Specification.new 'corosync-commander', CorosyncCommander::GEM_VERSION do |s|
+	s.description = 'Provides a simplified interface for issuing commands to nodes in a Corosync closed process group.'
+	s.summary = 'Sends/receives Corosync CPG commands'
+	s.homepage = 'http://github.com/phemmer/ruby-corosync-commander/'
+	s.author = 'Patrick Hemmer'
+	s.email = 'patrick.hemmer@gmail.com'
+	s.license = 'MIT'
+	s.files = %x{git ls-files}.split("\n")
+
+	s.add_runtime_dependency 'corosync', '~> 0.0.3'
+end

data/lib/corosync_commander/callback_list.rb

@@ -0,0 +1,65 @@
+class CorosyncCommander::CallbackList
+	include Enumerable
+
+	def initialize
+		@callbacks = {}
+		@callbacks.extend(Sync_m)
+	end
+
+  # Iterate through each command/callback
+  # @yieldparam command [String] Name of command
+  # @yieldparam callback [Proc] Proc to be executed upon receiving command
+	def each(&block)
+		callbacks = nil
+		@callbacks.synchronize(:SH) do
+			callbacks = @callbacks.dup
+		end
+		callbacks.each(&block)
+	end
+
+  # Assign a callback
+  # @example
+  #   cc.commands['my command'] = Proc.new do
+  #     puts "Hello world!"
+  #   end
+  # @param command [String] Name of command
+  # @param block [Proc] Proc to call when command is executed
+  # @return [Proc]
+	def []=(command, block)
+		@callbacks.synchronize(:EX) do
+			@callbacks[command] = block
+		end
+		block
+	end
+
+  # Assign a callback
+  # This is another method of assigning a callback
+  # @example
+  #   cc.commands.register('my command') do
+  #     puts "Hellow world!"
+  #   end
+  # @param command [String] Name of command
+  # @param block [Proc] Proc to call when command is executed
+  # @return [Proc]
+	def register(command, &block)
+		self[command] = block
+	end
+
+  # Retrieve a registered command callback
+  # @param command [String] Name of command
+  # @return [Proc]
+	def [](command)
+		@callbacks.synchronize(:SH) do
+			@callbacks[command]
+		end
+	end
+
+  # Delete a command callback
+  # @param command [String] Name of command
+  # @return [Proc] The deleted command callback
+	def delete(command)
+		@callbacks.synchronize(:EX) do
+			@callbacks.delete(command)
+		end
+	end
+end

data/lib/corosync_commander/execution/message.rb

@@ -0,0 +1,45 @@
+class CorosyncCommander
+	class Execution
+	end
+end
+class CorosyncCommander::Execution::Message
+	attr_reader :sender
+	attr_reader :recipients
+	attr_reader :execution_id
+	attr_accessor :type
+	attr_accessor :content
+
+	def self.from_cpg_message(data, sender)
+		data = JSON.parse(data)
+
+		recipients = Corosync::CPG::MemberList.new
+		data[0].each do |m|
+			nodeid,pid = m.split(':').map{|i| i.to_i}
+			recipients << Corosync::CPG::Member.new(nodeid,pid)
+		end
+
+		execution_id = data[1]
+
+		type = data[2]
+
+		content = data[3]
+
+		self.new(:sender => sender, :recipients => recipients, :execution_id => execution_id, :type => type, :content => content)
+	end
+
+	def initialize(params = {})
+		@sender = params[:sender]
+		@recipients = Corosync::CPG::MemberList.new(params[:recipients])
+		@execution_id = params[:execution_id]
+		@type = params[:type]
+		@content = params[:content]
+	end
+
+	def reply(content)
+		self.class.new(:recipients => [@sender], :execution_id => @execution_id, :type => 'response', :content => content)
+	end
+
+	def to_s
+		[@recipients.to_a, @execution_id, @type, @content].to_json
+	end
+end

data/lib/corosync_commander/execution.rb

@@ -0,0 +1,124 @@
+class CorosyncCommander
+end
+class CorosyncCommander::Execution
+	attr_reader :queue
+	attr_reader :id
+	attr_reader :recipients
+	attr_reader :command
+	attr_reader :args
+	attr_reader :pending_members
+
+	def initialize(cc, id, recipients, command, args)
+		@cc = cc
+		@id = id
+		@recipients = Corosync::CPG::MemberList.new(recipients)
+		@command = command
+		@args = args
+
+		@queue = Queue.new
+
+		@pending_members = nil
+
+		@responses = []
+	end
+
+	# Gets the next response, blocking if none has been returned yet.
+	# This will also raise an exception if the remote process raised an exception. This can be tested by calling `exception.is_a?(CorosyncCommander::RemoteException)`
+	# @return [CorosyncCommander::Execution::Message] The response from the remote host. Returns `nil` when there are no more responses.
+	def response
+		response = get_response
+
+		return nil if response.nil?
+
+		if response.type == 'exception'
+			e_class = Kernel.const_get(response.content[0])
+			e_class = StandardError if e_class.nil? or !(e_class <= Exception) # The remote node might have types we don't have. So if we don't have them use StandardError
+			e = e_class.new(response.content[1] + " (CorosyncCommander::RemoteException@#{response.sender})")
+			e.set_backtrace(response.content[2])
+			e.extend(CorosyncCommander::RemoteException)
+			e.sender = response.sender
+			raise e
+		end
+
+		response
+	end
+	alias_method :next, :response
+
+	# Provides an enumerator that can be looped through.
+	# Will raise an exception if the remote node generated an exception. Can be verified by calling `is_a?(CorosyncCommander::RemoteException)`.
+	# Restarting the enumerator will not restart from the first response, it will continue on to the next.
+	# @yieldparam response [Object] The response generated by the remote command.
+	# @yieldparam sender [Corosync::CPG::Member] The node which generated the response.
+	# @return [Enumerator]
+	def to_enum(ignore_exception = false)
+		Enumerator.new do |block|
+			begin
+				while response = self.response do
+					block.yield response.content, response.sender
+				end
+			rescue CorosyncCommander::RemoteException => e
+				raise e unless ignore_exception
+				retry
+			end
+		end
+	end
+
+	# Wait for all responses to come in, but discard them.
+	# Useful to block waiting for the remote commands to finish when you dont care about the result.
+	# @param ignore_exception [Boolean] Whether to ignore remote exceptions, or raise them. If `true`, remote exceptions will not raise an exception here.
+	# @return [Boolean] Returns `true` if no exceptions were raised, `false` otherwise.
+	def wait(ignore_exception = false)
+		success = true
+		begin
+			while response do end
+		rescue CorosyncCommander::RemoteException => e
+			success = false
+			retry if ignore_exception
+			raise e
+		end
+		success
+	end
+
+	# This is just so that we can remove the queue from execution_queues and avoid running unnecessary code on receipt of message/confchg
+	def discard
+		@cc.execution_queues.sync_synchronize(:EX) do
+			@cc.execution_queues.delete(@id)
+		end
+		@queue.clear
+		@queue = []
+	end
+
+	# Gets the next response message from the queue.
+	# This is used internally and is probably not what you want. See {#response}
+	# @return [CorosyncCommander::Execution::Message]
+	def get_response
+		return if !@queue.is_a?(Queue) # we've called `clear`
+
+		while @pending_members.nil?
+			message = @queue.shift
+
+			next if message.type == 'leave' # we havent received the echo, so we dont care yet
+
+			raise RuntimeError, "Received unexpected response while waiting for echo" if message.type != 'echo'
+
+			@pending_members = @recipients.size == 0 ? message.content.dup : message.content & @recipients
+		end
+
+		return if @pending_members.size == 0
+
+		message = @queue.shift
+
+		@pending_members.delete message.sender
+		if @pending_members.size == 0 then
+			self.discard
+		end
+
+		return if message.type == 'leave' # we already did @pending_members.delete above
+
+		message
+	end
+end
+
+module CorosyncCommander::RemoteException
+	attr_accessor :sender
+end

data/lib/corosync_commander.rb

@@ -0,0 +1,233 @@
+require 'corosync/cpg'
+require File.expand_path('../corosync_commander/execution', __FILE__)
+require File.expand_path('../corosync_commander/execution/message', __FILE__)
+require File.expand_path('../corosync_commander/callback_list', __FILE__)
+
+# This provides a simplified interface into Corosync::CPG.
+# The main use case is for sending commands to a remote server, and waiting for the responses.
+# 
+# This library takes care of:
+# * Ensuring a consistent message format.
+# * Sending messages to all, or just specific nodes.
+# * Invoking the appropriate callback (and passing parameters) based on the command sent.
+# * Resonding with the return value of the callback.
+# * Handling exceptions and sending them back to the sender.
+# * Knowing exactly how many responses should be coming back.
+#
+# @example
+#   cc = CorosyncCommander.new
+#   cc.commands.register('shell command') do |shellcmd|
+#     %x{#{shellcmd}}
+#   end
+#   cc.join('my group')
+#
+#   exe = cc.execute([], 'shell command', 'hostname')
+#
+#   enum = exe.to_enum
+#   hostnames = []
+#   begin
+#     enum.each do |response, node|
+#       hostname << response
+#     end
+#   rescue CorosyncCommander::RemoteException => e
+#     puts "Caught remote exception: #{e}"
+#     retry
+#   end
+#
+#   puts "Hostnames: #{hostnames.join(' ')}"
+#
+#
+# == IMPORTANT: Will not work without tuning ruby.
+# You cannot use this with MRI Ruby older than 2.0. Even with 2.0 you must tune ruby. This is because Corosync CPG (as of 1.4.3) allocates a 1mb buffer on the stack. Ruby 2.0 only allocates a 512kb stack for threads. This gem uses a thread for handling incoming messages. Thus if you try to use older ruby you will get segfaults.
+#
+# Ruby 2.0 allows increasing the thread stack size. You can do this with the RUBY_THREAD_MACHINE_STACK_SIZE environment variable. The advised value to set is 1.5mb.
+#   RUBY_THREAD_MACHINE_STACK_SIZE=1572864 ruby yourscript.rb
+class CorosyncCommander
+	require 'thread'
+	require 'sync'
+	require 'json'
+
+	attr_reader :cpg
+
+	attr_reader :execution_queues
+
+	# Creates a new instance and connects to CPG.
+	# If a group name is provided, it will join that group. Otherwise it will only connect. This is so that you can establish the command callbacks and avoid NotImplementedError exceptions
+	# @param group_name [String] Name of the group to join
+	def initialize(group_name = nil)
+		@cpg = Corosync::CPG.new
+		@cpg.on_message {|*args| cpg_message(*args)}
+		@cpg.on_confchg {|*args| cpg_confchg(*args)}
+		@cpg.connect
+		@cpg.fd.close_on_exec = true
+
+		@cpg_members = nil
+
+		# we can either share the msgid counter across all threads, or have a msgid counter on each thread and send the thread ID with each message. I prefer the former
+		@next_execution_id = 0
+		@next_execution_id_mutex = Mutex.new
+
+		@execution_queues = {}
+		@execution_queues.extend(Sync_m)
+
+		@command_callbacks = CorosyncCommander::CallbackList.new
+
+		@dispatch_thread = Thread.new do
+			Thread.current.abort_on_exception = true
+			loop do
+				@cpg.dispatch
+			end
+		end
+
+		if group_name then
+			join(group_name)
+		end
+	end
+
+	# Joins the specified group.
+	# This is provided separate from initialization so that callbacks can be registered before joining the group so that you wont get NotImplementedError exceptions
+	# @param group_name [String] Name of group to join
+	# @return [void]
+	def join(group_name)
+		@cpg.join(group_name)
+	end
+
+	# Shuts down the dispatch thread and disconnects CPG
+	# @return [void]
+	def stop
+		@dispatch_thread.kill
+		@dispatch_thread = nil
+		@cpg.disconnect
+		@cpg = nil
+		@cpg_members = nil
+	end
+
+	def next_execution_id()
+		id = nil
+		@next_execution_id_mutex.synchronize do
+			id = @next_execution_id += 1
+		end
+		id
+	end
+	private :next_execution_id
+
+	# Used as a callback on receipt of a CPG message
+	# @param message [String] data structure passed to @cpg.send
+	#   * message[0] == [Array<String>] Each string is "nodeid:pid" of the intended message recipients
+	#   * msgid == [Integer]
+	#     * In the event of a new message, this, combined with `member` will uniquely identify this message
+	#     * In the event of a reply, this is the message ID sent in the original message
+	#   * type == [String] command/response/exception
+	#   * args == [Array]
+	#     * In the event of a command, this will be the arguments passed to CorosyncCommander.send
+	#     * In the event of a response, this will be the return value of the command handler
+	#     * In the event of an exception, this will be the exception string and backtrace
+	# @param sender [Corosync::CPG::Member] Sender of the message
+	# @!visibility private
+	def cpg_message(message, sender)
+		message = CorosyncCommander::Execution::Message.from_cpg_message(message, sender)
+
+		if sender == @cpg.member || message.recipients.include?(@cpg.member)
+			execution_queue = nil
+			@execution_queues.sync_synchronize(:SH) do
+				execution_queue = @execution_queues[message.execution_id]
+			end
+			if !execution_queue.nil? then
+				# someone is listening
+				if sender == @cpg.member and message.type == 'command' then
+					# the Execution object needs a list of the members at the time it's message was received
+					message_echo = message.dup
+					message_echo.type = 'echo'
+					message_echo.content = @cpg_members
+					execution_queue << message_echo
+				else
+					execution_queue << message
+				end
+			end
+		end
+
+		if message.recipients.size > 0 and !message.recipients.include?(@cpg.member) then
+			return
+		end
+
+		if message.type == 'command' then
+			# we received a command from another node
+			begin
+				# see if we've got a registered callback
+				command_callback = nil
+
+				command_name = message.content[0]
+				command_callback = @command_callbacks[command_name]
+				if command_callback.nil? then
+					raise NotImplementedError, "No callback registered for command '#{command_name}'"
+				end
+
+				command_args = message.content[1]
+				reply_value = command_callback.call(*command_args)
+				message_reply = message.reply(reply_value)
+				@cpg.send(message_reply)
+			rescue => e
+				message_reply = message.reply([e.class, e.to_s, e.backtrace])
+				message_reply.type = 'exception'
+				@cpg.send(message_reply)
+			end
+		end
+	end
+
+	# @!visibility private
+	def cpg_confchg(member_list, left_list, join_list)
+		@cpg_members = member_list
+
+		# we look for any members leaving the cluster, and if so we notify all threads that are waiting for a response that they may have just lost a node
+		return if left_list.size == 0
+
+		messages = left_list.map do |member|
+			CorosyncCommander::Execution::Message.new(:sender => member, :type => 'leave')
+		end
+
+		@execution_queues.sync_synchronize(:SH) do
+			@execution_queues.each do |queue|
+				messages.each do |message|
+					queue << message
+				end
+			end
+		end
+	end
+
+	# @!attribute [r] commands
+	# @return [CorosyncCommander::CallbackList] List of command callbacks
+	def commands
+		@command_callbacks
+	end
+
+	# Execute a remote command.
+	# @param recipients [Array<Corosync::CPG::Member>] List of recipients to send to, or an empty array to broadcast to all members of the group.
+	# @param command [String] The name of the remote command to execute. If no such command exists on the remote node a NotImplementedError exception will be raised when enumerating the results.
+	# @param args Any further arguments will be passed to the command callback on the remote host.
+	# @return [CorosyncCommander::Execution]
+	def execute(recipients, command, *args)
+		execution = CorosyncCommander::Execution.new(self, next_execution_id, recipients, command, args)
+
+		message = CorosyncCommander::Execution::Message.new(:recipients => recipients, :execution_id => execution.id, :type => 'command', :content => [command, args])
+
+		@execution_queues.synchronize(:EX) do
+			@execution_queues[execution.id] = execution.queue
+		end
+		# Technique stolen from http://www.mikeperham.com/2010/02/24/the-trouble-with-ruby-finalizers/
+		#TODO We definitately need a spec test to validate the execution object gets garbage collected
+		ObjectSpace.define_finalizer(execution, execution_queue_finalizer(execution.id))
+
+		@cpg.send(message)
+
+		execution
+	end
+	# This is so that we remove our queue from the execution queue list when we get garbage collected.
+	def execution_queue_finalizer(execution_id)
+		proc do
+			@execution_queues.synchronize(:EX) do
+				@execution_queues.delete(execution_id)
+			end
+		end
+	end
+	private :execution_queue_finalizer
+end

data/lib/version.rb

@@ -0,0 +1,3 @@
+class CorosyncCommander
+	GEM_VERSION = '0.0.1'
+end

data/spec/corosync_commander.rb

@@ -0,0 +1,181 @@
+require 'spec_helper'
+require 'timeout'
+
+describe CorosyncCommander do
+	def fork_execute(wait, cc_recip, command, *args)
+		recip = cc_recip.cpg.member
+		pid = fork do
+			cc = CorosyncCommander.new(cc_recip.cpg.group)
+			exe = cc.execute([recip], command, *args)
+			exe.wait
+			exit!(0)
+		end
+		if wait then
+			status = Process.wait2(pid)
+			status.exitstatus
+		else
+			pid
+		end
+	end
+
+	before(:all) do
+		Timeout.timeout(1) do
+			@cc = CorosyncCommander.new("CorosyncCommander RSPEC #{Random.rand(2 ** 32)}")
+		end
+	end
+
+	it 'can call cpg_dispatch' do
+		# this is just so that if the thread test fails, we verify that it's not `dispatch` that's the issue
+		@cc.cpg.dispatch(0)
+	end
+
+	it 'can call cpg_dispatch on a thread' do
+		Timeout.timeout(2) do
+			pid = fork do
+				t = Thread.new do
+					@cc.cpg.dispatch(0)
+				end
+				t.join
+				exit!(0)
+			end
+			status = Process.wait2(pid)
+			expect(status[1]).to eq(0)
+		end
+	end
+
+	it 'registers a callback (block style)' do
+		@cc.commands.register 'summation' do |arg1,arg2|
+			arg1 + arg2
+		end
+
+		expect(@cc.commands['summation']).to be_a(Proc)
+	end
+
+	it 'registers a callback (assignment style)' do
+		@cc.commands['summation'] = Proc.new do |arg1,arg2|
+			arg1 + arg2
+		end
+		expect(@cc.commands['summation']).to be_a(Proc)
+	end
+
+	it 'calls the callback' do
+		exe = @cc.execute([], 'summation', 123, 456)
+		results = exe.to_enum.collect do |response,node|
+			response
+		end
+		
+		expect(results.size).to eq(1)
+		expect(results.first).to eq(123 + 456)
+	end
+
+=begin doesn't work for some reason. will investigate later as it's not critical
+	it 'removes queues on garbage collection' do
+		GC.start
+		GC.disable
+
+		@cc.commands.register('nothing') do end
+
+		queue_size_before = @cc.execution_queues.size
+
+		@cc.execute([], 'nothing')
+		queue_size_during = @cc.execution_queues.size
+
+		GC.enable
+		GC.start
+		queue_size_after = @cc.execution_queues.size
+
+		expect(queue_size_during).to eq(queue_size_before + 1)
+		expect(queue_size_after).to eq(queue_size_before)
+	end
+=end
+
+	it 'works with multiple processes' do
+		Timeout.timeout(1) do
+			sum = 0
+			num1 = Random.rand(2 ** 32)
+			num2 = Random.rand(2 ** 32)
+
+			@cc.commands.register('summation2') do |number|
+				sum += number
+			end
+
+			recipient = @cc.cpg.member
+			forkpid = fork do
+				cc = CorosyncCommander.new(@cc.cpg.group)
+				exe = cc.execute([recipient], 'summation2', num1)
+				exe.wait
+			end
+
+			exe = @cc.execute([recipient], 'summation2', num2)
+			exe.wait
+
+			result = Process.wait2(forkpid)
+
+			expect(result[1].exitstatus).to eq(0)
+
+			expect(sum).to eq(num1 + num2)
+		end
+	end
+
+	it 'captures remote exceptions' do
+		Timeout.timeout(5) do
+			@cc.commands.register('make exception') do
+				0/0
+			end
+			exe = @cc.execute(@cc.cpg.member, 'make exception')
+			expect{exe.wait}.to raise_error(ZeroDivisionError)
+		end
+	end
+
+	it 'resumes after exception' do
+		Timeout.timeout(5) do
+			@cc.commands.register('resumes after exception') do
+				'OK'
+			end
+
+			forkpid1 = fork do
+				cc = CorosyncCommander.new(@cc.cpg.group)
+				cc.commands.register('resumes after exception') do
+					0/0
+				end
+				sleep 5
+			end
+
+			forkpid2 = fork do
+				cc = CorosyncCommander.new(@cc.cpg.group)
+				cc.commands.register('resumes after exception') do
+					sleep 0.5 # make sure this response is not before the exception one
+					'OK'
+				end
+				sleep 5
+			end
+
+			sleep 1 # we have to wait for the forks to connect to the group
+
+			exe = @cc.execute([], 'resumes after exception')
+			enum = exe.to_enum
+			responses = []
+			exceptions = 0
+			begin
+				enum.each do |response,node|
+					responses << response
+				end
+			rescue CorosyncCommander::RemoteException
+				exceptions += 1
+				retry
+			end
+
+			[forkpid1,forkpid2].each do |forkpid|
+				begin
+					Process.kill('TERM', forkpid)
+				rescue Errno::ESRCH => e
+				end
+			end
+
+
+			expect(responses.size).to eq(2)
+			expect(exceptions).to eq(1)
+			expect(responses.find_all{|r| r == 'OK'}.size).to eq(2)
+		end
+	end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require File.expand_path('../../lib/corosync_commander', __FILE__)

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification
+name: corosync-commander
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Patrick Hemmer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: corosync
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+description: Provides a simplified interface for issuing commands to nodes in a Corosync
+  closed process group.
+email: patrick.hemmer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- Rakefile
+- corosync-commander.gemspec
+- lib/corosync_commander.rb
+- lib/corosync_commander/callback_list.rb
+- lib/corosync_commander/execution.rb
+- lib/corosync_commander/execution/message.rb
+- lib/version.rb
+- spec/corosync_commander.rb
+- spec/spec_helper.rb
+homepage: http://github.com/phemmer/ruby-corosync-commander/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Sends/receives Corosync CPG commands
+test_files: []
+has_rdoc: