wakame 0.4.1 → 0.4.2

data/lib/wakame/service.rb

@@ -97,7 +97,7 @@ module Wakame
       include ThreadImmutable
 
       attr_reader :dg, :instance_id, :status_changed_at, :rule_engine, :master
-      attr_reader :status
+      attr_reader :status, :lock_queue
 
       STATUS_OFFLINE = 0
       STATUS_ONLINE = 1
@@ -108,6 +108,7 @@ module Wakame
         @master = master
         @instance_id =Wakame.gen_id
         @rule_engine ||= RuleEngine.new(self)
+        @lock_queue = LockQueue.new(self)
         prepare
         
         instance_eval(&blk) if blk
@@ -289,7 +290,7 @@ module Wakame
           r[:instances][k]=i.dump_status
         }
         properties.each { |k, i|
-          r[:properties][k] = i.dump_status
+          r[:properties][k] = i.dump_attrs
           r[:properties][k][:instances] = each_instance(i.class).collect{|k, v| k }
         }
 
@@ -351,8 +352,109 @@ module Wakame
         
       end
       thread_immutable_methods :update_cluster_status
+      
+    end
 
+    class LockQueue
+      def initialize(cluster)
+        @service_cluster = cluster
+        @locks = {}
+        @id2res = {}
+
+        @queue_by_thread = {}
+        @qbt_m = ::Mutex.new
+      end
       
+      def set(resource, id)
+        # Ths Job ID already holds/reserves the lock regarding the resource.
+        return if @id2res.has_key?(id) && @id2res[id].has_key?(resource.to_s)
+
+        EM.barrier {
+          @locks[resource.to_s] ||= []
+          @id2res[id] ||= {}
+        
+          @id2res[id][resource.to_s]=1
+          @locks[resource.to_s] << id
+        }
+        Wakame.log.debug("#{self.class}: set(#{resource.to_s}, #{id})" + "\n#{self.inspect}")
+      end
+
+      def reset()
+        @locks.keys { |k|
+          @locks[k].clear
+        }
+        @id2res.clear
+      end
+
+      def test(id)
+        reslist = @id2res[id]
+        return :pass if reslist.nil? || reslist.empty?
+
+        # 
+        if reslist.keys.all? { |r| id == @locks[r.to_s][0] }
+          return :runnable
+        else
+          return :wait
+        end
+      end
+
+      def wait(id, tout=60*30)
+        @qbt_m.synchronize { @queue_by_thread[Thread.current] = ::Queue.new }
+
+        timeout(tout) {
+          while test(id) == :wait
+            Wakame.log.debug("#{self.class}: Job #{id} waits for locked resouces: #{@id2res[id].keys.join(', ')}")
+            break if id == @queue_by_thread[Thread.current].deq
+          end
+        }
+      ensure
+        @qbt_m.synchronize { @queue_by_thread.delete(Thread.current) }
+      end
+      
+      def quit(id)
+        EM.barrier {
+          case test(id)
+          when :runnable, :wait
+            @id2res[id].keys.each { |r| @locks[r.to_s].delete_if{ |i| i == id } }
+            @qbt_m.synchronize {
+              @queue_by_thread.each {|t, q| q.enq(id) }
+            }
+          end
+          
+          @id2res.delete(id)
+        }
+        Wakame.log.debug("#{self.class}: quit(#{id})" + "\n#{self.inspect}")
+      end
+
+      def clear_resource(resource)
+      end
+
+      def inspect
+        output = @locks.collect { |k, lst|
+          [k, lst].flatten
+        }
+        return "" if output.empty?
+
+        # Table display
+        maxcolws = (0..(output.size)).zip(*output).collect { |i| i.shift; i.map!{|i| (i.nil? ? "" : i).length }.max }
+        maxcol = maxcolws.size
+        maxcolws.reverse.each { |i| 
+          break if i > 0
+          maxcol -= 1
+        }
+
+        textrows = output.collect { |x|
+          buf=""
+          maxcol.times { |n|
+            buf << "|" + (x[n] || "").ljust(maxcolws[n])
+          }
+          buf << "|"
+        }
+
+        "+" + (["-"] * (textrows[0].length - 2)).join('') + "+\n" + \
+        textrows.join("\n") + \
+        "\n+" + (["-"] * (textrows[0].length - 2)).join('')+ "+"
+      end
     end
     
     
@@ -736,179 +838,8 @@ end
 
 module Wakame
   module Service
-
-
     module ApacheBasicProps
       attr_accessor :listen_port, :listen_port_https, :server_root
-      
-    end
-
-    class MySQL_Slave < Property
-      attr_reader :basedir, :mysqld_datadir, :mysqld_port, :mysqld_server_id, :mysqld_log_bin, :ebs_volume, :ebs_device
-
-      def initialize
-        super()
-        @template = ConfigurationTemplate::MySQLSlaveTemplate.new()
-        @basedir = '/home/wakame/mysql'
-
-        @mysqld_server_id = 2 # dynamic
-        @mysqld_port = 3307
-        @mysqld_datadir = File.expand_path('data-slave', @basedir)
-
-        @ebs_volume = 'vol-38bc5f51'  # master volume_id
-        @ebs_device = '/dev/sdn'      # slave mount point
-        @ebs_mount_option = 'noatime'
-
-        @mysqld_master_host = '10.249.2.115'
-        @mysqld_master_user = 'wakame-repl'
-        @mysqld_master_pass = 'wakame-slave'
-        @mysqld_master_port = 3306
-        @mysqld_master_datadir = File.expand_path('data', @basedir)
-
-        @duplicable = false
-      end
-
-      def before_start(svc, action)
-        vm_manipulator = VmManipulator.create
-
-        Wakame.log.debug("mkdir #{@mysqld_datadir}")
-        system("[ -d #{@mysqld_datadir} ] || mkdir -p #{@mysqld_datadir}")
-        Wakame.log.debug("[ -b #{@ebs_device} ]")
-        system("[ -b #{@ebs_device} ]")
-        if $? == 0
-          Wakame.log.debug("The EBS volume(slave) device is not ready to attach: #{@ebs_device}")
-          return
-        end
-
-        volume_map = vm_manipulator.describe_volume(@ebs_volume)
-        Wakame.log.debug("describe_volume(#{@ebs_volume}): #{volume_map.inspect}")
-        if volume_map['status'] == 'in-use'
-          # Nothin to be done
-        else
-          Wakame.log.debug("The EBS volume(slave) is not ready to attach: #{@ebs_volume}")
-          return
-        end
-
-        system("echo show master status | /usr/bin/mysql -h#{@mysqld_master_host} -P#{@mysqld_master_port} -u#{@mysqld_master_user}  -p#{@mysqld_master_pass}")
-        if $? != 0
-          raise "Can't connect mysql master: #{@mysqld_master_host}:#{@mysqld_master_port}"
-        end
-
-        system("echo 'FLUSH TABLES WITH READ LOCK;' | /usr/bin/mysql -h#{@mysqld_master_host} -P#{@mysqld_master_port} -u#{@mysqld_master_user}  -p#{@mysqld_master_pass} -s")
-        master_status = `echo show master status | /usr/bin/mysql -h#{@mysqld_master_host} -P#{@mysqld_master_port} -u#{@mysqld_master_user}  -p#{@mysqld_master_pass} -s`.to_s.split(/\t/)[0..1]
-#        p master_status
-
-        # mysql/data/master.info
-        master_infos = []
-        master_infos << 14
-        master_infos << master_status[0]
-        master_infos << master_status[1]
-        master_infos << @mysqld_master_host
-        master_infos << @mysqld_master_user
-        master_infos << @mysqld_master_pass
-        master_infos << @mysqld_master_port
-        master_infos << 60
-        master_infos << 0
-        master_infos << ""
-        master_infos << ""
-        master_infos << ""
-        master_infos << ""
-        master_infos << ""
-        master_infos << ""
-
-        tmp_output_basedir = File.expand_path(Wakame.gen_id, "/tmp")
-        FileUtils.mkdir_p tmp_output_basedir
-        master_info = File.expand_path('master.info', tmp_output_basedir)
-        file = File.new(master_info, "w")
-        file.puts(master_infos.join("\n"))
-        file.chmod(0664)
-        file.close
-
-        3.times do |i|
-          system("/bin/sync")
-          sleep 1.0
-        end
-
-        Wakame.log.debug("scp -i #{Wakame.config.ssh_private_key} -o \"UserKnownHostsFile #{Wakame.config.ssh_known_hosts}\" #{master_info} root@#{@mysqld_master_host}:#{@mysqld_master_datadir}/" )
-        system("scp -i #{Wakame.config.ssh_private_key} -o \"UserKnownHostsFile #{Wakame.config.ssh_known_hosts}\" #{master_info} root@#{@mysqld_master_host}:#{@mysqld_master_datadir}/" )
-        Wakame.log.debug("ssh -i #{Wakame.config.ssh_private_key} -o \"UserKnownHostsFile #{Wakame.config.ssh_known_hosts}\" root@#{@mysqld_master_host} chown mysql:mysql #{@mysqld_master_datadir}/master.info" )
-        system("ssh -i #{Wakame.config.ssh_private_key} -o \"UserKnownHostsFile #{Wakame.config.ssh_known_hosts}\" root@#{@mysqld_master_host} chown mysql:mysql #{@mysqld_master_datadir}/master.info" )
-
-        3.times do |i|
-          system("/bin/sync")
-          sleep 1.0
-        end
-
-        FileUtils.rm_rf tmp_output_basedir
-
-        # 2. snapshot
-        Wakame.log.debug("create_snapshot (#{@ebs_volume})")
-        snapshot_map = vm_manipulator.create_snapshot(@ebs_volume)
-        16.times do |i|
-          Wakame.log.debug("describe_snapshot(#{snapshot_map.snapshotId}) ... #{i}")
-          snapshot_map = vm_manipulator.describe_snapshot(snapshot_map["snapshotId"])
-          if snapshot_map["status"] == "completed"
-            break
-          end
-          sleep 1.0
-        end
-        if snapshot_map["status"] != "completed"
-          raise "#{snapshot_map.snapshotId} status is #{snapshot_map.status}"
-        end
-
-        # 3. unlock mysql-master
-        system("echo 'UNLOCK TABLES;' | /usr/bin/mysql -h#{@mysqld_master_host} -P#{@mysqld_master_port} -u#{@mysqld_master_user}  -p#{@mysqld_master_pass}")
-
-        # create volume /dev/xxxx
-        Wakame.log.debug("create_volume_from_snapshot(#{volume_map.availabilityZone}, #{snapshot_map.snapshotId})")
-        created_volume_from_snapshot_map = vm_manipulator.create_volume_from_snapshot(volume_map["availabilityZone"], snapshot_map["snapshotId"])
-        volume_from_snapshot_map = created_volume_from_snapshot_map
-        16.times do |i|
-          Wakame.log.debug("describe_snapshot(#{snapshot_map.snapshotId}) ... #{i}")
-          volume_from_snapshot_map = vm_manipulator.describe_snapshot(snapshot_map["snapshotId"])
-          if volume_from_snapshot_map["status"] == "completed"
-            break
-          end
-          sleep 1.0
-        end
-        if volume_from_snapshot_map["status"] != "completed"
-          raise "#{volume_from_snapshot_map.snapshotId} status is #{volume_from_snapshot_map.status}"
-        end
-
-        # attach volume
-        attach_volume_map = vm_manipulator.attach_volume(svc.agent.agent_id, created_volume_from_snapshot_map["volumeId"], @ebs_device)
-        16.times do |i|
-          Wakame.log.debug("describe_volume(#{attach_volume_map.volumeId}) ... #{i}")
-          attach_volume_map = vm_manipulator.describe_volume(created_volume_from_snapshot_map["volumeId"])
-          if attach_volume_map["status"] == "in-use"
-            break
-          end
-          sleep 1.0
-        end
-        if attach_volume_map["status"] != "in-use"
-          raise "#{attach_volume_map.volumeId} status is #{attach_volume_map.status}"
-        end
-      end
-        
-      def start
-        mount_point_dev=`df "#{@mysqld_datadir}" | awk 'NR==2 {print $1}'`
-        if mount_point_dev != @ebs_device
-          Wakame.log.debug("Mounting EBS volume: #{@ebs_device} as #{@mysqld_datadir} (with option: #{@ebs_mount_option})")
-          system("/bin/mount -o #{@ebs_mount_option} #{@ebs_device} #{@mysqld_datadir}")
-        end
-        system(Wakame.config.root + "/config/init.d/mysql-slave start")
-      end
-      
-      def check
-        system("/usr/bin/mysqladmin --defaults-file=/home/wakame/config/mysql-slave/my-slave.cnf ping > /dev/null")
-        return false if $? != 0
-        true
-      end
-      
-      def stop
-        system(Wakame.config.root + "/config/init.d/mysql-slave stop")
-      end
     end
-    
   end
 end

data/lib/wakame/trigger.rb

@@ -27,17 +27,8 @@ module Wakame
     end
 
     def trigger_action(action)
-      found = rule_engine.active_jobs.find { |id, job|
-        job[:src_rule].class == self.class
-      }
-
-      if found
-        Wakame.log.warn("#{self.class}: Exisiting Job \"#{found[:job_id]}\" was kicked from this rule and it's still running. Skipping...")
-        raise CancelActionError
-      end
-
       rule_engine.create_job_context(self, action)
-      action.bind_triggered_rule(self)
+      action.bind_trigger(self)
       
       rule_engine.run_action(action)
       action.job_id

data/lib/wakame/triggers/process_command.rb

@@ -2,6 +2,13 @@
 module Wakame
   module Triggers
     class ProcessCommand < Trigger
+      def_attribute :status
+      def_attribute :job_id
+      def_attribute :completion_status
+      def_attribute :parent_action
+      def_attribute :acquire_lock, false
+
+      attr_reader :trigger
       def register_hooks
         @@command_thread ||= Thread.new {
           while cmd = self.command_queue.deq_cmd
@@ -9,8 +16,8 @@ module Wakame
             begin
               EM.barrier {
                 Wakame.log.debug("#{self.class}: Being processed the command: #{cmd.class}")
-                cmd.run(self)
-                res = cmd
+                res = cmd.run(self)
+                res
               }
             rescue => e
               Wakame.log.error(e)

data/lib/wakame/util.rb

@@ -206,29 +206,29 @@ module AttributeHelper
       @attr_attributes ||= {}
     end
 
-    def attr(name, assignable=false)
-      attr_attributes[name.to_sym]={}
-      attr_without_trap(name, assignable)
-    end
+#     def attr(name, assignable=false)
+#       attr_attributes[name.to_sym]={}
+#       attr_without_trap(name, assignable)
+#     end
     
-    # Override Object.attr_accessor to 
-    def attr_accessor(*args)
-      args.each { |name|
-        attr(name, true)
-      }
-    end
+#     # Override Object.attr_accessor to 
+#     def attr_accessor(*args)
+#       args.each { |name|
+#         attr(name, true)
+#       }
+#     end
     
-    def attr_reader(*args)
-      args.each { |name|
-        attr(name, false)
-      }
-    end
+#     def attr_reader(*args)
+#       args.each { |name|
+#         attr(name, false)
+#       }
+#     end
     
-    def attr_writer(*args)
-      args.each { |name|
-        attr(name, true)
-      }
-    end
+#     def attr_writer(*args)
+#       args.each { |name|
+#         attr(name, true)
+#       }
+#     end
 
     def def_attribute(name, *args)
       attr = {}
@@ -276,7 +276,7 @@ module AttributeHelper
   private
   def self.included(klass)
     klass.class.class_eval {
-      alias :attr_without_trap :attr unless self.respond_to?(:attr_without_trap, true)
+      #alias :attr_without_trap :attr unless self.respond_to?(:attr_without_trap, true)
     }
     klass.extend ClassMethods
   end

data/lib/wakame/vm_manipulator.rb

@@ -172,6 +172,7 @@ module Wakame
             rkey = key.tr('-', '_')
             attrs[rkey.to_sym]=query_metadata_uri(key)
           }
+          attrs[:availability_zone] = query_metadata_uri('placement/availability-zone')
           attrs
         end
         module_function :fetch_local_attrs

data/tests/test_service.rb

@@ -1,13 +1,11 @@
 
 
-$:.unshift File.dirname(__FILE__) + '/../lib'
-require 'rubygems'
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+$:.unshift(File.dirname(__FILE__))
 
-require 'test/unit'
-require 'wakame'
-require 'wakame/service.rb'
+require 'setup_master.rb'
 
-WAKAME_ROOT="#{File.dirname(__FILE__)}/.."
+require 'wakame/service.rb'
 
 class TestService < Test::Unit::TestCase
   include Wakame::Service
@@ -57,4 +55,30 @@ class TestService < Test::Unit::TestCase
       spec.current.attrs
     }
   end
+
+
+  def test_queued_lock
+    q = Wakame::Service::LockQueue.new(nil)
+    q.set('Apache', '12345')
+    q.set('MySQL', '12345')
+    q.set('MySQL2', '12345')
+    q.set('Apache', '6789')
+    q.set('LB', '6789')
+    assert_equal(:runnable, q.test('12345'))
+    assert_equal(:wait, q.test('6789'))
+    assert_equal(:pass, q.test('unknown'))
+    #puts q.inspect
+    q.quit('12345')
+    assert_equal(:pass, q.test('12345'))
+    assert_equal(:runnable, q.test('6789'))
+    #puts q.inspect
+    q.set('Apache', '2345')
+    q.set('LB', '2345')
+    q.set('MySQL', '2345')
+    assert_equal(:runnable, q.test('6789'))
+    assert_equal(:wait, q.test('2345'))
+    q.quit('2345')
+    assert_equal(:runnable, q.test('6789'))
+    assert_equal(:pass, q.test('2345'))
+  end
 end

data/wakame_generators/resource/templates/ec2_elb/ec2_elb.rb

@@ -0,0 +1,80 @@
+class Ec2ELB < Wakame::Service::Resource
+  
+  def_attribute :elb_name, 'unakatsuo'
+  def_attribute :require_agent, false
+  def_attribute :just_unregister_when_stop, false
+  
+  def on_parent_changed(svc, action)
+  end
+
+  def start(svc, action)
+    elb = create_elb
+
+    parents = svc.parent_instances.dup
+
+    vm_slice_ids = parents.collect{|a| a.agent.attr[:instance_id] }
+    av_zones = parents.collect{|a| a.agent.attr[:availability_zone] }.uniq
+    Wakame.log.info("Setting up the ELB #{self.elb_name} with #{vm_slice_ids.join(', ')}")
+    begin
+      res = elb.describe_load_balancers(self.elb_name)
+      elbdesc = res[0]
+    rescue RightAws::AwsError => e
+      if e.include?(/\ALoadBalancerNotFound\Z/)
+
+        elb.create_load_balancer(self.elb_name, av_zones,[{ :protocol => :http, :load_balancer_port => 80,  :instance_port => 80 },
+                                                          { :protocol => :tcp,  :load_balancer_port => 443, :instance_port => 443 } ])
+      else
+        Wakame.log.error(e.errors.inspect)
+        raise e
+      end
+
+      res = elb.describe_load_balancers(self.elb_name)
+      elbdesc = res[0]
+    end
+
+    elb_members, elb_not_members = vm_slice_ids.partition{|i| elbdesc[:instances].member?(i) }
+    unless elb_not_members.empty?
+      elb.register_instances_with_load_balancer(self.elb_name, *elb_not_members)
+    end
+
+    EM.barrier {
+      svc.update_status(Wakame::Service::STATUS_ONLINE)
+    }
+  end
+  
+  def stop(svc, action)
+    elb = create_elb
+
+    if self.just_unregister_when_stop
+      parents = svc.parent_instances.dup
+      vm_slice_ids = parents.collect{|a| a.agent.attr[:instance_id] }
+      Wakame.log.info("Deregistering the VM instances (#{vm_slice_ids.join(', ')}) from ELB #{self.elb_name}")
+
+      elb.deregister_instances_with_load_balancer(self.elb_name, *vm_slice_id)
+    else
+      Wakame.log.info("Destroying the ELB #{self.elb_name}.")
+      # Ignore errors in case of any issues.
+      begin
+        elb.delete_load_balancer(self.elb_name)
+      rescue => e
+        Wakame.log.error(e)
+      end
+    end
+
+    EM.barrier {
+      svc.update_status(Wakame::Service::STATUS_OFFLINE)
+    }
+
+  end
+
+  private
+  def create_elb
+    require 'right_aws'
+    RightAws::ElbInterface.new(Wakame.config.aws_access_key, Wakame.config.aws_secret_key)
+  end
+
+  def setup_elb
+    elb = create_elb
+  end
+
+end

data/wakame_generators/resource/templates/mysql_master/conf/my.cnf

@@ -19,6 +19,7 @@
 [client]
 port		= <%= property.mysqld_port %>
 socket		= /var/run/mysqld/mysqld.sock
+default-character-set = utf8
 
 # Here is entries for some specific programs
 # The following values assume you have at least 32M ram
@@ -47,6 +48,7 @@ basedir		= /usr
 datadir		= <%= property.mysqld_datadir %>
 tmpdir		= /tmp
 language	= /usr/share/mysql/english
+default-character-set = utf8
 skip-external-locking
 #
 # Instead of skip-networking the default is now to listen only on
@@ -85,7 +87,7 @@ long_query_time = 2
 # The following can be used as easy to replay backup logs or for replication.
 # note: if you are setting up a replication slave, see README.Debian about
 #       other settings you may need to change.
-server-id		= <%= property.mysqld_server_id %>
+server-id		= 1
 log_bin			= <%= property.mysqld_log_bin %>
 #expire_logs_days	= 10
 #max_binlog_size         = 100M

data/wakame_generators/resource/templates/mysql_master/init.d/mysql

@@ -44,7 +44,7 @@ export HOME=/etc/mysql/
 #
 # Usage: void mysqld_get_param option
 mysqld_get_param() {
-	/usr/sbin/mysqld --print-defaults \
+	/usr/sbin/mysqld --defaults-file=$CONF --print-defaults \
 		| tr " " "\n" \
 		| grep -- "--$1" \
 		| tail -n 1 \
@@ -114,8 +114,8 @@ case "${1:-''}" in
 	    if mysqld_status check_alive warn; then
                 log_end_msg 0
 	        # Now start mysqlcheck or whatever the admin wants.
-	        output=$(/etc/mysql/debian-start)
-		[ -n "$output" ] && log_action_msg "$output"
+	        #output=$(/etc/mysql/debian-start)
+		#[ -n "$output" ] && log_action_msg "$output"
 	    else
 	        log_end_msg 1
 		log_failure_msg "Please take a look at the syslog"

data/wakame_generators/resource/templates/mysql_master/mysql_master.rb

@@ -3,13 +3,15 @@ class MySQL_Master < Wakame::Service::Resource
 
   def_attribute :duplicable, false
   def_attribute :mysqld_basedir, '/home/wakame/mysql'
-  def_attribute :mysqld_server_id, 1
   def_attribute :mysqld_port, 3306
 
   def_attribute :ebs_volume, ''
   def_attribute :ebs_device, '/dev/sdm'
   def_attribute :ebs_mount_option, 'noatime'
 
+  def_attribute :repl_user, 'wakame-repl'
+  def_attribute :repl_pass, 'wakame-slave'
+
   def basedir
     File.join(Wakame.config.root_path, 'cluster', 'resources', 'mysql_master')
   end
@@ -28,12 +30,7 @@ class MySQL_Master < Wakame::Service::Resource
     template.chmod("init.d/mysql", 0755)
   end
 
-
   def start(svc, action)
-    vm_manipulator = Wakame::VmManipulator.create
-
-    # $ echo "GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO 'wakame-repl'@'%' IDENTIFIED BY 'wakame-slave';" | /usr/bin/mysql -h#{mysql_master_ip} -uroot
-
     require 'right_aws'
     ec2 = RightAws::Ec2.new(Wakame.config.aws_access_key, Wakame.config.aws_secret_key)
 
@@ -69,7 +66,6 @@ class MySQL_Master < Wakame::Service::Resource
         }
       }
       cond.wait
-      
     end
 
     cond = ConditionalWait.new { |cond|
@@ -84,6 +80,11 @@ class MySQL_Master < Wakame::Service::Resource
                          :command, "/usr/bin/mysqladmin --defaults-file=#{svc.agent.root_path}/tmp/config/mysql_master/conf/my.cnf ping > /dev/null") { |req|
     }
 
+    action.actor_request(svc.agent.agent_id, '/system/sync') { |req|
+      req.wait
+      Wakame.log.debug("sync")
+    }
+
     action.actor_request(svc.agent.agent_id, '/system/mount', self.ebs_device, self.mysqld_datadir, self.ebs_mount_option) { |req|
       req.wait
       Wakame.log.debug("MySQL volume was mounted: #{self.mysqld_datadir}")
@@ -98,18 +99,16 @@ class MySQL_Master < Wakame::Service::Resource
   end
   
   def stop(svc, action)
-    cond = ConditionalWait.new { |cond|
-      cond.wait_event(Wakame::Event::ServiceOffline) { |event|
+    cond = ConditionalWait.new { |c|
+      c.wait_event(Wakame::Event::ServiceOffline) { |event|
         event.instance_id == svc.instance_id
       }
-    }
-    
+    }    
     action.actor_request(svc.agent.agent_id, '/daemon/stop', 'mysql_master', 'init.d/mysql') { |req| req.wait }
     action.actor_request(svc.agent.agent_id, '/system/umount', self.mysqld_datadir) { |req|
       req.wait
       Wakame.log.debug("MySQL volume unmounted")
     }
-
     cond.wait
 
     action.actor_request(svc.agent.agent_id,