wakame 0.4.2 → 0.5.0

data/lib/wakame/{rule_engine.rb → action_manager.rb}

@@ -1,4 +1,5 @@
 
+require 'thread'
 require 'timeout'
 
 module Wakame
@@ -6,14 +7,12 @@ module Wakame
   class CancelBroadcast < StandardError; end
   class GlobalLockError < StandardError; end
 
-  class RuleEngine
-    
-    FORWARD_ATTRS=[:command_queue, :agent_monitor, :service_cluster, :master]
-
-    attr_reader :triggers, :active_jobs
+  class ActionManager
+    include Manager
+    attr_reader :active_jobs, :lock_queue
 
     def master
-      service_cluster.master
+      Wakame::Master.instance
     end
 
     def command_queue
@@ -24,38 +23,16 @@ module Wakame
       master.agent_monitor
     end
 
-    def service_cluster
-      @service_cluster
-    end
-
-    def initialize(service_cluster, &blk)
-      @service_cluster = service_cluster
-      @triggers = []
-      
+    def initialize()
       @active_jobs = {}
       @job_history = []
-      instance_eval(&blk) if blk
+      @lock_queue = LockQueue.new
     end
 
-    def register_trigger(trigger)
-      Wakame.log.debug("Registering trigger #{trigger.class}")
-      trigger.bind_engine(self)
-      trigger.register_hooks
-      @triggers << trigger
+    def init
     end
 
-    def create_job_context(trigger, root_action)
-      root_action.job_id = job_id = Wakame.gen_id
-
-      @active_jobs[job_id] = {
-        :job_id=>job_id,
-        :src_trigger=>trigger,
-        :create_at=>Time.now,
-        :start_at=>nil,
-        :complete_at=>nil,
-        :root_action=>root_action,
-        :notes=>{}
-      }
+    def terminate
     end
 
     def cancel_action(job_id)
@@ -92,7 +69,19 @@ module Wakame
       end
     end
 
+    def trigger_action(action)
+      raise ArguemntError unless action.is_a?(Action)
+      context = create_job_context(action)
+      action.action_manager = self
+      action.job_id = context[:job_id]
+
+      run_action(action)
+      action.job_id
+    end
+
+
     def run_action(action)
+      raise ArguemntError unless action.is_a?(Action)
       job_context = @active_jobs[action.job_id]
       raise "The job session is killed.: job_id=#{action.job_id}" if job_context.nil?
 
@@ -110,7 +99,7 @@ module Wakame
             begin
               action.bind_thread(Thread.current)
               action.status = :running
-              Wakame.log.debug("Start action : #{action.class.to_s} triggered by [#{action.trigger.class}]")
+              Wakame.log.debug("Start action : #{action.class.to_s} #{action.parent_action.nil? ? '' : ('sub-action of ' + action.parent_action.class.to_s)}")
               ED.fire_event(Event::ActionStart.new(action))
               begin
                 action.run
@@ -129,7 +118,7 @@ module Wakame
               ED.fire_event(Event::ActionFailed.new(action, e))
               res = e
             rescue => e
-              Wakame.log.debug("Failed action : #{action.class.to_s} due to #{e}")
+              Wakame.log.error("Failed action : #{action.class.to_s} due to #{e}")
               Wakame.log.error(e)
               action.completion_status = :failed
               begin
@@ -161,7 +150,7 @@ module Wakame
             
             actary = []
             job_context[:root_action].walk_subactions {|a| actary << a }
-            Wakame.log.debug(actary.collect{|a| {a.class.to_s=>a.status}}.inspect)
+            #Wakame.log.debug(actary.collect{|a| {a.class.to_s=>a.status}}.inspect)
 
             if res.is_a?(Exception)
               job_context[:exception]=res
@@ -178,7 +167,7 @@ module Wakame
               job_context[:complete_at]=Time.now
               @job_history << job_context
               @active_jobs.delete(job_context[:job_id])
-              service_cluster.lock_queue.quit(job_context[:job_id])
+              @lock_queue.quit(job_context[:job_id])
             end
           }
         rescue => e
@@ -187,5 +176,128 @@ module Wakame
       }
     end
 
+    private
+    def create_job_context(root_action)
+      raise ArguemntError unless root_action.is_a?(Action)
+      root_action.job_id = job_id = Wakame.gen_id
+
+      @active_jobs[job_id] = {
+        :job_id=>job_id,
+        :create_at=>Time.now,
+        :start_at=>nil,
+        :complete_at=>nil,
+        :root_action=>root_action,
+        :notes=>{}
+      }
+    end
+
   end
+
+  class LockQueue
+    def initialize()
+      @locks = {}
+      @id2res = {}
+
+      @queue_by_thread = {}
+      @qbt_m = ::Mutex.new
+    end
+    
+    def set(resource, id)
+      # Ths Job ID already holds/reserves the lock regarding the resource.
+      return if @id2res.has_key?(id) && @id2res[id].has_key?(resource.to_s)
+
+      # Need to use EM.barrier while RuleEngine is using EventMachine's threads.
+      #StatusDB.barrier {
+      EM.barrier {
+        @locks[resource.to_s] ||= []
+        @id2res[id] ||= {}
+        
+        @id2res[id][resource.to_s]=1
+        @locks[resource.to_s] << id
+      }
+      Wakame.log.debug("#{self.class}: set(#{resource.to_s}, #{id})" + "\n#{self.inspect}")
+    end
+
+    def reset()
+      @locks.keys { |k|
+        @locks[k].clear
+      }
+      @id2res.clear
+    end
+
+    def test(id)
+      reslist = @id2res[id]
+      return :pass if reslist.nil? || reslist.empty?
+
+      # 
+      if reslist.keys.all? { |r| id == @locks[r.to_s][0] }
+        return :runnable
+      else
+        return :wait
+      end
+    end
+
+    def wait(id, tout=60*30)
+      @qbt_m.synchronize { @queue_by_thread[Thread.current] = ::Queue.new }
+
+      timeout(tout) {
+        while test(id) == :wait
+          Wakame.log.debug("#{self.class}: Job #{id} waits for locked resouces: #{@id2res[id].keys.join(', ')}")
+          break if id == @queue_by_thread[Thread.current].deq
+        end
+      }
+    ensure
+      @qbt_m.synchronize { @queue_by_thread.delete(Thread.current) }
+    end
+    
+    def quit(id)
+      # Need to use EM.barrier while RuleEngine is using EventMachine's threads.
+      #StatusDB.barrier {
+      EM.barrier {
+        case test(id)
+        when :runnable, :wait
+          @id2res[id].keys.each { |r| @locks[r.to_s].delete_if{ |i| i == id } }
+          @locks.delete_if{ |k,v| v.nil? || v.empty? }
+
+          @qbt_m.synchronize {
+            @queue_by_thread.each {|t, q| q.enq(id) }
+          }
+        end
+        
+        @id2res.delete(id)
+      }
+      Wakame.log.debug("#{self.class}: quit(#{id})" + "\n#{self.inspect}")
+    end
+
+    def clear_resource(resource)
+    end
+
+    def inspect
+      output = @locks.collect { |k, lst|
+        [k, lst].flatten
+      }
+      return "" if output.empty?
+
+      # Table display
+      maxcolws = (0..(output.size)).zip(*output).collect { |i| i.shift; i.map!{|i| (i.nil? ? "" : i).length }.max }
+      maxcol = maxcolws.size
+      maxcolws.reverse.each { |i| 
+        break if i > 0
+        maxcol -= 1
+      }
+
+      textrows = output.collect { |x|
+        buf=""
+        maxcol.times { |n|
+          buf << "|" + (x[n] || "").ljust(maxcolws[n])
+        }
+        buf << "|"
+      }
+
+      "+" + (["-"] * (textrows[0].length - 2)).join('') + "+\n" + \
+      textrows.join("\n") + \
+      "\n+" + (["-"] * (textrows[0].length - 2)).join('')+ "+"
+    end
+  end
+
 end

data/lib/wakame/actions/deploy_config.rb

@@ -0,0 +1,35 @@
+module Wakame
+  module Actions
+    class DeployConfig < Action
+      def initialize(svc)
+        @svc = svc
+      end
+
+      def run
+        Wakame.log.debug("#{self.class}: run() Begin: #{@svc.resource.class}")
+        raise "CloudHost is not mapped Agent: CloudHost.id=#{@svc.cloud_host.id}" unless @svc.cloud_host.mapped?
+
+        acquire_lock { |lst|
+          lst << @svc.resource.class.to_s
+        }
+
+        begin
+          tmpl = Wakame::Template.new(@svc)
+          tmpl.render_config
+          
+          src_path = tmpl.tmp_basedir.dup
+          src_path.sub!('/$', '') if File.directory? src_path
+          
+          dest_path = File.expand_path("tmp/config/" + File.basename(tmpl.basedir), @svc.cloud_host.root_path)
+          #Wakame::Util.exec("rsync -e 'ssh -i #{Wakame.config.ssh_private_key} -o \"UserKnownHostsFile #{Wakame.config.ssh_known_hosts}\"' -au #{src_path}/ root@#{@svc.cloud_host.agent_ip}:#{dest_path}")
+          Wakame::Util.exec("rsync -e 'ssh -i #{Wakame.config.ssh_private_key} -o \"StrictHostKeyChecking no\"' -au #{src_path}/ root@#{@svc.cloud_host.agent_ip}:#{dest_path}")
+          
+        ensure
+          tmpl.cleanup if tmpl
+        end
+
+        Wakame.log.debug("#{self.class}: run() End: #{@svc.resource.class}")
+      end
+    end
+  end
+end

data/lib/wakame/actions/launch_cluster.rb

@@ -2,30 +2,28 @@ module Wakame
   module Actions
     class LaunchCluster < Action
       def initialize
-        @acquire_lock = true
       end
 
       def run
         if service_cluster.status == Service::ServiceCluster::STATUS_ONLINE
-          Wakame.log.info("The service cluster is up & running already")
-          raise CancelActionError
+          Wakame.log.info("Ignore to launch the cluster as is already ONLINE: #{service_cluster.name}")
+          return
         end
 
-        EM.barrier {
-          service_cluster.launch
-        }
         levels = service_cluster.dg.levels
-        Wakame.log.debug("#{self.class}: Resource launch order: " + levels.collect {|lv| '['+ lv.collect{|prop| "#{prop.class}" }.join(', ') + ']' }.join(', '))
+        Wakame.log.debug("#{self.class}: Scheduled resource launch order: " + levels.collect {|lv| '['+ lv.collect{|prop| "#{prop.class}" }.join(', ') + ']' }.join(', '))
         acquire_lock { |list|
           levels.each {|lv| list << lv.collect{|res| res.class } }
         }
 
         levels.each { |lv|
-          lv.each { |svc_prop|
-            trigger_action(PropagateInstances.new(svc_prop))
+          Wakame.log.info("#{self.class}: Launching resources: #{lv.collect{|prop| "#{prop.class}" }.join(', ')}")
+          lv.each { |resource|
+            cluster.each_instance(resource) { |svc|
+              trigger_action(StartService.new(svc))
+            }
           }
           flush_subactions
-          Wakame.log.debug("#{self.class}: DG level next")
         }
       end
 

data/lib/wakame/actions/launch_vm.rb

@@ -1,51 +1,57 @@
 module Wakame
   module Actions
     class LaunchVM < Action
-      def initialize(notes_key, ref_agent=nil)
-        @ref_agent = ref_agent
+      def initialize(notes_key, vm_spec)
         @notes_key = notes_key
+        @vm_spec = vm_spec
       end
 
       USER_DATA_TMPL=<<__END__
 node=agent
-amqp_server=amqp://%s/
-agent_id=%s
+amqp_server=%s
 __END__
 
       def run
         require 'right_aws'
         ec2 = RightAws::Ec2.new(Wakame.config.aws_access_key, Wakame.config.aws_secret_key, {:cache=>false})
 
-        # Fetch the reference vm slice attributes.
-        @ref_agent ||= master.agent_monitor.master_local
-        ref_attr = ec2.describe_instances([@ref_agent.attr[:instance_id]])
-        ref_attr = ref_attr[0]
-        
+        ref_attr = @vm_spec.merge(cluster.template_vm_attr)
 
-        user_data = sprintf(USER_DATA_TMPL, master.attr[:local_ipv4], '')
+        user_data = sprintf(USER_DATA_TMPL, cluster.advertised_amqp_servers)
         Wakame.log.debug("#{self.class}: Lauching VM: #{ref_attr.inspect}\nuser_data: #{user_data}")
-        res = ec2.run_instances(ref_attr[:aws_image_id], 1, 1,
-                                ref_attr[:aws_groups],
-                                ref_attr[:ssh_key_name],
+        res = ec2.run_instances(ref_attr[:image_id], 1, 1,
+                                ref_attr[:security_groups],
+                                ref_attr[:key_name],
                                 user_data,
                                 'public', # addressing_type
-                                ref_attr[:aws_instance_type], # instance_type
+                                ref_attr[:instance_type], # instance_type
                                 nil, # kernel_id
                                 nil, # ramdisk_id
-                                ref_attr[:aws_availability_zone], # availability_zone
+                                ref_attr[:availability_zone], # availability_zone
                                 nil # block_device_mappings
                                 )[0]
         inst_id = res[:aws_instance_id]
 
         ConditionalWait.wait { | cond |
           cond.wait_event(Event::AgentMonitored) { |event|
-            event.agent.attr[:instance_id] == inst_id
+            event.agent.vm_attr[:instance_id] == inst_id
           }
           
-          cond.poll(5, 100) {
-            d = ec2.describe_instances([inst_id])[0]
-            Wakame.log.debug("#{self.class}: Polling describe_instances(#{inst_id}): #{d[:aws_state]} ")
-            d[:aws_state] == "running"
+          cond.poll(5, 360) {
+            begin
+              d = ec2.describe_instances([inst_id])[0]
+              Wakame.log.debug("#{self.class}: Polling describe_instances(#{inst_id}): #{d[:aws_state]} ")
+              d[:aws_state] == "running"
+            rescue RightAws::AwsError => e
+              if e.include?(%r'InvalidInstanceID.NotFound')
+                # describe_instances() sometime fails due to this error when it is called just after sending run_instances().
+                # This is the timing issue in AWS so that continues the polling check.
+                Wakame.log.warn("Polling to describe_instances() got InvalidInstanceID.NotFound error with #{inst_id}. Retry again.")
+                next
+              else
+                raise e
+              end
+            end
           }
         }
 

data/lib/wakame/actions/migrate_service.rb

@@ -2,44 +2,55 @@
 module Wakame
   module Actions
     class MigrateService < Action
-      def initialize(service_instance, dest_agent=nil)
-        @service_instance = service_instance
-        @destination_agent = dest_agent
+      def initialize(svc, dest_cloud_host=nil)
+        raise ArgumentError unless svc.is_a?(Service::ServiceInstance)
+        @svc = svc
+        @dest_cloud_host = dest_cloud_host
       end
 
       def run
         acquire_lock { |list|
-          list << @service_instance.resource.class
+          list << @svc.resource.class
         }
 
-        raise CancelActionError if @service_instance.status == Service::STATUS_MIGRATING
+        if @svc.status == Service::STATUS_MIGRATING 
+          Wakame.log.info("Ignore to migrate the service as is already MIGRATING: #{@svc.resource.class}")
+          return
+        end
 
-        EM.barrier {
-          @service_instance.update_status(Service::STATUS_MIGRATING)
+        StatusDB.barrier {
+          @svc.update_status(Service::STATUS_MIGRATING)
         }
-        prop = @service_instance.resource
-        if prop.duplicable
-          clone_service(prop)
+        if @svc.resource.duplicable
+          clone_service()
           flush_subactions
-          trigger_action(StopService.new(@service_instance))
+          trigger_action(StopService.new(@svc))
         else
-          
-          trigger_action(StopService.new(@service_instance))
+          trigger_action(StopService.new(@svc))
           flush_subactions
-          clone_service(prop)
+          clone_service()
         end
         flush_subactions
       end
 
+      def on_failed
+        StatusDB.barrier {
+          @svc.update_status(Service::STATUS_FAIL)
+          if @new_svc
+            @new_svc.update_status(Service::STATUS_FAIL)
+          end
+        }
+      end
+
       private
-      def clone_service(resource)
+      def clone_service()
         new_svc = nil
-        EM.barrier {
-          new_svc = service_cluster.propagate(resource, true)
+        StatusDB.barrier {
+          new_svc = service_cluster.propagate_service(@svc.id, @dest_cloud_host, true)
         }
 
-        trigger_action(StartService.new(new_svc, @destination_agent))
-        new_svc
+        trigger_action(StartService.new(new_svc))
+        @new_svc = new_svc
       end
     end
   end

data/lib/wakame/actions/notify_child_changed.rb

@@ -0,0 +1,34 @@
+module Wakame
+  module Actions
+    class NotifyChildChanged < Action
+      def initialize(svc)
+        @child_svc = svc
+      end
+      
+      def run
+        parents = []
+        StatusDB.barrier {
+          parents = @child_svc.parent_instances
+        }
+
+        acquire_lock { |lst|
+          lst << parents.map{|c| c.resource.class.to_s }.uniq
+        }
+        
+        Wakame.log.debug("#{self.class}: Parent nodes for #{@child_svc.resource.class}: " + parents.map{|c| c.resource.class }.uniq.inspect )
+
+        parents.each { |svc|
+          if svc.status != Service::STATUS_ONLINE
+            next
+          end
+
+          trigger_action { |proc_action|
+            svc.resource.on_child_changed(svc, proc_action)
+          }
+        }
+        flush_subactions
+
+      end
+    end
+  end
+end

data/lib/wakame/actions/notify_parent_changed.rb

@@ -0,0 +1,34 @@
+module Wakame
+  module Actions
+    class NotifyParentChanged < Action
+      def initialize(svc)
+        @parent_svc = svc
+      end
+      
+      def run
+        children = []
+        StatusDB.barrier {
+          children = @parent_svc.child_instances
+        }
+
+        acquire_lock { |lst|
+          lst << children.map{|c| c.resource.class.to_s }.uniq
+        }
+        
+        Wakame.log.debug("#{self.class}: Child nodes for #{@parent_svc.resource.class}: " + children.map{|c| c.resource.class }.uniq.inspect )
+
+        children.each { |svc|
+          if svc.status != Service::STATUS_ONLINE
+            next
+          end
+
+          trigger_action { |proc_action|
+            svc.resource.on_parent_changed(svc, proc_action)
+          }
+        }
+        flush_subactions
+
+      end
+    end
+  end
+end

data/lib/wakame/actions/propagate_resource.rb

@@ -0,0 +1,27 @@
+module Wakame
+  module Actions
+    class PropagateResource < Action
+
+      def initialize(resource, cloud_host_id)
+        raise ArgumentError unless resource.is_a?(Wakame::Service::Resource)
+        @resource = resource
+        @cloud_host_id = cloud_host_id
+      end
+
+      def run
+        acquire_lock { |ary|
+          ary << @resource.class.to_s
+        }
+
+        newsvc=nil
+        StatusDB.barrier {
+          newsvc = service_cluster.propagate_resource(@resource, @cloud_host_id)
+        }
+
+        trigger_action(StartService.new(newsvc))
+        flush_subactions
+      end
+
+    end
+  end
+end

data/lib/wakame/actions/propagate_service.rb

@@ -0,0 +1,27 @@
+module Wakame
+  module Actions
+    class PropagateService < Action
+
+      def initialize(svc, cloud_host_id=nil)
+        raise ArgumentError unless svc.is_a?(Wakame::Service::ServiceInstance)
+        @svc = svc
+        @cloud_host_id = cloud_host_id
+      end
+
+
+      def run
+        acquire_lock { |ary|
+          ary << @svc.resource.class.to_s
+        }
+
+        newsvc = nil
+        StatusDB.barrier {
+          newsvc = cluster.propagate_service(@svc.id, @cloud_host_id)
+        }
+        trigger_action(StartService.new(newsvc))
+        flush_subactions
+      end
+
+    end
+  end
+end

data/lib/wakame/actions/reload_service.rb

@@ -1,21 +1,33 @@
 module Wakame
   module Actions
     class ReloadService < Action
-      def initialize(service_instance)
-        @service_instance = service_instance
+      def initialize(svc)
+        @svc = svc
       end
 
       def run
-        raise "Agent is not bound on this service : #{@service_instance}" if @service_instance.agent.nil?
-        raise "The assigned agent for the service instance #{@service_instance.instance_id} is not online."  unless @service_instance.agent.status == Service::Agent::STATUS_ONLINE
-
         # Skip to act when the service is having below status.
-        #if @service_instance.status == Service::STATUS_STARTING || @service_instance.status == Service::STATUS_ONLINE
-        #  raise "Canceled as the service is being or already ONLINE: #{@service_instance.property}"
-        #end
+        if @svc.status != Service::STATUS_ONLINE
+          raise "Canceled as the service is being or already ONLINE: #{@svc.resource.class}"
+        end
+
+        StatusDB.barrier {
+          @svc.update_status(Service::STATUS_RELOADING)
+        }
+
+	@svc.resource.reload(@svc, self)
+
+        StatusDB.barrier {
+          @svc.update_status(Service::STATUS_ONLINE)
+        }
+      end
 
-	@service_instance.resource.reload(@service_instance, self)
+      def on_failed
+        StatusDB.barrier {
+          @svc.update_status(Service::STATUS_FAIL)
+        }
       end
+      
     end
   end
 end

data/lib/wakame/actions/shutdown_cluster.rb

@@ -2,7 +2,7 @@ module Wakame
   module Actions
     class ShutdownCluster < Action
       def run
-        levels = service_cluster.dg.levels
+        levels = cluster.dg.levels
         Wakame.log.debug("#{self.class}: Resource shutdown order: " + levels.collect {|lv| '['+ lv.collect{|prop| "#{prop.class}" }.join(', ') + ']' }.join(', '))
         acquire_lock { |list|
           levels.each {|lv| list << lv.collect{|res| res.class } }
@@ -17,8 +17,8 @@ module Wakame
           flush_subactions
         }
 
-        agent_monitor.registered_agents.each { |id, agent|
-          trigger_action(ShutdownVM.new(agent))
+        agent_monitor.agent_pool.group_active.keys.each { |agent_id|
+          trigger_action(ShutdownVM.new(Service::Agent.find(agent_id)))
         }
       end
     end