wakame 0.4.2 → 0.5.0

data/lib/wakame/master.rb

@@ -3,63 +3,84 @@
 require 'rubygems'
 
 require 'wakame'
-require 'wakame/packets'
-require 'wakame/service'
 require 'wakame/queue_declare'
-require 'wakame/vm_manipulator'
 
 module Wakame
 
+  module Manager
+    attr_accessor :master
+
+    def init
+    end
+    
+    def start
+    end
+    
+    def stop
+    end
+    
+    def reload
+    end
+    
+    def terminate
+    end
+  end
+  
+
   class AgentMonitor
+    include Manager
     include ThreadImmutable
-    attr_reader :registered_agents, :unregistered_agents, :master, :gc_period
 
-
-    def initialize(master)
-      bind_thread
-      @master = master
-      @registered_agents = {}
-      @unregistered_agents = {}
+    def init
       @agent_timeout = 31.to_f
       @agent_kill_timeout = @agent_timeout * 2
       @gc_period = 20.to_f
 
+      Service::AgentPool.reset
+
       # GC event trigger for agent timer & status
-      calc_agent_timeout = proc {
-        #Wakame.log.debug("Started agent GC : agents.size=#{@registered_agents.size}")
-        kill_list=[]
-        registered_agents.each { |agent_id, agent|
-          next if agent.status == Service::Agent::STATUS_OFFLINE
-          diff_time = Time.now - agent.last_ping_at
-          #Wakame.log.debug "AgentMonitor GC : #{agent_id}: #{diff_time}"
-          if diff_time > @agent_timeout.to_f
-            agent.status = Service::Agent::STATUS_TIMEOUT
-          end
+      @agent_timeout_timer = EM::PeriodicTimer.new(@gc_period) {
+        StatusDB.pass {
+          #Wakame.log.debug("Started agent GC : agents.size=#{@registered_agents.size}")
+          [self.agent_pool.group_active.keys, self.agent_pool.group_observed.keys].flatten.uniq.each { |agent_id|
+            agent = Service::Agent.find(agent_id)
+            #next if agent.status == Service::Agent::STATUS_OFFLINE
+            
+            diff_time = Time.now - agent.last_ping_at_time
+            #Wakame.log.debug "AgentMonitor GC : #{agent_id}: #{diff_time}"
+            if diff_time > @agent_timeout.to_f
+              agent.update_status(Service::Agent::STATUS_TIMEOUT)
+            end
+            
+            if diff_time > @agent_kill_timeout.to_f
+              agent_pool.unregister(agent)
+            end
+          }
           
-          if diff_time > @agent_kill_timeout.to_f
-            kill_list << agent_id
-          end
-        }
-        
-        kill_list.each { |agent_id|
-          agent = @registered_agents.delete(agent_id)
-          ED.fire_event(Event::AgentUnMonitored.new(agent)) unless agent.nil?
+          #Wakame.log.debug("Finished agent GC")
         }
-        #Wakame.log.debug("Finished agent GC")
       }
       
-      @agent_timeout_timer = EventMachine::PeriodicTimer.new(@gc_period, calc_agent_timeout)
       
       master.add_subscriber('registry') { |data|
         data = eval(data)
+        
+        StatusDB.pass {
+          agent_id = data[:agent_id]
+          
+          agent = agent_pool.create_or_find(agent_id)
+          
+          case data[:class_type]
+          when 'Wakame::Packets::Register'
+            agent.root_path = data[:root_path]
+            agent.vm_attr = data[:attrs]
+            agent.save
+            agent_pool.register(agent)
+          when 'Wakame::Packets::UnRegister'
+            agent_pool.unregister(agent)
+          end
+        }
 
-        agent_id = data[:agent_id]
-        case data[:type]
-        when 'Wakame::Packets::Register'
-          register_agent(data)
-        when 'Wakame::Packets::UnRegister'
-          unregister_agent(agent_id)
-        end
       }
 
       master.add_subscriber('ping') { |data|
@@ -69,232 +90,329 @@ module Wakame
 
         # Variable update function for the common members
         set_report_values = proc { |agent|
-          agent.status = Service::Agent::STATUS_ONLINE
-          agent.uptime = 0
-          agent.last_ping_at = Time.parse(ping[:responded_at])
-          
-          agent.attr = ping[:attrs]
-          
-          agent.services.clear
-          ping.services.each { |svc_id, i|
-            agent.services[svc_id] = master.service_cluster.instances[svc_id]
-          }
+          agent.last_ping_at = ping[:responded_at]
+          agent.vm_attr = ping[:attrs]
+
+          agent.renew_reported_services(ping[:services])
+          agent.save
+
+          agent.update_status(Service::Agent::STATUS_ONLINE)
         }
         
-        agent = agent(ping[:agent_id])
-        if agent.nil?
-          agent = Service::Agent.new(ping[:agent_id])
-          
-          set_report_values.call(agent)
+        StatusDB.pass { 
+          agent = Service::Agent.find(ping[:agent_id])
+          if agent.nil?
+            agent = Service::Agent.new
+            agent.id = ping[:agent_id]
+            
+            set_report_values.call(agent)
+
+            agent_pool.register_as_observed(agent)
+          else
+            set_report_values.call(agent)
+          end
           
-          unregistered_agents[ping[:agent_id]]=agent
-        else
-          set_report_values.call(agent)
-        end
-        
-        
-        ED.fire_event(Event::AgentPong.new(agent))
+          EventDispatcher.fire_event(Event::AgentPong.new(agent))
+        }
       }
       
       master.add_subscriber('agent_event') { |data|
         response = eval(data)
-#p response
-        case response[:type]
-        when 'Wakame::Packets::ServiceStatusChanged'
-          svc_inst = Service::ServiceInstance.instance_collection[response[:svc_id]]
-          if svc_inst
-            response_time = Time.parse(response[:responded_at])
-            svc_inst.update_status(response[:new_status], response_time, response[:fail_message])
-            
-#             tmp_event = Event::ServiceStatusChanged.new(response[:svc_id], svc_inst.property, response[:status], response[:previous_status])
-#             tmp_event.time = response_time
-#             ED.fire_event(tmp_event)
-            
-#             if response[:previous_status] != Service::STATUS_ONLINE && response[:new_status] == Service::STATUS_ONLINE
-#               tmp_event = Event::ServiceOnline.new(tmp_event.instance_id, svc_inst.property)
-#               tmp_event.time = response_time
-#               ED.fire_event(tmp_event)
-#             elsif response[:previous_status] != Service::STATUS_OFFLINE && response[:new_status] == Service::STATUS_OFFLINE
-#               tmp_event = Event::ServiceOffline.new(tmp_event.instance_id, svc_inst.property)
-#               tmp_event.time = response_time
-#               ED.fire_event(tmp_event)
-#             elsif response[:previous_status] != Service::STATUS_FAIL && response[:new_status] == Service::STATUS_FAIL
-#               tmp_event = Event::ServiceFailed.new(tmp_event.instance_id, svc_inst.property, response[:fail_message])
-#               tmp_event.time = response_time
-#               ED.fire_event(tmp_event)
-#             end
-          end
-        when 'Wakame::Packets::ActorResponse'
-          case response[:status]
-          when Actor::STATUS_RUNNING
-              ED.fire_event(Event::ActorProgress.new(response[:agent_id], response[:token], 0))
+        StatusDB.pass {
+          case response[:class_type]
+          when 'Wakame::Packets::StatusCheckResult'
+            svc_inst = Service::ServiceInstance.find(response[:svc_id])
+            if svc_inst
+              svc_inst.monitor_status = response[:status]
+              svc_inst.save
+            else
+              Wakame.log.error("#{self.class}: Unknown service ID: #{response[:svc_id]}")
+            end
+          when 'Wakame::Packets::ServiceStatusChanged'
+            svc_inst = Service::ServiceInstance.find(response[:svc_id])
+            if svc_inst
+              response_time = Time.parse(response[:responded_at])
+              svc_inst.update_status(response[:new_status], response_time, response[:fail_message])
+            end
+          when 'Wakame::Packets::ActorResponse'
+            case response[:status]
+            when Actor::STATUS_RUNNING
+              EventDispatcher.fire_event(Event::ActorProgress.new(response[:agent_id], response[:token], 0))
+            when Actor::STATUS_FAILED
+              EventDispatcher.fire_event(Event::ActorComplete.new(response[:agent_id], response[:token], response[:status], nil))
+            else
+              EventDispatcher.fire_event(Event::ActorComplete.new(response[:agent_id], response[:token], response[:status], response[:opts][:return_value]))
+            end
           else
-              ED.fire_event(Event::ActorComplete.new(response[:agent_id], response[:token], response[:status], response[:opts][:return_value]))
+            Wakame.log.warn("#{self.class}: Unhandled agent response: #{response[:class_type]}")
           end
-        else
-          Wakame.log.warn("#{self.class}: Unhandled agent response: #{response[:type]}")
-        end
+        }
       }
-    
-    end
 
-
-    def agent(agent_id)
-      registered_agents[agent_id] || unregistered_agents[agent_id]
     end
 
-    def register_agent(data)
-      agent_id = data[:agent_id]
-      agent = registered_agents[agent_id]
-      if agent.nil?
-        agent = unregistered_agents[agent_id]
-        if agent.nil?
-          # The agent is going to be registered at first time.
-          agent = Service::Agent.new(agent_id)
-          registered_agents[agent_id] = agent
-        else
-          # Move the reference from unregistered group to the registered group.
-          registered_agents[agent_id] = unregistered_agents[agent_id]
-          unregistered_agents.delete(agent_id)
-        end
-        Wakame.log.debug("The Agent has been registered: #{data.inspect}")
-        #Wakame.log.debug(unregistered_agents)
-        ED.fire_event(Event::AgentMonitored.new(agent))
-      end
-      agent.root_path = data[:root_path]
-      agent.attr = data[:attrs]
+    def terminate
+      @agent_timeout_timer.cancel
     end
 
-    def unregister_agent(agent_id)
-      agent = registered_agents[agent_id]
-      if agent
-        unregistered_agents[agent_id] = registered_agents[agent_id]
-        registered_agents.delete(agent_id)
-        ED.fire_event(Event::AgentUnMonitored.new(agent))
-      end
+    def agent_pool
+      Service::AgentPool.instance
     end
 
+   end
 
-#     def bind_agent(service_instance, &filter)
-#       agent_id, agent = @agents.find { |agent_id, agent|
 
-#         next false if agent.has_service_type?(service_instance.property.class)
-#         filter.call(agent)
-#       }
-#       return nil if agent.nil?
-#       service_instance.bind_agent(agent)
-#       agent
-#     end
 
-#     def unbind_agent(service_instance)
-#       service_instance.unbind_agent
-#     end
-    
-    # Retruns the master local agent object
-    def master_local
-      agent = registered_agents[@master.master_local_agent_id]
-      puts "#{agent} = registered_agents[#{@master.master_local_agent_id}]"
-      raise "Master does not identify the master local agent yet." if agent.nil?
-      agent
-    end
+   class ClusterManager
+     include Manager
+
+     class ClusterConfigLoader
+
+       def load(cluster_rb_path=Wakame.config.cluster_config_path)
+         Wakame.log.info("#{self.class}: Loading cluster.rb: #{cluster_rb_path}")
+         @loaded_cluster_names = {}
+
+         eval(File.readlines(cluster_rb_path).join(''), binding)
+
+         # Clear uninitialized cluster data in the store.
+         Service::ServiceCluster.find_all.each { |cluster|
+           cluster.delete unless @loaded_cluster_names.has_key?(cluster.name)
+         }
+
+         @loaded_cluster_names
+       end
 
-    def each_online(&blk)
-      registered_agents.each { |k, v|
-        next if v.status != Service::Agent::STATUS_ONLINE
-        blk.call(v)
-      }
-    end
 
-     def dump_status
-       ag = []
-       res = {:registered=>[], :unregistered=>[]}
+       private
+       def define_cluster(name, &blk)
+         cluster = Service::ServiceCluster.find(Service::ServiceCluster.id(name))
+         if cluster.nil?
+           cluster = Service::ServiceCluster.new
+           cluster.name = name
+         end
+
+         Service::AgentPool.reset
+         cluster.reset
+
+         blk.call(cluster)
+
+         cluster.save
+
+         Wakame.log.info("#{self.class}: Loaded Service Cluster: #{cluster.name}")
+         @loaded_cluster_names[name]=cluster.id
+       end
+
+     end
+
+     def init
+       # Periodical cluster status updater
+       @status_check_timer = EM::PeriodicTimer.new(5) {
+         StatusDB.pass {
+           clusters.each { |cluster_id|
+             Service::ServiceCluster.find(cluster_id).update_cluster_status
+           }
+         }
+       }
        
-       @registered_agents.each { |key, a|
-         res[:registered] << a.dump_status
+       # Event based cluster status updater
+       @check_event_tickets = []
+       [Event::ServiceOnline, Event::ServiceOffline, Event::ServiceFailed].each { |evclass|
+         @check_event_tickets << EventDispatcher.subscribe(evclass) { |event|
+           StatusDB.pass {
+             clusters.each { |cluster_id|
+               Service::ServiceCluster.find(cluster_id).update_cluster_status
+             }
+           }
+         }
        }
-       @unregistered_agents.each { |key, a|
-         res[:unregistered] << a.dump_status
+
+     end
+
+     def reload
+     end
+
+     def terminate
+       @status_check_timer.cancel
+       @check_event_tickets.each { |t|
+         EventDispatcher.unsubscribe(t)
        }
-       res
      end
-  end
 
-  class Master
-    include Wakame::AMQPClient
-    include Wakame::QueueDeclare
+     def clusters
+       ServiceClusterPool.all.map{|r| r.service_cluster_id }
+     end
+
+     def register(cluster)
+       raise ArgumentError unless cluster.is_a?(Service::ServiceCluster)
+       ServiceClusterPool.register_cluster(cluster.name)
+     end
 
-    define_queue 'agent_event', 'agent_event'
-    define_queue 'ping', 'ping'
-    define_queue 'registry', 'registry'
+     def unregister(cluster_id)
+       @clusters.delete(cluster_id)
+     end
 
-    attr_reader :command_queue, :agent_monitor, :configuration, :service_cluster, :started_at
+     def load_config_cluster
+       ClusterConfigLoader.new.load.each { |name, id|
+         ServiceClusterPool.register_cluster(name)
+       }
+       resolve_template_vm_attr
+     end
 
-    def initialize(opts={})
-      pre_setup
 
-      connect(opts) {
-        post_setup
-      }
-      Wakame.log.info("Started master process : WAKAME_ROOT=#{Wakame.config.root_path} WAKAME_ENV=#{Wakame.config.environment}")
+     private
+     def resolve_template_vm_attr
+       ServiceClusterPool.each_cluster { |cluster|
+         cluster_id = cluster.id
+
+         if cluster.template_vm_attr.nil? || cluster.template_vm_attr.empty?
+           update_template_spec = lambda { |agent|
+             raise ArgumentError unless agent.is_a?(Service::Agent)
+
+             require 'right_aws'
+             ec2 = RightAws::Ec2.new(Wakame.config.aws_access_key, Wakame.config.aws_secret_key)
+             
+             ref_attr = ec2.describe_instances([agent.vm_attr[:instance_id]])
+             ref_attr = ref_attr[0]
+                 
+             cluster = Service::ServiceCluster.find(cluster_id)
+             spec = cluster.template_vm_spec
+             Service::VmSpec::EC2.vm_attr_defs.each { |k, v|
+               spec.attrs[k] = ref_attr[v[:right_aws_key]]
+             }
+             cluster.save
+             
+             Wakame.log.debug("ServiceCluster \"#{cluster.name}\" template_vm_attr based on VM \"#{agent.vm_attr[:instance_id]}\" : #{spec.attrs.inspect}")
+           }
+
+
+           agent_id = Service::AgentPool.instance.group_active.keys.first
+           if agent_id.nil?
+             # Set a single shot event handler to set the template values up from the first connected agent.
+             EventDispatcher.subscribe_once(Event::AgentMonitored) { |event|
+               StatusDB.pass {
+                 update_template_spec.call(event.agent)
+               }
+             }
+           else
+             StatusDB.pass {
+               update_template_spec.call(Service::Agent.find(agent_id))
+             }
+           end
+         end
+
+         if cluster.advertised_amqp_servers.nil?
+           StatusDB.pass {
+             cluster = Service::ServiceCluster.find(cluster_id)
+             cluster.advertised_amqp_servers = master.amqp_server_uri.to_s
+             cluster.save
+             Wakame.log.debug("ServiceCluster \"#{cluster.name}\" advertised_amqp_servers: #{cluster.advertised_amqp_servers}")
+           }
+         end
+
+       }
+     end
+
+
+     class ServiceClusterPool < Sequel::Model
+       plugin :schema
+       
+       def self.initialize_table
+         set_schema do
+           primary_key :id, :type => Integer
+           varchar :service_cluster_id
+         end
+         create_table unless table_exists?
+       end
+       
+       
+       def self.register_cluster(name)
+         id = Service::ServiceCluster.id(name)
+         
+         self.find_or_create(:service_cluster_id=>id)
+       end
+       
+       def self.unregister_cluster(name)
+         id = Service::ServiceCluster.id(name)
+         self.delete(:service_cluster_id=>id)
+       end
+       
+       def self.each_cluster(&blk)
+         self.all.each { |m|
+           cluster = Service::ServiceCluster.find(m.service_cluster_id)
+           blk.call(cluster)
+         }
+       end
     end
 
+     ServiceClusterPool.initialize_table
+   end
 
-#     def send_agent_command(command, agent_id=nil)
-#       raise TypeError unless command.is_a? Packets::RequestBase
-#       EM.next_tick {
-#         if agent_id
-#           publish_to('agent_command', "agent_id.#{agent_id}", Marshal.dump(command))
-#         else
-#           publish_to('agent_command', '*', Marshal.dump(command))
-#         end
-#       }
-#     end
+   class Master
+     include Wakame::AMQPClient
+     include Wakame::QueueDeclare
 
-    def actor_request(agent_id, path, *args)
-      request = Wakame::Packets::ActorRequest.new(agent_id, Util.gen_id, path, *args)
-      ActorRequest.new(self, request)
+     define_queue 'agent_event', 'agent_event'
+     define_queue 'ping', 'ping'
+     define_queue 'registry', 'registry'
+
+     attr_reader :command_queue, :agent_monitor, :cluster_manager, :action_manager, :started_at
+     attr_reader :managers
+
+    def initialize(opts={})
+      pre_setup
     end
 
 
-    def attr
-      agent_monitor.master_local.attr
+    def actor_request(agent_id, path, *args)
+      request = Wakame::Packets::ActorRequest.new(agent_id, Util.gen_id, path, *args)
+      ActorRequest.new(self, request)
     end
 
 
     def cleanup
+      @managers.each { |m| m.terminate }
       @command_queue.shutdown
     end
 
-    def master_local_agent_id
-      @master_local_agent_id
+    def register_manager(manager)
+      raise ArgumentError unless manager.kind_of? Manager
+      manager.master = self
+      @managers << manager
+      manager
     end
 
-    private
-    def determine_agent_id
-      if Wakame.config.environment == :EC2
-        @master_local_agent_id = VmManipulator::EC2::MetadataService.query_metadata_uri('instance-id')
-      else
-        @master_local_agent_id = VmManipulator::StandAlone::INSTANCE_ID
-      end
+    # post_setup
+    def init
+      raise 'has to be put in EM.run context' unless EM.reactor_running?
+      @command_queue = register_manager(CommandQueue.new)
+
+      # WorkerThread has to run earlier than other managers.
+      @agent_monitor = register_manager(AgentMonitor.new)
+      @cluster_manager = register_manager(ClusterManager.new)
+      @action_manager = register_manager(ActionManager.new)
+
+      @managers.each {|m|
+        Wakame.log.debug("Initializing Manager Module: #{m.class}")
+        m.init
+      }
+
+      Wakame.log.info("Started master process : WAKAME_ROOT=#{Wakame.config.root_path} WAKAME_ENV=#{Wakame.config.environment}")
     end
 
+
+    private
     def pre_setup
-      determine_agent_id
       @started_at = Time.now
+      @managers = []
+
+      StatusDB::WorkerThread.init
 
-      EM.barrier {
+      StatusDB.pass {
         Wakame.log.debug("Binding thread info to EventDispatcher.")
         EventDispatcher.instance.bind_thread(Thread.current)
       }
     end
 
-    def post_setup
-      raise 'has to be put in EM.run context' unless EM.reactor_running?
-      @command_queue = CommandQueue.new(self)
-      @agent_monitor = AgentMonitor.new(self)
-
-      @service_cluster = Util.new_(Wakame.config.cluster_class, self)
-    end
 
   end
 
@@ -317,12 +435,12 @@ module Wakame
     def request
       raise "The request has already been sent." if @requested
 
-      @event_ticket = ED.subscribe(Event::ActorComplete) { |event|
+      @event_ticket = EventDispatcher.subscribe(Event::ActorComplete) { |event|
         if event.token == @packet.token
          
           # Any of status except RUNNING are accomplishment of the actor request.
           Wakame.log.debug("#{self.class}: The actor request has been completed: token=#{self.token}, status=#{event.status}, return_value=#{event.return_value}")
-          ED.unsubscribe(@event_ticket)
+          EventDispatcher.unsubscribe(@event_ticket)
           @return_value = event.return_value
           @wait_lock.enq([event.status, event.return_value])
         end
@@ -364,7 +482,7 @@ module Wakame
       }
     end
     alias :wait :wait_completion
-
+    
     private
     def check_requested?
       raise "The request has not been sent yet." unless @requested

data/lib/wakame/monitor/agent.rb

@@ -23,7 +23,11 @@ class Wakame::Monitor::Agent
 
 
   def check
-    attrs = Wakame::VmManipulator.create.fetch_local_attrs
+    if Wakame.config.environment == :EC2
+      attrs = Wakame::Agent.ec2_fetch_local_attrs
+    else
+      attrs = {}
+    end
 
     res = {:attrs=>attrs, :monitors=>[], :actors=>[], :services=>{}}
     EM.barrier {