wakame 0.5.0 → 0.5.1

data/lib/wakame/runner/agent.rb

@@ -66,6 +66,8 @@ module Wakame
           daemonize(@options[:log_file])
         end
 
+        Initializer.run(:process_agent)
+
         EM.epoll if Wakame.config.eventmachine_use_epoll
         EM.run {
           Wakame::Agent.start(opts)

data/lib/wakame/runner/master.rb

@@ -66,7 +66,8 @@ module Wakame
         end
 
         change_privilege(@options[:uid]) if @options[:uid]
-        
+
+        Initializer.run(:process_master)
 
         EM.epoll if Wakame.config.eventmachine_use_epoll
         EM.run {

data/lib/wakame/service.rb

@@ -24,6 +24,7 @@ module Wakame
     STATUS_MIGRATING = 7
     STATUS_ENTERING = 8
     STATUS_QUITTING = 9
+    STATUS_RUNNING = 10
 
     # Data model for agent
     # Status life cycle:
@@ -35,11 +36,15 @@ module Wakame
       STATUS_ONLINE  = 1
       STATUS_UNKNOWN = 2
       STATUS_TIMEOUT = 3
+      STATUS_RUNNING = 4
+      STATUS_REGISTERRING = 5
+      STATUS_TERMINATING = 6
       
       property :last_ping_at
       property :vm_attr
       property :root_path
       property :status, {:read_only=>true, :default=>STATUS_INIT}
+      property :monitor_status, {:read_only=>true, :default=>STATUS_INIT}
       property :reported_services, {:read_only=>true, :default=>{}}
       property :cloud_host_id
 
@@ -56,11 +61,15 @@ module Wakame
       end
 
       def agent_ip
-        vm_attr[:local_ipv4]
+        vm_attr[:private_dns_name]
       end
 
       def vm_id
-        vm_attr[:instance_id]
+        vm_attr[:aws_instance_id]
+      end
+
+      def cloud_host
+        self.cloud_host_id.nil? ? nil : CloudHost.find(self.cloud_host_id)
       end
 
       # Tentative...
@@ -73,6 +82,13 @@ module Wakame
         if @status != new_status
           @status = new_status
           self.save
+        end
+      end
+
+      def update_monitor_status(new_status)
+        if @monitor_status != new_status
+          @monitor_status = new_status
+          self.save
 
           ED.fire_event(Event::AgentStatusChanged.new(self))
           # Send status specific event
@@ -83,6 +99,9 @@ module Wakame
         end
       end
 
+      def actor_request(path, *args)
+        Master.instance.actor_request(self.id, path, *args)
+      end
 
       def renew_reported_services(svc_id_list)
         reported_services.clear
@@ -101,120 +120,22 @@ module Wakame
       end
 
       def terminate
-      end
-    end
-
-    class AgentPool < StatusDB::Model
-      property :group_observed, {:default=>{}}
-      property :group_active, {:default=>{}}
-
-      ID=self.to_s
-
-      def self.instance
-        a = self.find(ID)
-        if a.nil?
-          a = self.new
-          a.save
-        end
-        a
-      end
-
-      def self.reset
-        ap = self.instance
-        
-        nil_cloud_host_id = lambda { |agent_id|
-          agent = Agent.find(agent_id)
-          agent.cloud_host_id = nil
-          agent.save
-        }
-
-        ap.group_observed.keys.each &nil_cloud_host_id
-        ap.group_active.keys.each &nil_cloud_host_id
-        ap.save
-      end
-
-
-      def id
-        ID
-      end
-
-      def create_or_find(agent_id)
-        agent = Service::Agent.find(agent_id)
-        if agent.nil?
-          agent = Service::Agent.new
-          agent.id = agent_id
-          Wakame.log.debug("#{self.class}: Created new agent object with Agent ID: #{agent_id}")
-        end
-        agent
-      end
-
-      def register_as_observed(agent)
-        agent_id = agent.id
-        if group_observed.has_key?(agent_id)
-        else
-          if group_active.has_key?(agent_id)
-            # Move the reference from unregistered group to the registered group.
-            group_active.delete(agent_id)
-            group_observed[agent_id]=1
-          else
-            # The agent is going to be registered at first time.
-            group_observed[agent_id]=1
-          end
-
-          self.save
-          Wakame.log.debug("#{self.class}: Register agent to observed group: #{agent_id}")
-        end
-      end
-
-      def register(agent)
-        raise ArgumentError unless agent.is_a?(Agent)
-        agent_id = agent.id
-        if group_active.has_key?(agent_id)
-        else
-          if group_observed.has_key?(agent_id)
-            # Move the reference from unregistered group to the registered group.
-            group_observed.delete(agent_id)
-            group_active[agent_id]=1
-          else
-            # The agent is going to be registered at first time.
-            group_active[agent_id]=1
-          end
-
-          self.save
-
-          Wakame.log.debug("#{self.class}: Register agent to active group: #{agent_id}")
-          ED.fire_event(Event::AgentMonitored.new(agent))
+        if mapped?
+          raise "#{self.class}[#{self.id}]: CloudHost is assigned live services." if cloud_host.assigned_services.all? {|svc_id| ServiceInstance.find(svc_id).status == Service::STATUS_TERMINATE }
+          CloudHost.delete(cloud_host_id)
         end
+        self.delete
       end
 
-      def unregister(agent)
-        raise ArgumentError unless agent.is_a?(Agent)
-        agent_id = agent.id
-        
-        unregistered = false
-        if group_active.has_key?(agent_id)
-          group_active.delete(agent_id)
-          unregistered = true
-        end
-
-        if group_observed.has_key?(agent_id)
-          group_observed.delete(agent_id)
-          unregistered = true
+      def update_vm_attr
+        if Wakame.config.environment == :EC2
+          require 'right_aws'
+          ec2 = RightAws::Ec2.new(Wakame.config.aws_access_key, Wakame.config.aws_secret_key)
+          dat = ec2.describe_instances([self.id])
+          @vm_attr = dat[0]
         end
-
-        if unregistered
-          self.save
-          
-          Wakame.log.debug("#{self.class}: Unregister agent: #{agent.id}")
-          ED.fire_event(Event::AgentUnMonitored.new(agent))
-        end
-      end
-
-      def find_agent(agent_id)
-        raise "The agent ID \"#{agent_id}\" is not registered in the pool" unless group_active.has_key? agent_id
-        Agent.find(agent_id) || raise("The agent ID #{agent_id} is registered. but not in the database.")
+        self.save
       end
-
     end
 
     class CloudHost < StatusDB::Model
@@ -232,14 +153,14 @@ module Wakame
         
       def map_agent(agent)
         raise ArgumentError unless agent.is_a?(Agent)
-        raise "Ensure to call unmap_agent() prior to mapping new agent" if self.mapped?
-        raise "The agent \"#{agent.id}\" is already mapped to the cloud host: #{agent.cloud_host_id}" if agent.mapped?
+        raise "Ensure to call unmap_agent() prior to mapping new agent: #{self.agent_id}" if self.mapped?
 
         self.agent_id = agent.id
         agent.cloud_host_id = self.id
 
         self.save
         agent.save
+        ED.fire_event(Event::AgentMappedCloudHost.new(self.id, agent.id))
       end
 
       def unmap_agent()
@@ -252,6 +173,7 @@ module Wakame
 
           self.agent_id = nil
           self.save
+          ED.fire_event(Event::AgentUnmappedCloudHost.new(self.id, agent.id))
         end
       end
 
@@ -266,6 +188,35 @@ module Wakame
         spec
       end
 
+      # Combine all the monitorring data from associated Resources.
+      # Example of return value:
+      # { '/service' => {
+      #     'svcid1' => {:type=>:pidfile, :path=>'/var/run/pid1.pid'},
+      #     'svcid2' => {:type=>:command, :cmdline=>'ps -ef | grep key'}
+      #   }
+      # }
+      def live_monitors(live_stats=[Service::STATUS_ENTERING, Service::STATUS_STARTING, Service::STATUS_RUNNING])
+        mon = {}
+        c = ServiceCluster.find(cluster_id)
+        c.services.keys.each { |svc_id|
+          svc = ServiceInstance.find(svc_id)
+          next unless svc.cloud_host_id == self.id
+          next unless live_stats.member?(svc.status)
+
+          svc.resource.monitors.each { |path, data|
+
+            data.dup.each { |k, v|
+              if v.is_a? String
+                data[k] = instance_eval('"' + v.gsub(/%\{/, '#{') + '"')
+              end
+            }
+
+            mon[path] ||= {}
+            mon[path][svc_id] = data
+          }
+        }
+        mon
+      end
 
       # Delegate methods for Agent class
 
@@ -333,10 +284,13 @@ module Wakame
       STATUS_OFFLINE = 0
       STATUS_ONLINE = 1
       STATUS_PARTIAL_ONLINE = 2
+      STATUS_FROZEN = 3
+      STATUS_UNFROZEN = 4
 
       property :name
       property :status, {:readonly=>true, :default=>STATUS_OFFLINE}
       property :status_changed_at, {:readonly=>true, :default=>proc{Time.now} }
+      property :freeze_status, {:readonly=>true, :default=>STATUS_UNFROZEN}
       property :services, {:default=>{}}
       property :resources, {:default=>{}}
       property :cloud_hosts, {:default=>{}}
@@ -368,6 +322,7 @@ module Wakame
       end
 
       def reset
+        check_freeze
         services.clear
         resources.clear
         cloud_hosts.clear
@@ -407,6 +362,7 @@ module Wakame
 
       # 
       def add_resource(resource, &blk)
+        check_freeze
         if resource.is_a?(Class) && resource <= Resource
           resource = resource.new
         elsif resource.is_a? Resource
@@ -429,6 +385,7 @@ module Wakame
 
       # Set dependency between two resources.
       def set_dependency(res_name1, res_name2)
+        check_freeze
         validate_arg = proc {|o|
           o = Util.build_const(o) if o.is_a? String
           raise ArgumentError unless o.is_a?(Class) && o <= Resource
@@ -470,6 +427,7 @@ module Wakame
       #thread_immutable_methods :launch
 
       def destroy(svc_id)
+        check_freeze
         raise("Unknown service instance : #{svc_id}") unless self.services.has_key?(svc_id)
         svc = ServiceInstance.find(svc_id)
         svc.unbind_cluster
@@ -484,12 +442,15 @@ module Wakame
 
         svc.delete
         self.save
+
+        ED.fire_event(Event::ServiceDestroied.new(svc_id))
       end
       thread_immutable_methods :destroy
 
 
       #def propagate(resource, force=false)
       def propagate_resource(resource, cloud_host_id=nil, force=false)
+        check_freeze
         res_id = Resource.id(resource)
         res_obj = (self.resources.has_key?(res_id) && Resource.find(res_id)) || raise("Unregistered resource: #{resource.to_s}")
         raise ArgumentError if res_obj.require_agent && cloud_host_id.nil?
@@ -516,12 +477,14 @@ module Wakame
         svc.save
         self.save
 
+        ED.fire_event(Event::ServicePropagated.new(svc.id))
         svc
       end
       thread_immutable_methods :propagate_resource
       alias :propagate :propagate_resource
 
       def propagate_service(svc_id, cloud_host_id=nil, force=false)
+        check_freeze
         src_svc = (self.services.has_key?(svc_id) && ServiceInstance.find(svc_id)) || raise("Unregistered service: #{svc_id.to_s}")
         res_obj = src_svc.resource
 
@@ -552,11 +515,13 @@ module Wakame
         svc.save
         self.save
 
+        ED.fire_event(Event::ServicePropagated.new(svc.id))
         svc
       end
       thread_immutable_methods :propagate_service
 
       def add_cloud_host(&blk)
+        check_freeze
         h = CloudHost.new
         h.cluster_id = self.id
         self.cloud_hosts[h.id]=1
@@ -570,13 +535,19 @@ module Wakame
       end
 
       def remove_cloud_host(cloud_host_id)
+        check_freeze
         if self.cloud_hosts.has_key?(cloud_host_id)
           self.cloud_hosts.delete(cloud_host_id)
         
           self.save
         end
 
-        CloudHost.delete(cloud_host_id) rescue nil
+        begin
+          ch = CloudHost.find(cloud_host_id)
+          ch.unmap_agent
+          ch.delete
+        rescue => e
+        end
       end
 
       def find_service(svc_id)
@@ -651,17 +622,15 @@ module Wakame
 
       alias :instances :services
 
-      #private
-
       def update_cluster_status
         onlines = []
         all_offline = false
 
         onlines = self.each_instance.select { |i|
-          i.status == Service::STATUS_ONLINE
+          i.monitor_status == Service::STATUS_ONLINE
         }
         all_offline = self.each_instance.all? { |i|
-          i.status == Service::STATUS_OFFLINE
+          i.monitor_status == Service::STATUS_OFFLINE
         }
         #Wakame.log.debug "online instances: #{onlines.size}, assigned instances: #{self.instances.size}"
 
@@ -677,6 +646,28 @@ module Wakame
       end
       thread_immutable_methods :update_cluster_status
 
+
+      def update_freeze_status(new_status)
+        raise "Invalid freeze status: #{new_status}" unless [STATUS_FROZEN, STATUS_UNFROZEN].member?(new_status)
+        if @freeze_status != new_status
+          @freeze_status = new_status
+
+          self.save
+
+          case @freeze_status 
+          when STATUS_FROZEN
+            ED.fire_event(Event::ClusterFrozen.new(id))
+          when STATUS_UNFROZEN
+            ED.fire_event(Event::ClusterUnfrozen.new(id))
+          end
+        end
+      end
+
+      private
+      def check_freeze
+        raise "#{self.class}: The cluster #{self.name} is in freeze mode." if freeze_status == STATUS_FROZEN
+      end
+      
     end
 
     
@@ -776,30 +767,47 @@ module Wakame
       property :resource_id
       property :cluster_id
       property :status, {:read_only=>true, :default=>Service::STATUS_INIT}
-      property :monitor_status, {:default=>Service::STATUS_INIT}
-      property :status_changed_at, {:read_only=>true, :default=>proc{Time.now}}
+      property :monitor_status, {:default=>Service::STATUS_UNKNOWN}
+      property :status_changed_at, {:read_only=>true, :default=>Time.at(0)}
+      property :monitor_status_changed_at, {:read_only=>true, :default=>Time.at(0)}
 
-      def update_status(new_status, changed_at=Time.now, fail_message=nil)
+      def update_status(new_status)
         if @status != new_status
           prev_status = @status
           @status = new_status
-          @status_changed_at = changed_at
+          @status_changed_at = Time.now
+          
+          self.save
+
+          ED.fire_event(Event::ServiceStatusChanged.new(self.id, new_status, prev_status))
+        end
+      end
+
+      def update_monitor_status(new_status, changed_at=Time.now, fail_message=nil)
+        raise "Invalid status has been assigned: #{new_status}" unless [STATUS_UNKNOWN, STATUS_ONLINE, STATUS_OFFLINE, STATUS_FAIL].member?(new_status)
+
+        if @monitor_status != new_status
+
+          Wakame.log.debug("#{self.class}: #{self.id} update_monitor_status #{new_status}, #{@monitor_status}")
+          prev_status = @monitor_status
+          @monitor_status = new_status
+          @monitor_status_changed_at = changed_at
           
           self.save
 
-          event = Event::ServiceStatusChanged.new(self.id, resource, new_status, prev_status)
-          event.time = @status_changed_at.dup
+          event = Event::ServiceMonitorStatusChanged.new(self.id, new_status, prev_status)
+          event.time = changed_at.dup
           ED.fire_event(event)
 
           tmp_event = nil
           if prev_status != Service::STATUS_ONLINE && new_status == Service::STATUS_ONLINE
-            tmp_event = Event::ServiceOnline.new(self.id, self.resource)
+            tmp_event = Event::ServiceOnline.new(self.id)
             tmp_event.time = @status_changed_at.dup
           elsif prev_status != Service::STATUS_OFFLINE && new_status == Service::STATUS_OFFLINE
-            tmp_event = Event::ServiceOffline.new(self.id, self.resource)
+            tmp_event = Event::ServiceOffline.new(self.id)
             tmp_event.time = @status_changed_at.dup
           elsif prev_status != Service::STATUS_FAIL && new_status == Service::STATUS_FAIL
-            tmp_event = Event::ServiceFailed.new(self.id, self.resource, fail_message)
+            tmp_event = Event::ServiceFailed.new(self.id, fail_message)
             tmp_event.time = @status_changed_at.dup
           end
           ED.fire_event(tmp_event) if tmp_event
@@ -822,7 +830,7 @@ module Wakame
         self.cloud_host_id = new_cloud_host.id
         self.save
         
-        ED.fire_event(Event::ServiceBoundHost.new(self, new_cloud_host))
+        ED.fire_event(Event::ServiceBoundCloudHost.new(self.id, new_cloud_host.id))
       end
       thread_immutable_methods :bind_cloud_host
       
@@ -834,7 +842,7 @@ module Wakame
         
         self.save
 
-        ED.fire_event(Event::ServiceUnboundHost.new(self, old_item))
+        ED.fire_event(Event::ServiceUnboundCloudHost.new(self.id, old_item.id))
         old_item
       end
       thread_immutable_methods :unbind_cloud_host
@@ -1011,7 +1019,7 @@ module Wakame
 
         def satisfy?(vm_attr, diff)
           # Compare critical variables which will return false if they are not same.
-          return false unless [:availability_zone, :instance_type, :image_id].all? { |k| diff[k].nil? ? true : diff[k] == vm_attr[k] }
+          return false unless [:availability_zone, :instance_type, :image_id].all? { |k| diff[k].nil? ? true : diff[k] == vm_attr[self.class.vm_attr_defs[k][:right_aws_key]] }
           true
         end
 
@@ -1048,6 +1056,8 @@ module Wakame
       property :max_instances, {:default=>1}
       property :startup, {:default=>true}
       property :require_agent, {:default=>true}
+      property :tags, {:read_only=>true, :default=>[]}
+      property :monitors, {:default=>{}}
 
       def self.inherited(klass)
         klass.class_eval {