wakame 0.4.0

data/lib/wakame/master.rb

@@ -0,0 +1,369 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+
+require 'wakame'
+require 'wakame/packets'
+require 'wakame/service'
+require 'wakame/queue_declare'
+require 'wakame/vm_manipulator'
+
+module Wakame
+
+  class AgentMonitor
+    include ThreadImmutable
+    attr_reader :registered_agents, :unregistered_agents, :master, :gc_period
+
+
+    def initialize(master)
+      bind_thread
+      @master = master
+      @registered_agents = {}
+      @unregistered_agents = {}
+      @agent_timeout = 31.to_f
+      @agent_kill_timeout = @agent_timeout * 2
+      @gc_period = 20.to_f
+
+      # GC event trigger for agent timer & status
+      calc_agent_timeout = proc {
+        #Wakame.log.debug("Started agent GC : agents.size=#{@registered_agents.size}")
+        kill_list=[]
+        registered_agents.each { |agent_id, agent|
+          next if agent.status == Service::Agent::STATUS_OFFLINE
+          diff_time = Time.now - agent.last_ping_at
+          #Wakame.log.debug "AgentMonitor GC : #{agent_id}: #{diff_time}"
+          if diff_time > @agent_timeout.to_f
+            agent.status = Service::Agent::STATUS_TIMEOUT
+          end
+          
+          if diff_time > @agent_kill_timeout.to_f
+            kill_list << agent_id
+          end
+        }
+        
+        kill_list.each { |agent_id|
+          agent = @agents.delete(agent_id)
+          ED.fire_event(Event::AgentUnMonitored.new(agent)) unless agent.nil?
+        }
+        #Wakame.log.debug("Finished agent GC")
+      }
+      
+      @agent_timeout_timer = EventMachine::PeriodicTimer.new(@gc_period, calc_agent_timeout)
+      
+      master.add_subscriber('registry') { |data|
+        data = eval(data)
+
+        agent_id = data[:agent_id]
+        case data[:type]
+        when 'Wakame::Packets::Register'
+          register_agent(data)
+        when 'Wakame::Packets::UnRegister'
+          unregister_agent(agent_id)
+        end
+      }
+
+      master.add_subscriber('ping') { |data|
+        ping = eval(data)
+        # Skip the old ping responses before starting master node.
+        next if Time.parse(ping[:responded_at]) < master.started_at
+
+        # Variable update function for the common members
+        set_report_values = proc { |agent|
+          agent.status = Service::Agent::STATUS_ONLINE
+          agent.uptime = 0
+          agent.last_ping_at = Time.parse(ping[:responded_at])
+          
+          agent.attr = ping[:attrs]
+          
+          agent.services.clear
+          ping.services.each { |svc_id, i|
+            agent.services[svc_id] = master.service_cluster.instances[svc_id]
+          }
+        }
+        
+        agent = agent(ping[:agent_id])
+        if agent.nil?
+          agent = Service::Agent.new(ping[:agent_id])
+          
+          set_report_values.call(agent)
+          
+          unregistered_agents[ping[:agent_id]]=agent
+        else
+          set_report_values.call(agent)
+        end
+        
+        
+        ED.fire_event(Event::AgentPong.new(agent))
+      }
+      
+      master.add_subscriber('agent_event') { |data|
+        response = eval(data)
+#p response
+        case response[:type]
+        when 'Wakame::Packets::ServiceStatusChanged'
+          svc_inst = Service::ServiceInstance.instance_collection[response[:svc_id]]
+          if svc_inst
+            response_time = Time.parse(response[:responded_at])
+            svc_inst.update_status(response[:new_status], response_time, response[:fail_message])
+            
+#             tmp_event = Event::ServiceStatusChanged.new(response[:svc_id], svc_inst.property, response[:status], response[:previous_status])
+#             tmp_event.time = response_time
+#             ED.fire_event(tmp_event)
+            
+#             if response[:previous_status] != Service::STATUS_ONLINE && response[:new_status] == Service::STATUS_ONLINE
+#               tmp_event = Event::ServiceOnline.new(tmp_event.instance_id, svc_inst.property)
+#               tmp_event.time = response_time
+#               ED.fire_event(tmp_event)
+#             elsif response[:previous_status] != Service::STATUS_OFFLINE && response[:new_status] == Service::STATUS_OFFLINE
+#               tmp_event = Event::ServiceOffline.new(tmp_event.instance_id, svc_inst.property)
+#               tmp_event.time = response_time
+#               ED.fire_event(tmp_event)
+#             elsif response[:previous_status] != Service::STATUS_FAIL && response[:new_status] == Service::STATUS_FAIL
+#               tmp_event = Event::ServiceFailed.new(tmp_event.instance_id, svc_inst.property, response[:fail_message])
+#               tmp_event.time = response_time
+#               ED.fire_event(tmp_event)
+#             end
+          end
+        when 'Wakame::Packets::ActorResponse'
+          case response[:status]
+          when Actor::STATUS_RUNNING
+              ED.fire_event(Event::ActorProgress.new(response[:agent_id], response[:token], 0))
+          else
+              ED.fire_event(Event::ActorComplete.new(response[:agent_id], response[:token], response[:status]))
+          end
+        else
+          Wakame.log.warn("#{self.class}: Unhandled agent response: #{response[:type]}")
+        end
+      }
+    
+    end
+
+
+    def agent(agent_id)
+      registered_agents[agent_id] || unregistered_agents[agent_id]
+    end
+
+    def register_agent(data)
+      agent_id = data[:agent_id]
+      agent = registered_agents[agent_id]
+      if agent.nil?
+        agent = unregistered_agents[agent_id]
+        if agent.nil?
+          # The agent is going to be registered at first time.
+          agent = Service::Agent.new(agent_id)
+          registered_agents[agent_id] = agent
+        else
+          # Move the reference from unregistered group to the registered group.
+          registered_agents[agent_id] = unregistered_agents[agent_id]
+          unregistered_agents.delete(agent_id)
+        end
+        Wakame.log.debug("The Agent has been registered: #{data.inspect}")
+        #Wakame.log.debug(unregistered_agents)
+        ED.fire_event(Event::AgentMonitored.new(agent))
+      end
+      agent.root_path = data[:root_path]
+    end
+
+    def unregister_agent(agent_id)
+      agent = registered_agents[agent_id]
+      if agent
+        unregistered_agents[agent_id] = registered_agents[agent_id]
+        registered_agents.delete(agent_id)
+        ED.fire_event(Event::AgentUnMonitored.new(agent))
+      end
+    end
+
+
+#     def bind_agent(service_instance, &filter)
+#       agent_id, agent = @agents.find { |agent_id, agent|
+
+#         next false if agent.has_service_type?(service_instance.property.class)
+#         filter.call(agent)
+#       }
+#       return nil if agent.nil?
+#       service_instance.bind_agent(agent)
+#       agent
+#     end
+
+#     def unbind_agent(service_instance)
+#       service_instance.unbind_agent
+#     end
+    
+    # Retruns the master local agent object
+    def master_local
+      agent = registered_agents[@master.master_local_agent_id]
+      puts "#{agent} = registered_agents[#{@master.master_local_agent_id}]"
+      raise "Master does not identify the master local agent yet." if agent.nil?
+      agent
+    end
+
+    def each_online(&blk)
+      registered_agents.each { |k, v|
+        next if v.status != Service::Agent::STATUS_ONLINE
+        blk.call(v)
+      }
+    end
+
+     def dump_status
+       ag = []
+       res = {:registered=>[], :unregistered=>[]}
+       
+       @registered_agents.each { |key, a|
+         res[:registered] << a.dump_status
+       }
+       @unregistered_agents.each { |key, a|
+         res[:unregistered] << a.dump_status
+       }
+       res
+     end
+  end
+
+  class Master
+    include Wakame::AMQPClient
+    include Wakame::QueueDeclare
+
+    define_queue 'agent_event', 'agent_event'
+    define_queue 'ping', 'ping'
+    define_queue 'registry', 'registry'
+
+    attr_reader :command_queue, :agent_monitor, :configuration, :service_cluster, :started_at
+
+    def initialize(opts={})
+      pre_setup
+
+      connect(opts) {
+        post_setup
+      }
+      Wakame.log.info("Started master process : WAKAME_ROOT=#{Wakame.config.root_path} WAKAME_ENV=#{Wakame.config.environment}")
+    end
+
+
+#     def send_agent_command(command, agent_id=nil)
+#       raise TypeError unless command.is_a? Packets::RequestBase
+#       EM.next_tick {
+#         if agent_id
+#           publish_to('agent_command', "agent_id.#{agent_id}", Marshal.dump(command))
+#         else
+#           publish_to('agent_command', '*', Marshal.dump(command))
+#         end
+#       }
+#     end
+
+    def actor_request(agent_id, path, *args)
+      request = Wakame::Packets::ActorRequest.new(agent_id, Util.gen_id, path, *args)
+      ActorRequest.new(self, request)
+    end
+
+
+    def attr
+      agent_monitor.master_local.attr
+    end
+
+
+    def cleanup
+      @command_queue.shutdown
+    end
+
+    def master_local_agent_id
+      @master_local_agent_id
+    end
+
+    private
+    def determine_agent_id
+      if Wakame.config.environment == :EC2
+        @master_local_agent_id = VmManipulator::EC2::MetadataService.query_metadata_uri('instance-id')
+      else
+        @master_local_agent_id = VmManipulator::StandAlone::INSTANCE_ID
+      end
+    end
+
+    def pre_setup
+      determine_agent_id
+      @started_at = Time.now
+
+      EM.barrier {
+        Wakame.log.debug("Binding thread info to EventDispatcher.")
+        EventDispatcher.instance.bind_thread(Thread.current)
+      }
+    end
+
+    def post_setup
+      raise 'has to be put in EM.run context' unless EM.reactor_running?
+      @command_queue = CommandQueue.new(self)
+      @agent_monitor = AgentMonitor.new(self)
+
+      @service_cluster = Util.new_(Wakame.config.cluster_class, self)
+    end
+
+  end
+
+
+  class ActorRequest
+    attr_reader :master
+
+    def initialize(master, packet)
+      raise TypeError unless packet.is_a?(Wakame::Packets::ActorRequest)
+
+      @master = master
+      @packet = packet
+      @requested = false
+      @event_ticket = nil
+      @wait_lock = ::Queue.new
+    end
+
+
+    def request
+      raise "The request has already been sent." if @requested
+
+      @event_ticket = ED.subscribe(Event::ActorComplete) { |event|
+       if event.token == @packet.token
+         
+         # Any of status except RUNNING are accomplishment of the actor request.
+         Wakame.log.debug("#{self.class}: The actor request has been completed: token=#{self.token}, status=#{event.status}")
+         ED.unsubscribe(@event_ticket)
+         @wait_lock.enq(event.status)
+       end
+      }
+      Wakame.log.debug("#{self.class}: Send the actor request: #{@packet.path}@#{@packet.agent_id}, token=#{self.token}")
+      master.publish_to('agent_command', "agent_id.#{@packet.agent_id}", @packet.marshal)
+      @requested = true
+      self
+    end
+
+
+    def token
+      @packet.token
+    end
+
+    def progress
+      check_requested?
+      raise NotImplementedError
+    end
+
+    def cancel
+      check_requested?
+      raise NotImplementedError
+      
+      #master.publish_to('agent_command', "agent_id.#{@packet.agent_id}", Wakame::Packets::ActorCancel.new(@packet.agent_id, ).marshal)
+      #ED.unsubscribe(@event_ticket)
+    end
+
+    def wait_completion(tout=60*30)
+      check_requested?
+      timeout(tout) {
+        Wakame.log.debug("#{self.class}: Waiting a response from the actor: #{@packet.path}@#{@packet.agent_id}, token=#{@packet.token}")
+        ret_status = @wait_lock.deq
+        Wakame.log.debug("#{self.class}: A response (status=#{ret_status}) back from the actor: #{@packet.path}@#{@packet.agent_id}, token=#{@packet.token}")
+        if ret_status == Actor::STATUS_FAILED
+          raise RuntimeError, "Failed status has been returned: Actor Request #{token}"
+        end
+      }
+    end
+    alias :wait :wait_completion
+
+    private
+    def check_requested?
+      raise "The request has not been sent yet." unless @requested
+    end
+  end
+end

data/lib/wakame/monitor/agent.rb

@@ -0,0 +1,50 @@
+
+class Wakame::Monitor::Agent
+  include Wakame::Monitor
+  
+  def initialize
+    @status = STATUS_ONLINE
+  end
+
+  def send_ping(hash)
+    publish_to('ping', Wakame::Packets::Ping.new(agent, hash[:attrs], hash[:actors], hash[:monitors], hash[:services]).marshal)
+  end
+
+  def setup(path)
+    # Send the first ping signal as soon as possible since the ping contanins vital information to construct the Agent object on master node.
+    send_ping(check())
+
+    # Setup periodical ping publisher.
+    @timer = CheckerTimer.new(10) {
+      send_ping(check())
+    }
+    @timer.start
+  end
+
+
+  def check
+    if Wakame.environment == :EC2
+      require 'wakame/vm_manipulator'
+      attrs = Wakame::VmManipulator::EC2::MetadataService.fetch_local_attrs
+    else
+      attrs = Wakame::VmManipulator::StandAlone.fetch_local_attrs
+    end
+
+    res = {:attrs=>attrs, :monitors=>[], :actors=>[], :services=>{}}
+    EM.barrier {
+      agent.monitor_registry.monitors.each { |key, m|
+        res[:monitors] << {:class=>m.class.to_s}
+      }
+      agent.actor_registry.actors.each { |key, a|
+        res[:actors] << {:class=>a.class.to_s}
+      }
+
+      svcmon = agent.monitor_registry.find_monitor('/service')
+      svcmon.checkers.each { |svc_id, a|
+        res[:services][svc_id]={:status=>a.status}
+      }
+    }
+
+    res
+  end
+end

data/lib/wakame/monitor/service.rb

@@ -0,0 +1,183 @@
+
+require 'open4'
+require 'wakame'
+
+class Wakame::Monitor::Service
+
+  class ServiceChecker
+    #include Wakame::Packets::Agent
+    attr_reader :timer, :svc_id
+    attr_accessor :last_checked_at, :status
+
+    def initialize(svc_id, svc_mon)
+      @svc_id = svc_id
+      @service_monitor = svc_mon
+      @status = Wakame::Service::STATUS_OFFLINE
+      count = 0
+      @timer = Wakame::Monitor::CheckerTimer.new(3) {
+        self.signal_checker
+      }
+    end
+
+    def start
+      if !@timer.running?
+        @timer.start
+        @service_monitor.send_event(Wakame::Packets::MonitoringStarted.new(@service_monitor.agent, self.svc_id))
+        Wakame.log.debug("#{self.class}: Started the checker")
+      end
+    end
+
+    def stop
+      if @timer.running?
+        @timer.stop
+        @service_monitor.send_event(Wakame::Packets::MonitoringStopped.new(@service_monitor.agent, self.svc_id))
+        Wakame.log.debug("#{self.class}: Stopped the checker")
+      end
+    end
+
+    def check
+    end
+
+    protected
+    def signal_checker
+      EventMachine.defer proc {
+        res = begin
+                self.last_checked_at = Time.now
+                res = self.check
+                res 
+              rescue => e
+                Wakame.log.error("#{self.class}: #{e}")
+                Wakame.log.error(e)
+                e
+              end
+        Thread.pass
+        res
+      }, proc { |res|
+
+        case res
+        when Exception
+          update_status(Wakame::Service::STATUS_FAIL) 
+        when Wakame::Service::STATUS_ONLINE, Wakame::Service::STATUS_OFFLINE
+          update_status(res) 
+        else
+          Wakame.log.error("#{self.class}: Unknown response type from the checker: #{self.svc_id}, ")
+        end
+      }
+    end
+
+    def update_status(new_status)
+      prev_status = self.status
+      if prev_status != new_status
+        self.status = new_status
+        @service_monitor.send_event(Wakame::Packets::ServiceStatusChanged.new(@service_monitor.agent, self.svc_id, prev_status, new_status))
+      end
+    end
+  end
+
+  class PidFileChecker < ServiceChecker
+    def initialize(svc_id, svc_mon, pidpath)
+      super(svc_id, svc_mon)
+      @pidpath = pidpath
+    end
+    
+    def check
+      return Wakame::Service::STATUS_OFFLINE unless File.exist?(@pidpath)
+      #cmdstat = ::Open4.popen4("ps -p \"`cat '#{@pidpath}'`\" > /dev/null"){}
+      #cmdstat.exitstatus == 0 ? Wakame::Service::STATUS_ONLINE : Wakame::Service::STATUS_OFFLINE
+
+      cmdres = system("ps -p \"`cat '#{@pidpath}'`\" > /dev/null")
+      # system() returns true or false.
+      cmdres ? Wakame::Service::STATUS_ONLINE : Wakame::Service::STATUS_OFFLINE
+    end
+  end
+
+  class CommandChecker < ServiceChecker
+    attr_reader :command
+
+    def initialize(svc_id, svc_mon, cmdstr)
+      super(svc_id, svc_mon)
+      @command = cmdstr
+    end
+
+    def check()
+      outputs =[]
+      cmdstat = ::Open4.popen4(@command) { |pid, stdin, stdout, stderr|
+        stdout.each { |l|
+          outputs << l
+        }
+        stderr.each { |l|
+          outputs << l
+        }
+      }
+      Wakame.log.debug("#{self.class}: Exit Status #{@command}: #{cmdstat}")
+      if outputs.size > 0
+        @service_monitor.send_event(Wakame::Packets::MonitoringOutput.new(@service_monitor.agent, self.svc_id, outputs.join('')))
+      end
+      cmdstat.exitstatus == 0 ? Wakame::Service::STATUS_ONLINE : Wakame::Service::STATUS_OFFLINE
+    end
+  end
+
+  include Wakame::Monitor
+
+  attr_reader :checkers
+
+  def initialize
+    @status = Wakame::Service::STATUS_ONLINE
+    @checkers = {}
+  end
+
+  def setup(path)
+  end
+
+  def handle_request(request)
+    svc_id = request[:svc_id]
+    case request[:command]
+    when :start
+      register(svc_id, request[:cmdstr])
+    when :stop
+      unregister(svc_id)
+    end
+  end
+
+  def send_event(a)
+    Wakame.log.debug("Sending back a event: #{a.class}")
+    publish_to('agent_event', a.marshal)
+  end
+
+  def dump_attrs
+    
+  end
+
+  def find_checker(svc_id)
+    @checkers[svc_id]
+  end
+
+  def register(svc_id, checker_type, *args)
+    chk = @checkers[svc_id]
+    if chk
+      Wakame.log.error("#{self.class}: Service registory duplication. #{svc_id}")
+      raise "Service registory duplication. #{svc_id}"
+    end
+    case checker_type.to_sym
+    when :pidfile
+      chk = PidFileChecker.new(svc_id, self, args[0])
+    when :command
+      chk = CommandChecker.new(svc_id, self, args[0])
+    else
+      raise "Unsupported checker type: #{checker_type}"
+    end
+    chk.start
+    @checkers[svc_id]=chk
+    Wakame.log.info("#{self.class}: Registered service checker for #{svc_id}")
+  end
+
+  def unregister(svc_id)
+    chk = @checkers[svc_id]
+    if chk
+      chk.timer.stop
+      @checkers.delete(svc_id)
+      Wakame.log.info("#{self.class}: Unregistered service checker for #{svc_id}")
+    end
+  end
+
+end

data/lib/wakame/monitor.rb

@@ -0,0 +1,69 @@
+
+
+
+module Wakame
+  module Monitor
+    STATUS_OFFLINE=0
+    STATUS_ONLINE=1
+    STATUS_FAIL=2
+
+    def self.included(klass)
+      klass.class_eval {
+        attr_accessor :status, :agent
+
+      }
+    end
+
+    def handle_request(request)
+    end
+
+    def setup(assigned_path)
+    end
+
+    def enable
+    end
+
+    def disable
+    end
+
+    def publish_to(exchange, data)
+      agent.publish_to(exchange, data)
+    end
+
+  end
+end
+
+
+module Wakame
+  module Monitor
+    class CheckerTimer < EventMachine::PeriodicTimer
+      def initialize(time, &blk)
+        @interval = time
+        @code = proc {
+          begin
+            blk.call
+          rescue => e
+            Wakame.log.error(e)
+          end
+        }
+        stop
+      end
+
+      def start
+        if !running?
+          @cancelled = false
+          schedule
+        end
+      end
+
+      def stop
+        @cancelled = true
+      end
+
+      def running?
+        !@cancelled 
+      end
+
+    end
+  end
+end