wakame 0.4.0

data/contrib/imagesetup.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+[ $UID -eq 0 ] || { echo "Run with root user."; exit 1; }
+
+apt-get update;
+apt-get -y upgrade;
+
+apt-get -y install apache2-mpm-prefork libapache2-mod-rpaf
+apt-get -y install mysql-server mysql-client
+apt-get -y install erlang-nox
+apt-get -y install unzip zip rsync libopenssl-ruby libhmac-ruby rubygems irb ri rdoc sysstat
+
+apt-get clean
+
+(cd /tmp;
+wget http://www.rabbitmq.com/releases/rabbitmq-server/v1.5.4/rabbitmq-server_1.5.4-1_all.deb;
+dpkg -i rabbitmq-server_1.5.4-1_all.deb;
+)
+
+
+if ! getent group wakame >/dev/null; then
+    addgroup --system wakame
+fi
+
+if ! getent passwd wakame >/dev/null; then
+    adduser --system --ingroup wakame --disabled-password wakame
+fi
+
+mkdir /home/wakame/config
+chown wakame:wakame /home/wakame/config
+mkdir /home/wakame/mysql
+mkdir /home/wakame/mysql/data
+mkdir /home/wakame/mysql/data-slave
+chown mysql:mysql /home/wakame/mysql/data /home/wakame/mysql/data-slave
+
+if ! grep GEM_HOME /etc/environemnt >/dev/null; then
+    cat <<EOF > /etc/environment
+GEM_HOME=/usr/local/gems
+EOF
+fi
+
+cat <<EOF > /etc/default/wakame
+WAKAME_HOME=/home/wakame/corelib
+GEM_HOME=/usr/local/gems
+EOF
+
+update-rc.d -f apache2 remove
+update-rc.d -f mysql remove
+update-rc.d -f mysql-ndb remove
+update-rc.d -f mysql-ndb-mgm remove
+# Disable apparmor
+update-rc.d -f apparmor remove
+
+cat <<EOF > /usr/local/bin/passenger_ruby.sh
+#!/bin/sh
+export GEM_PATH="/usr/local/gems"
+exec /usr/bin/ruby $@
+EOF
+chmod 755 /usr/local/bin/passenger_ruby.sh
+
+# Create root ssh key
+ssh-keygen -t rsa -N '' -f /home/wakame/config/root.id_rsa
+chown wakame:wakame /home/wakame/config/root.id_rsa /home/wakame/config/root.id_rsa.pub
+cat /home/wakame/config/root.id_rsa.pub >> /root/.ssh/authorized_keys
+
+#gem install rake rails eventmachine amqp log4r daemons passenger hoe amazon-ec2 --no-ri --no-rdoc
+#(cd $GEM_HOME/gems/passenger-*; rake)
+
+#update-rc.d wakame-master defaults 41
+#update-rc.d wakame-agent defaults 40
+
+# 
+# /root/.ssh/authorized_keys /root/.bash_history
+# /home/ubuntu/.ssh/authorized_keys /home/ubuntu/.bash_history
+# Clear logs
+# logrotate -f /etc/logrotate.conf
+# rm -f /var/log/apache2/* /var/log/rabbitmq/* /var/log/wakame-* /var/log/*.gz /var/log/*.0 /var/log/*.1

data/lib/ext/eventmachine.rb

@@ -0,0 +1,86 @@
+
+require 'thread'
+
+module EventMachine
+#   def self::defer op, callback = nil
+#     @need_threadqueue ||= 0
+#     if @need_threadqueue == 0
+#       @need_threadqueue = 1
+#       require 'thread'
+#       @threadqueue = Queue.new
+#       @resultqueue = Queue.new
+#       @thread_g = ThreadGroup.new
+#       20.times {|ix|
+#         t = Thread.new {
+#           my_ix = ix
+#           loop {
+#             op,cback = @threadqueue.pop
+#             begin
+#               result = op.call
+#               @resultqueue << [result, cback]
+#             rescue => e
+#               puts "#{e} in EM defer thread pool : #{Thread.current}"
+#               raise e
+#             ensure
+#               EventMachine.signal_loopbreak
+#             end
+#           }
+#         }
+#         @thread_g.add(t)
+#       }
+#     end
+#
+#     @threadqueue << [op,callback]
+#   end
+
+  # Redefine EM's threadpool
+  def self.spawn_threadpool
+    until @threadpool.size == 20
+      thread = Thread.new {
+        loop {
+          op, cback = *@threadqueue.pop
+          begin
+            result = op.call
+            @resultqueue << [result, cback]
+          rescue => e
+            puts "#{e} in EM defer thread pool : #{Thread.current}"
+          ensure
+            EventMachine.signal_loopbreak
+          end
+        }
+      }
+      @threadpool << thread
+    end
+  end
+  
+  def self.barrier(&blk)
+    # Presumably, Thread.main will return the EM main loop thread.
+    if EM.reactor_thread?
+      return blk.call
+    end
+
+    raise "Eventmachine is not ready to accept the next_tick() call." unless self.reactor_running?
+
+    @q ||= ::Queue.new
+    time_start = ::Time.now
+
+    self.next_tick {
+    #self.add_timer(0) {
+      begin
+        res = blk.call
+        @q << [true, res]
+      rescue => e
+        @q << [false, e]
+      end
+    }
+    
+    res = @q.shift
+    time_elapsed = ::Time.now - time_start
+    Wakame.log.debug("EM.barrier: elapsed time for #{blk}: #{time_elapsed} sec (#{$eventmachine_library})") if time_elapsed > 0.05
+    if res[0] == false && res[1].is_a?(Exception)
+      raise res[1]
+    end
+    res[1]
+  end
+
+end

data/lib/ext/shellwords.rb

@@ -0,0 +1,172 @@
+#
+# shellwords.rb: Manipulates strings a la UNIX Bourne shell
+#
+
+#
+# This module manipulates strings according to the word parsing rules
+# of the UNIX Bourne shell.
+#
+# The shellwords() function was originally a port of shellwords.pl,
+# but modified to conform to POSIX / SUSv3 (IEEE Std 1003.1-2001).
+#
+# Authors:
+#   - Wakou Aoyama
+#   - Akinori MUSHA <knu@iDaemons.org>
+#
+# Contact:
+#   - Akinori MUSHA <knu@iDaemons.org> (current maintainer)
+#
+module Shellwords
+  #
+  # Splits a string into an array of tokens in the same way the UNIX
+  # Bourne shell does.
+  #
+  #   argv = Shellwords.split('here are "two words"')
+  #   argv #=> ["here", "are", "two words"]
+  #
+  # +String#shellsplit+ is a shorthand for this function.
+  #
+  #   argv = 'here are "two words"'.shellsplit
+  #   argv #=> ["here", "are", "two words"]
+  #
+  def shellsplit(line)
+    line = String.new(line) rescue
+      raise(ArgumentError, "Argument must be a string")
+    line.lstrip!
+    words = []
+    until line.empty?
+      field = ''
+      loop do
+	if line.sub!(/\A"(([^"\\]|\\.)*)"/, '') then
+	  snippet = $1.gsub(/\\(.)/, '\1')
+	elsif line =~ /\A"/ then
+	  raise ArgumentError, "Unmatched double quote: #{line}"
+	elsif line.sub!(/\A'([^']*)'/, '') then
+	  snippet = $1
+	elsif line =~ /\A'/ then
+	  raise ArgumentError, "Unmatched single quote: #{line}"
+	elsif line.sub!(/\A\\(.)?/, '') then
+	  snippet = $1 || '\\'
+	elsif line.sub!(/\A([^\s\\'"]+)/, '') then
+	  snippet = $1
+	else
+	  line.lstrip!
+	  break
+	end
+	field.concat(snippet)
+      end
+      words.push(field)
+    end
+    words
+  end
+
+  alias shellwords shellsplit
+
+  module_function :shellsplit, :shellwords
+
+  class << self
+    alias split shellsplit
+  end
+
+  #
+  # Escapes a string so that it can be safely used in a Bourne shell
+  # command line.
+  #
+  # Note that a resulted string should be used unquoted and is not
+  # intended for use in double quotes nor in single quotes.
+  #
+  #   open("| grep #{Shellwords.escape(pattern)} file") { |pipe|
+  #     # ...
+  #   }
+  #
+  # +String#shellescape+ is a shorthand for this function.
+  #
+  #   open("| grep #{pattern.shellescape} file") { |pipe|
+  #     # ...
+  #   }
+  #
+  def shellescape(str)
+    # An empty argument will be skipped, so return empty quotes.
+    return "''" if str.empty?
+
+    str = str.dup
+
+    # Process as a single byte sequence because not all shell
+    # implementations are multibyte aware.
+    str.gsub!(/([^A-Za-z0-9_\-.,:\/@\n])/n, "\\\\\\1")
+
+    # A LF cannot be escaped with a backslash because a backslash + LF
+    # combo is regarded as line continuation and simply ignored.
+    str.gsub!(/\n/, "'\n'")
+
+    return str
+  end
+
+  module_function :shellescape
+
+  class << self
+    alias escape shellescape
+  end
+
+  #
+  # Builds a command line string from an argument list +array+ joining
+  # all elements escaped for Bourne shell and separated by a space.
+  #
+  #   open('|' + Shellwords.join(['grep', pattern, *files])) { |pipe|
+  #     # ...
+  #   }
+  #
+  # +Array#shelljoin+ is a shorthand for this function.
+  #
+  #   open('|' + ['grep', pattern, *files].shelljoin) { |pipe|
+  #     # ...
+  #   }
+  #
+  def shelljoin(array)
+    array.map { |arg| shellescape(arg) }.join(' ')
+  end
+
+  module_function :shelljoin
+
+  class << self
+    alias join shelljoin
+  end
+end
+
+class String
+  #
+  # call-seq:
+  #   str.shellsplit => array
+  #
+  # Splits +str+ into an array of tokens in the same way the UNIX
+  # Bourne shell does.  See +Shellwords::shellsplit+ for details.
+  #
+  def shellsplit
+    Shellwords.split(self)
+  end
+
+  #
+  # call-seq:
+  #   str.shellescape => string
+  #
+  # Escapes +str+ so that it can be safely used in a Bourne shell
+  # command line.  See +Shellwords::shellescape+ for details.
+  #
+  def shellescape
+    Shellwords.escape(self)
+  end
+end
+
+class Array
+  #
+  # call-seq:
+  #   array.shelljoin => string
+  #
+  # Builds a command line string from an argument list +array+ joining
+  # all elements escaped for Bourne shell and separated by a space.
+  # See +Shellwords::shelljoin+ for details.
+  #
+  def shelljoin
+    Shellwords.join(self)
+  end
+end

data/lib/ext/uri.rb

@@ -0,0 +1,15 @@
+require 'uri/generic'
+
+module URI
+  class AMQP < Generic
+    COMPONENT = [
+      :scheme,
+      :userinfo, :host, :port,
+      :path
+    ].freeze
+
+    alias :vhost :path
+  end
+
+  @@schemes['AMQP'] = AMQP
+end

data/lib/wakame/action.rb

@@ -0,0 +1,156 @@
+module Wakame
+  class Action
+    include AttributeHelper
+    include ThreadImmutable
+    
+    def_attribute :job_id
+    def_attribute :status, :ready
+    def_attribute :completion_status
+    def_attribute :parent_action
+    def_attribute :acquire_lock, false
+    
+    attr_reader :trigger
+
+    def master
+      trigger.master
+    end
+    
+    def agent_monitor
+      trigger.agent_monitor
+    end
+    
+    def service_cluster
+      trigger.service_cluster
+    end
+    def status=(status)
+      if @status != status
+        @status = status
+        # Notify to observers after updating the attribute
+        notify
+      end
+      @status
+    end
+    thread_immutable_methods :status=
+      
+    def subactions
+      @subactions ||= []
+    end
+    
+    def bind_triggered_rule(trigger)
+      @trigger = trigger
+    end
+    
+    def trigger_action(subaction, opts={})
+      if opts.is_a? Hash
+        succ_proc = opts[:success] || opts[:succ]
+        fail_proc = opts[:fail]
+      end
+      subactions << subaction
+      subaction.parent_action = self
+      #subaction.observers << self
+      
+      async_trigger_action(subaction, succ_proc, fail_proc)
+    end
+    
+    def flush_subactions(sec=nil)
+      job_context = trigger.rule_engine.active_jobs[self.job_id]
+      return if job_context.nil?
+      
+      timeout(sec.nil? ? nil : sec) {
+        until all_subactions_complete?
+          #Wakame.log.debug "#{self.class} all_subactions_complete?=#{all_subactions_complete?}"
+          src = notify_queue.deq
+          # Exit the current action when a subaction notified exception.
+          if src.is_a?(Exception)
+            raise src
+          end
+          #Wakame.log.debug "#{self.class} notified by #{src.class}, all_subactions_complete?=#{all_subactions_complete?}"
+        end
+      }
+    end
+    
+    def all_subactions_complete?
+      subactions.each { |a|
+        #Wakame.log.debug("#{a.class}.status=#{a.status}")
+        return false unless a.status == :complete && a.all_subactions_complete?
+      }
+      true
+    end
+    
+    def notify_queue
+      @notify_queue ||= ::Queue.new
+    end
+    
+    def notify(src=nil)
+      #Wakame.log.debug("#{self.class}.notify() has been called")
+      src = self if src.nil?
+      if status == :complete && parent_action
+        # Escalate the notification to parent if the action is finished.
+        parent_action.notify(src)
+      else
+        notify_queue.clear if notify_queue.size > 0
+        notify_queue.enq(src) #if notify_queue.num_waiting > 0
+      end
+    end
+    
+    
+    def walk_subactions(&blk)
+      blk.call(self)
+      self.subactions.each{ |a|
+        a.walk_subactions(&blk)
+      }
+    end
+    
+    def actor_request(agent_id, path, *args, &blk)
+      request = master.actor_request(agent_id, path, *args)
+      if blk
+        request.request
+        blk.call(request)
+      end
+      request
+    end
+    
+    def sync_actor_request(agent_id, path, *args)
+      request = actor_request(agent_id, path, *args).request
+      request.wait
+    end
+    
+    def notes
+      trigger.rule_engine.active_jobs[self.job_id][:notes]
+    end
+    
+    def run
+      raise NotImplementedError
+    end
+    
+    def on_failed
+    end
+    
+    def on_canceled
+    end
+    
+    private
+    def sync_trigger_action(action, succ_proc, fail_proc)
+      action.job_id = self.job_id
+      action.bind_triggered_rule(self.trigger)
+      
+      Wakame.log.debug("Start nested action in SYNC: #{action.class.to_s}")
+      begin
+        action.run
+        succ_proc.call if succ_proc
+      rescue => e
+        fail_proc.call if fail_proc
+        raise
+      end
+      Wakame.log.debug("Complete nested action : #{action.class.to_s}")
+    end
+    
+    def async_trigger_action(action, succ_proc, fail_proc)
+      action.job_id = self.job_id
+      action.bind_triggered_rule(self.trigger)
+      
+      trigger.rule_engine.run_action(action)
+    end
+    
+  end
+end

data/lib/wakame/actions/destroy_instances.rb

@@ -0,0 +1,39 @@
+module Wakame
+  module Actions
+    class DestroyInstances < Action
+      def initialize(svc_prop)
+        @svc_prop = svc_prop
+      end
+
+      def run
+        svc_to_stop=[]
+
+        EM.barrier {
+          online_svc = []
+          service_cluster.each_instance(@svc_prop.class) { |svc_inst|
+            if svc_inst.status == Service::STATUS_ONLINE
+              online_svc << svc_inst
+            end
+          }
+
+          svc_count = service_cluster.instance_count(@svc_prop)
+          if svc_count < online_svc.size
+            online_svc.delete_if { |svc|
+              svc.agent.agent_id == master.attr[:instance_id]
+            }
+        
+            ((online_svc.size - svc_count) + 1).times {
+              svc_to_stop << online_svc.shift
+            }
+            Wakame.log.debug("#{self.class}: online_svc.size=#{online_svc.size}, svc_to_stop.size=#{svc_to_stop.size}")
+          end
+        }
+
+        svc_to_stop.each { |svc_inst|
+          trigger_action(StopService.new(svc_inst))
+        }
+        flush_subactions
+      end
+    end
+  end
+end

data/lib/wakame/actions/launch_cluster.rb

@@ -0,0 +1,31 @@
+module Wakame
+  module Actions
+    class LaunchCluster < Action
+      def initialize
+        @acquire_lock = true
+      end
+
+      def run
+        if service_cluster.status == Service::ServiceCluster::STATUS_ONLINE
+          Wakame.log.info("The service cluster is up & running already")
+          raise CancelActionError
+        end
+
+        EM.barrier {
+          service_cluster.launch
+        }
+
+        Wakame.log.debug("#{self.class}: Resource Launch Order: " + service_cluster.dg.levels.collect {|lv| '['+ lv.collect{|prop| "#{prop.class}" }.join(', ') + ']' }.join(', '))
+
+        service_cluster.dg.levels.each { |lv|
+          lv.each { |svc_prop|
+            trigger_action(PropagateInstances.new(svc_prop))
+          }
+          flush_subactions
+          Wakame.log.debug("#{self.class}: DG level next")
+        }
+      end
+
+    end
+  end
+end

data/lib/wakame/actions/migrate_service.rb

@@ -0,0 +1,65 @@
+
+require 'wakame/actions/util'
+
+module Wakame
+  module Actions
+    class MigrateService < Action
+      include Actions::Util
+
+      def initialize(service_instance, dest_agent=nil)
+        @service_instance = service_instance
+        @destination_agent = dest_agent
+      end
+
+      def run
+        raise CancelActionError if @service_instance.status == Service::STATUS_MIGRATING
+
+        EM.barrier {
+          @service_instance.update_status(Service::STATUS_MIGRATING)
+        }
+        prop = @service_instance.property
+        if prop.duplicable
+          clone_service(prop)
+          flush_subactions
+          trigger_action(StopService.new(@service_instance))
+        else
+          
+          trigger_action(StopService.new(@service_instance))
+          flush_subactions
+          clone_service(prop)
+        end
+        flush_subactions
+      end
+
+      private
+      def clone_service(resource)
+        new_svc = nil
+        EM.barrier {
+          new_svc = service_cluster.propagate(resource, true)
+        }
+
+        agent = @destination_agent
+        if agent.nil?
+          EM.barrier {
+            agent = arrange_agent(resource)
+          }
+          if agent.nil?
+            inst_id = start_instance(master.attr[:ami_id], resource.vm_spec.current.attrs)
+            agent = agent_monitor.registered_agents[inst_id]
+          end
+        end
+
+        if !(agent && test_agent_candidate(resource, agent))
+          raise "Found confiction(s) when the agent is assigned to sevice: #{resource} #{agent} "
+        end
+
+        EM.barrier {
+          new_svc.bind_agent(agent)
+        }
+
+        trigger_action(StartService.new(new_svc))
+        new_svc
+      end
+    end
+  end
+end