wakame 0.4.2 → 0.5.0

data/History.txt

@@ -1,3 +1,11 @@
+== 0.5.0 2009-09-18
+
+* Add Memcached resource.
+
+* Use SQL based persistent store for the status information.
+
+* Many internal changes along with the above change.
+
 == 0.4.2 2009-07-24
 
 * Add Web API for master control. (Note that the API design is not fixed yet)

data/Rakefile

@@ -17,8 +17,7 @@ __END__
     gem.rubyforge_project = 'wakame'
     gem.executables = ['wakame']
     [['amqp','>= 0.6.0'],
-     ['amazon-ec2','>= 0.3.6'],
-     ['right_aws','>= 1.10.0'],
+     ['right_aws','>= 1.10.0'], # Ec2ELB works with 1.10.99 in their github.
      ['eventmachine','>= 0.12.8'],
      ['rake', '>= 0.8.7'],
      ['log4r', '>= 1.0.5'],
@@ -28,7 +27,8 @@ __END__
      ['jeweler', '>= 1.0.0'],
      ['rack', '>= 1.0.0'],
      ['thin', '>= 1.2.2'],
-     ['json', '>= 1.1.7']
+     ['json', '>= 1.1.7'],
+     ['sequel', '>= 3.2.0']
     ].each { |i|
       gem.add_dependency(i[0], i[1])
     }

data/VERSION

@@ -1 +1 @@
-0.4.2
+0.5.0

data/app_generators/wakame/templates/cluster/resources/markers/http_application_server.rb

@@ -0,0 +1,3 @@
+module HttpApplicationServer
+end
+

data/app_generators/wakame/templates/cluster/resources/markers/http_asset_server.rb

@@ -0,0 +1,2 @@
+module HttpAssetServer
+end

data/app_generators/wakame/templates/cluster/resources/markers/http_server.rb

@@ -0,0 +1,9 @@
+module HttpServer
+  def self.included(klass)
+    klass.class_eval {
+      property :listen_port
+      property :listen_port_https
+    }
+  end
+end
+

data/app_generators/wakame/templates/config/cluster.rb

@@ -1,100 +1,37 @@
-class WebCluster < Wakame::Service::ServiceCluster
-  attr_accessor :propagation_priority
+define_cluster('WebCluster1') { |c|
+  c.add_resource(Apache_APP.new) { |r|
+    r.listen_port = 8001
+    r.max_instances = 5
+  }
+  c.add_resource(Nginx.new)
+  c.add_resource(Ec2ELB.new) { |r|
+    r.elb_name = 'xxxxxxx'
+  }
+  c.add_resource(MySQL_Master.new) {|r|
+    r.mysqld_basedir = '/home/wakame/mysql'
+    r.ebs_volume = 'vol-xxxxxxx'
+    r.ebs_device = '/dev/sdm'
+  }
+  
+  c.set_dependency(Apache_APP, Nginx)
+  c.set_dependency(Nginx, Ec2ELB)
+  c.set_dependency(MySQL_Master, Apache_APP)
+
+  host = c.add_cloud_host { |h|
+    #h.vm_spec.availability_zone = 'us-east-1a'
+  }
+  c.propagate(Nginx, host.id)
+  c.propagate(Apache_APP, host.id)
+  c.propagate(MySQL_Master, host.id)
+  c.propagate(Ec2ELB)
+
+  c.define_triggers {|r|
+    #r.register_trigger(Wakame::Triggers::MaintainSshKnownHosts.new)
+    #r.register_trigger(Wakame::Triggers::LoadHistoryMonitor.new)
+    #r.register_trigger(Wakame::Triggers::InstanceCountUpdate.new)
+    #r.register_trigger(Wakame::Triggers::ScaleOutWhenHighLoad.new)
+    #r.register_trigger(Wakame::Triggers::ShutdownUnusedVM.new)
+  }
+
+}
 
-  module HttpAppServer; end
-  module HttpAssetServer; end
-  module HttpLoadBalanceServer; end
-
-  VirtualHost = Class.new(OpenStruct)
-  def initialize(master, &blk)
-    super(master) { |c|
-      c.add_resource(Ec2ElasticIp.new)
-      c.add_resource(Apache_WWW.new)
-      c.add_resource(Apache_APP.new)
-      c.add_resource(Apache_LB.new)
-      c.add_resource(MySQL_Master.new)
-      #c.add_resource(MySQL_Slave.new)
-
-      c.set_dependency(Apache_LB, Ec2ElasticIp)
-      c.set_dependency(Apache_WWW, Apache_LB)
-      c.set_dependency(Apache_APP, Apache_LB)
-      c.set_dependency(MySQL_Master, Apache_APP)
-      #c.set_dependency(MySQL_Master, MySQL_Slave)
-    }
-
-    define_rule { |r|
-      r.register_trigger(Wakame::Triggers::ProcessCommand.new)
-      r.register_trigger(Wakame::Triggers::MaintainSshKnownHosts.new)
-      #r.register_trigger(Wakame::Triggers::LoadHistoryMonitor.new)
-      #r.register_trigger(Wakame::Triggers::InstanceCountUpdate.new)
-      #r.register_trigger(Wakame::Triggers::ScaleOutWhenHighLoad.new)
-      #r.register_trigger(Wakame::Triggers::ShutdownUnusedVM.new)
-    }
-
-     add_virtual_host(VirtualHost.new(:server_name=>'aaa.test', :document_root=>'/home/wakame/app/development/test/public'))
-     add_virtual_host(VirtualHost.new(:server_name=>'bbb.test', :document_root=>'/home/wakame/app/development/test/public'))
-
-  end
-
-  def virtual_hosts
-    @virtual_hosts ||= []
-  end
-
-  def add_virtual_host(vh)
-    virtual_hosts << vh
-  end
-
-
-  def each_app(&blk)
-    each_instance(HttpAppServer) { |n|
-      blk.call(n)
-    }
-  end
-
-  def each_www(&blk)
-    each_instance(HttpAssetServer) { |n|
-      blk.call(n)
-    }
-  end
-
-  def each_mysql(&blk)
-    each_instance(MySQL_Master) { |n|
-      blk.call(n)
-    }
-  end
-
-  def each_mysql_master(&blk)
-    each_instance(MySQL_Master) { |n|
-      blk.call(n)
-    }
-  end
-
-  def fetch_mysql_master_ip
-    mysql_master_ip = nil
-    each_mysql_master do |mysql|
-      mysql_master_ip = mysql.agent.agent_ip
-    end
-    mysql_master_ip
-  end
-
-  def each_mysql_slave(&blk)
-    each_instance(MySQL_Slave) { |n|
-      blk.call(n)
-    }
-  end
-
-  def fetch_mysql_slave_ip
-    mysql_slave_ips = []
-    each_mysql_slave do |mysql|
-      mysql_slave_ips << mysql.agent.agent_ip
-    end
-    mysql_master_ips
-  end
-
-#  def each_postgresql(&blk)
-#    each_instance(PostgreSQL_Master) { |n|
-#      blk.call(n)
-#    }
-#  end
-
-end

data/app_generators/wakame/templates/config/init.d/wakame-agent

@@ -19,9 +19,9 @@ PIDFILE=/var/run/wakame/wakame-agent.pid
 #LSBNAMES='-l'  # Uncomment for LSB name support in /etc/cron.d/
 
 . /lib/lsb/init-functions
-if [ -z $WAKAME_HOME ]; then
+if [ -z $WAKAME_ROOT ]; then
     [ -f /etc/default/wakame ] && . /etc/default/wakame
-    export GEM_HOME WAKAME_HOME WAKAME_ENV
+    export GEM_HOME WAKAME_ROOT WAKAME_ENV
 fi
 
 start_agent()
@@ -40,7 +40,7 @@ start_agent()
 	    opts="$opts -s $i"
 	done
     fi
-    start-stop-daemon --start --quiet --pidfile $PIDFILE --name wakame-agent --startas $WAKAME_HOME/bin/wakame-agent -- $opts $LSBNAMES
+    start-stop-daemon --start --quiet --pidfile $PIDFILE --name wakame-agent --startas $WAKAME_ROOT/bin/wakame-agent -- $opts $LSBNAMES
 }
 
 stop_agent()

data/app_generators/wakame/templates/config/init.d/wakame-master

@@ -21,9 +21,9 @@ PIDFILE=/var/run/wakame/wakame-master.pid
 #LSBNAMES='-l'  # Uncomment for LSB name support in /etc/cron.d/
 
 . /lib/lsb/init-functions
-if [ -z $WAKAME_HOME ]; then
+if [ -z $WAKAME_ROOT ]; then
     [ -f /etc/default/wakame ] && . /etc/default/wakame
-    export GEM_HOME WAKAME_HOME WAKAME_ENV
+    export GEM_HOME WAKAME_ROOT WAKAME_ENV
 fi
 
 start_master()
@@ -41,7 +41,7 @@ start_master()
       chown "$USER:$GROUP" $pid_dir
     fi
     
-    start-stop-daemon --start --quiet --pidfile $PIDFILE --name wakame-master --startas $WAKAME_HOME/bin/wakame-master -- -u $USER $LSBNAMES
+    start-stop-daemon --start --quiet --pidfile $PIDFILE --name wakame-master --startas $WAKAME_ROOT/bin/wakame-master -- -u $USER $LSBNAMES
     
 }
 stop_master()

data/app_generators/wakame/wakame_generator.rb

@@ -14,6 +14,7 @@ class WakameGenerator < RubiGen::Base
     cluster/actions
     cluster/triggers
     cluster/resources
+    cluster/resources/markers
     cluster/commands
     config
     config/init.d
@@ -58,7 +59,10 @@ class WakameGenerator < RubiGen::Base
       BASEDIRS.each { |path| m.directory path }
 
       m.file_copy_each %w(Rakefile README)
-      m.file_copy_each %w(config/boot.rb config/cluster.rb config/environments/common.rb config/environments/stand_alone.rb  config/environments/ec2.rb)
+      m.file_copy_each %w(config/boot.rb config/cluster.rb config/environments/common.rb config/environments/stand_alone.rb  config/environments/ec2.rb
+                          cluster/resources/markers/http_application_server.rb
+                          cluster/resources/markers/http_asset_server.rb
+                          cluster/resources/markers/http_server.rb)
       m.dependency "install_rubigen_scripts", [destination_root, :wakame]
 
       %w(wakame-master wakame-agent wakameadm).each do |script|

data/contrib/imagesetup.sh

@@ -9,12 +9,14 @@ apt-get -y install apache2-mpm-prefork libapache2-mod-rpaf
 apt-get -y install mysql-server mysql-client
 apt-get -y install erlang-nox
 apt-get -y install unzip zip rsync libopenssl-ruby libhmac-ruby rubygems irb ri rdoc sysstat
+apt-get -y install apache2-prefork-dev ruby-dev make g++ libopenssl-ruby subversion
+apt-get -y install memcached
 
 apt-get clean
 
 (cd /tmp;
-wget http://www.rabbitmq.com/releases/rabbitmq-server/v1.5.4/rabbitmq-server_1.5.4-1_all.deb;
-dpkg -i rabbitmq-server_1.5.4-1_all.deb;
+wget http://www.rabbitmq.com/releases/rabbitmq-server/v1.6.0/rabbitmq-server_1.6.0-1_all.deb
+dpkg -i rabbitmq-server_1.6.0-1_all.deb
 )
 
 
@@ -40,7 +42,7 @@ EOF
 fi
 
 cat <<EOF > /etc/default/wakame
-WAKAME_HOME=/home/wakame/corelib
+WAKAME_ROOT=/home/wakame/wakame.proj
 GEM_HOME=/usr/local/gems
 EOF
 
@@ -48,12 +50,14 @@ update-rc.d -f apache2 remove
 update-rc.d -f mysql remove
 update-rc.d -f mysql-ndb remove
 update-rc.d -f mysql-ndb-mgm remove
+update-rc.d -f mysql-ndb-mgm memcached
 # Disable apparmor
 update-rc.d -f apparmor remove
 
-cat <<EOF > /usr/local/bin/passenger_ruby.sh
+cat <<'EOF' > /usr/local/bin/passenger_ruby.sh
 #!/bin/sh
-export GEM_PATH="/usr/local/gems"
+. /etc/environment
+export RUBYLIB GEM_HOME
 exec /usr/bin/ruby $@
 EOF
 chmod 755 /usr/local/bin/passenger_ruby.sh

data/lib/ext/uri.rb

@@ -8,7 +8,20 @@ module URI
       :path
     ].freeze
 
+    DEFAULT_PORT=5672
+
+    def self.build(args)
+      tmp = Util::make_components_hash(self, args)
+      return super(tmp)
+    end
+
+    def initialize(*args)
+      args[5] = '/' if args[5].nil? || args[5] == ''
+      super(*args)
+    end
+
     alias :vhost :path
+    alias :vhost= :path=
   end
 
   @@schemes['AMQP'] = AMQP

data/lib/wakame/action.rb

@@ -8,19 +8,25 @@ module Wakame
     def_attribute :completion_status
     def_attribute :parent_action
     
-    attr_reader :trigger
+    attr_accessor :action_manager
 
     def master
-      trigger.master
+      action_manager.master
     end
     
     def agent_monitor
-      trigger.agent_monitor
+      master.agent_monitor
     end
     
+    # Tentative utility method for 
     def service_cluster
-      trigger.service_cluster
+      cluster_id = master.cluster_manager.clusters.first
+      raise "There is no cluster loaded" if cluster_id.nil?
+
+      Service::ServiceCluster.find(cluster_id)
     end
+    alias :cluster :service_cluster
+
     def status=(status)
       if @status != status
         @status = status
@@ -35,26 +41,27 @@ module Wakame
       @subactions ||= []
     end
     
-    def bind_trigger(trigger)
-      @trigger = trigger
-    end
-    
-    def trigger_action(subaction, opts={})
+    def trigger_action(subaction=nil, &blk)
+      if blk 
+        subaction = ProcAction.new(blk)
+      end
+
       subactions << subaction
       subaction.parent_action = self
       subaction.job_id = self.job_id
-      subaction.bind_trigger(self.trigger)
+      subaction.action_manager = self.action_manager
       
-      trigger.rule_engine.run_action(subaction)
+      action_manager.run_action(subaction)
     end
     
-    def flush_subactions(sec=nil)
-      job_context = trigger.rule_engine.active_jobs[self.job_id]
+    def flush_subactions(sec=60*30)
+      job_context = action_manager.active_jobs[self.job_id]
       return if job_context.nil?
       
-      timeout(sec.nil? ? nil : sec) {
+      timeout(sec) {
         until all_subactions_complete?
           #Wakame.log.debug "#{self.class} all_subactions_complete?=#{all_subactions_complete?}"
+          Thread.pass
           src = notify_queue.deq
           # Exit the current action when a subaction notified exception.
           if src.is_a?(Exception)
@@ -64,6 +71,7 @@ module Wakame
         end
       }
     end
+    thread_immutable_methods :flush_subactions
     
     def all_subactions_complete?
       subactions.each { |a|
@@ -76,6 +84,7 @@ module Wakame
     def notify_queue
       @notify_queue ||= ::Queue.new
     end
+    private :notify_queue
     
     def notify(src=nil)
       #Wakame.log.debug("#{self.class}.notify() has been called")
@@ -96,7 +105,7 @@ module Wakame
         a.walk_subactions(&blk)
       }
     end
-    
+        
     def actor_request(agent_id, path, *args, &blk)
       request = master.actor_request(agent_id, path, *args)
       if blk
@@ -110,23 +119,24 @@ module Wakame
       request = actor_request(agent_id, path, *args).request
       request.wait
     end
-    
+
     def notes
-      trigger.rule_engine.active_jobs[self.job_id][:notes]
+      action_manager.active_jobs[self.job_id][:notes]
     end
 
     # Set the lock flags to resources 
     def acquire_lock(&blk)
-      EM.barrier {
+      StatusDB.barrier {
         reslist = []
         blk.call(reslist)
         reslist.flatten!
-        reslist.each {|r| service_cluster.lock_queue.set(r.to_s, self.job_id) }
+        reslist.each {|r| action_manager.lock_queue.set(r.to_s, self.job_id) }
       }
       
-      service_cluster.lock_queue.wait(self.job_id)
+      action_manager.lock_queue.wait(self.job_id)
     end
-    
+    thread_immutable_methods :acquire_lock
+
     def run
       raise NotImplementedError
     end
@@ -137,5 +147,20 @@ module Wakame
     def on_canceled
     end
 
+
+    class ProcAction < Action
+      def initialize(proc_obj)
+        raise ArgumentError unless proc_obj.is_a? Proc
+        @proc_obj = proc_obj
+      end
+
+      def run()
+        @proc_obj.call(self)
+      end
+    end
+
+
   end
+
+
 end