wakame 0.4.0

data/app_generators/wakame/templates/cluster/resources/mysql_master/mysql_master.rb

@@ -0,0 +1,174 @@
+
+class MySQL_Master < Wakame::Service::Resource
+
+  def_attribute :duplicable, false
+  def_attribute :mysqld_basedir, '/home/wakame/mysql'
+  def_attribute :mysqld_server_id, 1
+  def_attribute :mysqld_port, 3306
+
+  def_attribute :ebs_volume, ''
+  def_attribute :ebs_device, '/dev/sdm'
+  def_attribute :ebs_mount_option, 'noatime'
+
+  def basedir
+    File.join(Wakame.config.root_path, 'cluster', 'resources', 'mysql_master')
+  end
+  
+  def mysqld_datadir
+    File.expand_path('data', mysqld_basedir)
+  end
+
+  def mysqld_log_bin
+    File.expand_path('mysql-bin.log', mysqld_datadir)
+  end
+
+  def render_config(template)
+    template.cp(%w(init.d/mysql))
+    template.render(%w(conf/my.cnf))
+    template.chmod("init.d/mysql", 0755)
+  end
+
+
+  def start(svc, action)
+    vm_manipulator = Wakame::VmManipulator.create
+
+    # $ echo "GRANT REPLICATION SLAVE, REPLICATION CLIENT, RELOAD ON *.* TO 'wakame-repl'@'%' IDENTIFIED BY 'wakame-slave';" | /usr/bin/mysql -h#{mysql_master_ip} -uroot
+
+    require 'right_aws'
+    ec2 = RightAws::Ec2.new(Wakame.config.aws_access_key, Wakame.config.aws_secret_key)
+
+    res = ec2.describe_volumes([@ebs_volume])[0]
+    ec2_instance_id = res[:aws_instance_id]
+    if res[:aws_status] == 'in-use' && ec2_instance_id == svc.agent.attr[:instance_id]
+       # Nothin to be done
+    elsif res[:aws_status] == 'in-use' && ec2_instance_id != svc.agent.attr[:instance_id]
+      ec2.detach_volume(@ebs_volume)
+      cond = ConditionalWait.new { |c|
+        c.poll {
+          res1 = ec2.describe_volumes([@ebs_volume])[0]
+          res1[:aws_status] == 'available'
+        }
+      }
+      cond.wait
+
+      ec2.attach_volume(@ebs_volume, svc.agent.attr[:instance_id], @ebs_device)
+      cond = ConditionalWait.new { |c|
+        c.poll {
+          res1 = ec2.describe_volumes([@ebs_volume])[0]
+          res1[:aws_status] == 'in-use'
+        }
+      }
+      cond.wait
+
+    elsif res[:aws_status] == 'available'
+      ec2.attach_volume(@ebs_volume, svc.agent.attr[:instance_id], @ebs_device)
+      cond = ConditionalWait.new { |c|
+        c.poll {
+          res1 = ec2.describe_volumes([@ebs_volume])[0]
+          res1[:aws_status] == 'in-use'
+        }
+      }
+      cond.wait
+      
+    end
+    # in-use:
+#    res = vm_manipulator.describe_volume(@ebs_volume)
+#    Wakame.log.debug("describe_volume(#{@ebs_volume}): #{res.inspect}")
+#    ec2_instance_id=nil
+#     if res['attachmentSet']
+#       ec2_instance_id = res['attachmentSet']['item'][0]['instanceId']
+#     end
+
+#     if res['status'] == 'in-use' && ec2_instance_id == svc.agent.agent_id
+#       # Nothin to be done
+#     elsif res['status'] == 'in-use' && ec2_instance_id != svc.agent.agent_id
+#       vm_manipulator.detach_volume(@ebs_volume)
+#       sleep 1.0
+#       res = vm_manipulator.attach_volume(svc.agent.agent_id, @ebs_volume, @ebs_device)
+#       Wakame.log.debug(res.inspect)
+#       # sync
+#       3.times do |i|
+#         system("/bin/sync")
+#         sleep 1.0
+#       end
+#     elsif res['status'] == 'available'
+#       res = vm_manipulator.attach_volume(svc.agent.agent_id, @ebs_volume, @ebs_device)
+#       Wakame.log.debug(res.inspect)
+#       # sync
+#       3.times do |i|
+#         system("/bin/sync")
+#         sleep 1.0
+#       end
+#     else
+#       raise "The EBS volume is not ready to attach: #{@ebs_volume}"
+#     end
+
+    cond = ConditionalWait.new { |cond|
+      cond.wait_event(Wakame::Event::ServiceOnline) { |event|
+        event.instance_id == svc.instance_id
+      }
+    }
+
+    action.actor_request(svc.agent.agent_id,
+                         '/service_monitor/register',
+                         svc.instance_id,
+                         :command, "/usr/bin/mysqladmin --defaults-file=#{svc.agent.root_path}/tmp/config/mysql_master/conf/my.cnf ping > /dev/null") { |req|
+    }
+
+    action.actor_request(svc.agent.agent_id, '/system/mount', @ebs_device, @mysqld_datadir, @ebs_mount_option) { |req|
+      req.wait
+      Wakame.log.debug("MySQL volume was mounted: #{@mysqld_datadir}")
+    }
+
+    action.actor_request(svc.agent.agent_id, '/daemon/start', 'mysql_master', 'init.d/mysql') { |req|
+      req.wait
+      Wakame.log.debug("MySQL process started")
+    }
+
+    #request.wait
+    cond.wait
+
+
+#     mount_point_dev=`df "#{@mysqld_datadir}" | awk 'NR==2 {print $1}'`
+#     if mount_point_dev != @ebs_device
+#       Wakame.log.debug("Mounting EBS volume: #{@ebs_device} as #{@mysqld_datadir} (with option: #{@ebs_mount_option})")
+#       system("/bin/mount -o #{@ebs_mount_option} #{@ebs_device} #{@mysqld_datadir}")
+#       # sync
+#       3.times do |i|
+#         system("/bin/sync")
+#         sleep 1.0
+#       end
+#     end
+#     system(Wakame.config.root + "/config/init.d/mysql start")
+  end
+  
+  #def check
+  #  system("/usr/bin/mysqladmin --defaults-file=/home/wakame/config/mysql/my.cnf ping > /dev/null")
+  #  return false if $? != 0
+  #  true
+  #end
+  
+  def stop(svc, action)
+    cond = ConditionalWait.new { |cond|
+      cond.wait_event(Wakame::Event::ServiceOffline) { |event|
+        event.instance_id == svc.instance_id
+      }
+    }
+    
+    action.actor_request(svc.agent.agent_id, '/daemon/stop', 'mysql_master', 'init.d/mysql') { |req| req.wait }
+    action.actor_request(svc.agent.agent_id, '/system/umount', @mysqld_datadir) { |req|
+      req.wait
+      Wakame.log.debug("MySQL volume unmounted")
+    }
+
+    #request.wait
+    cond.wait
+
+    action.actor_request(svc.agent.agent_id,
+                         '/service_monitor/unregister',
+                         svc.instance_id).request
+
+
+    #system(Wakame.config.root + "/config/init.d/mysql stop")
+  end
+end

data/app_generators/wakame/templates/config/boot.rb

@@ -0,0 +1,85 @@
+
+require 'pathname'
+WAKAME_ROOT = Pathname.new((ENV['WAKAME_ROOT'] || "#{File.dirname(__FILE__)}/..").dup).realpath unless defined?(WAKAME_ROOT)
+
+module Wakame
+  module Bootstrap
+    class << self
+      def boot_master!
+        boot!(:process_master)
+      end
+      def boot_agent!
+        boot!(:process_agent)
+      end
+      def boot_cli!
+        boot!(:process_cli)
+      end
+
+      def boot!(method=:process)
+        unless booted?
+          run_bootstrap
+          Wakame::Initializer.run(method)
+        end
+      end
+
+      def booted?
+        #defined?(Wakame::Initializer)
+        !@bootclass.nil?
+      end
+
+      def boot_type
+        @bootclass.class
+      end
+
+      def run_bootstrap
+        @bootclass = (framework_bundled? ? VendorBoot : GemBoot).new
+        @bootclass.run
+      end
+
+      def framework_bundled?
+        File.exists?("#{WAKAME_ROOT}/vendor/wakame")
+      end
+    end
+
+    class Boot
+      def run
+        load_initializer
+      end
+
+      def load_initializer
+      end
+    end
+
+    class VendorBoot < Boot
+      def load_initializer
+        require "#{WAKAME_ROOT}/vendor/wakame/lib/wakame/initializer"
+        require "#{WAKAME_ROOT}/vendor/wakame/lib/wakame/configuration"
+      end
+    end
+
+    class GemBoot < Boot
+      def load_initializer
+        load_rubygems
+        gem 'wakame'
+        require 'wakame/initializer'
+        require 'wakame/configuration'
+      end
+
+
+      def load_rubygems
+        require 'rubygems'
+        unless rubygems_version >= '1.3.0'
+          $stderr.puts "Rails requires RubyGems >= #{min_version} (you have #{rubygems_version}). Please `gem update --system` and try again."
+          exit 1
+        end
+
+      rescue LoadError
+        $stderr.puts "Rails requires RubyGems >= #{min_version}. Please install RubyGems and try again: http://rubygems.rubyforge.org"
+        exit 1
+      end
+
+
+    end
+
+  end
+end

data/app_generators/wakame/templates/config/cluster.rb

@@ -0,0 +1,64 @@
+class WebCluster < Wakame::Service::ServiceCluster
+  attr_accessor :propagation_priority
+
+  module HttpAppServer; end
+  module HttpAssetServer; end
+  module HttpLoadBalanceServer; end
+
+  VirtualHost = Class.new(OpenStruct)
+  def initialize(master, &blk)
+    super(master) { |c|
+      c.add_resource(Ec2ElasticIp.new)
+      c.add_resource(Apache_WWW.new)
+      c.add_resource(Apache_APP.new)
+      c.add_resource(Apache_LB.new)
+      c.add_resource(MySQL_Master.new)
+
+      c.set_dependency(Apache_LB, Ec2ElasticIp)
+      c.set_dependency(Apache_WWW, Apache_LB)
+      c.set_dependency(Apache_APP, Apache_LB)
+      c.set_dependency(MySQL_Master, Apache_APP)
+    }
+
+    define_rule { |r|
+      r.register_trigger(Wakame::Triggers::ProcessCommand.new)
+      r.register_trigger(Wakame::Triggers::MaintainSshKnownHosts.new)
+      #r.register_trigger(Wakame::Triggers::LoadHistoryMonitor.new)
+      #r.register_trigger(Wakame::Triggers::InstanceCountUpdate.new)
+      #r.register_trigger(Wakame::Triggers::ScaleOutWhenHighLoad.new)
+      #r.register_trigger(Wakame::Triggers::ShutdownUnusedVM.new)
+    }
+
+     add_virtual_host(VirtualHost.new(:server_name=>'aaa.test', :document_root=>'/home/wakame/app/development/test/public'))
+     add_virtual_host(VirtualHost.new(:server_name=>'bbb.test', :document_root=>'/home/wakame/app/development/test/public'))
+
+  end
+
+  def virtual_hosts
+    @virtual_hosts ||= []
+  end
+
+  def add_virtual_host(vh)
+    virtual_hosts << vh
+  end
+
+
+  def each_app(&blk)
+    each_instance(HttpAppServer) { |n|
+      blk.call(n)
+    }
+  end
+
+  def each_www(&blk)
+    each_instance(HttpAssetServer) { |n|
+      blk.call(n)
+    }
+  end
+
+  def each_mysql(&blk)
+    each_instance(MySQL_Master) { |n|
+      blk.call(n)
+    }
+  end
+
+end

data/app_generators/wakame/templates/config/environments/ec2.rb

@@ -0,0 +1,3 @@
+
+config.aws_access_key = ''
+config.aws_secret_key = ''

data/app_generators/wakame/templates/config/init.d/wakame-agent

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Start/stop the wakame agent daemon.
+#
+### BEGIN INIT INFO
+# Provides:          wakame-agent
+# Required-Start:    $local_fs $network rabbitmq-server wakame-master
+# Required-Stop:     $local_fs $network rabbitmq-server wakame-master
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: 
+# Description:       cron is a standard UNIX program that runs user-specified 
+#                    programs at periodic scheduled times. vixie cron adds a 
+#                    number of features to the basic UNIX cron, including better
+#                    security and more powerful configuration options.
+### END INIT INFO
+
+PIDFILE=/var/run/wakame/wakame-agent.pid
+
+#LSBNAMES='-l'  # Uncomment for LSB name support in /etc/cron.d/
+
+. /lib/lsb/init-functions
+if [ -z $WAKAME_HOME ]; then
+    [ -f /etc/default/wakame ] && . /etc/default/wakame
+    export GEM_HOME WAKAME_HOME WAKAME_ENV
+fi
+
+start_agent()
+{
+    typeset user_data i
+    # Prepare the parent directory for the pidfile.
+    typeset pid_dir=`dirname $PIDFILE`
+    [ -d $pid_dir ] || {
+      mkdir -p /var/run/wakame
+    }
+
+    user_data="`curl -s -f --retry 3 http://169.254.169.254/2008-02-01/user-data`"
+    if [ "$?" = 0 -a -n "$user_data" ]; then
+	for i in `echo "$user_data" | awk -F\= '$1 == "amqp_server" {print $2}'`
+	do
+	    opts="$opts -s $i"
+	done
+    fi
+    start-stop-daemon --start --quiet --pidfile $PIDFILE --name wakame-agent --startas $WAKAME_HOME/bin/wakame-agent -- $opts $LSBNAMES
+}
+
+stop_agent()
+{
+    start-stop-daemon --stop --quiet --pidfile $PIDFILE --name wakame-agent
+}
+
+case "$1" in
+start)	log_daemon_msg "Starting agent process for Wakame" "wakame-agent"
+	start_agent
+        log_end_msg $?
+	;;
+stop)	log_daemon_msg "Stopping agent process for Wakame" "wakame-agent"
+	stop_agent
+        log_end_msg $?
+        ;;
+restart) log_daemon_msg "Restarting agent process for Wakame" "wakame-agent" 
+	stop_agent
+	start_agent
+        log_end_msg $?
+        ;;
+reload|force-reload) log_daemon_msg "Reloading configuration files for " ""
+        log_end_msg 0
+        ;;
+*)	log_action_msg "Usage: /etc/init.d/wakame-agent {start|stop|restart|reload|force-reload}"
+        exit 2
+        ;;
+esac
+exit 0

data/app_generators/wakame/templates/config/init.d/wakame-master

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Start/stop the wakame master daemon.
+#
+### BEGIN INIT INFO
+# Provides:          wakame-master
+# Required-Start:    $local_fs $network rabbitmq-server
+# Required-Stop:     $local_fs $network rabbitmq-server
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: 
+# Description:       cron is a standard UNIX program that runs user-specified 
+#                    programs at periodic scheduled times. vixie cron adds a 
+#                    number of features to the basic UNIX cron, including better
+#                    security and more powerful configuration options.
+### END INIT INFO
+
+USER=wakame
+GROUP=wakame
+PIDFILE=/var/run/wakame/wakame-master.pid
+
+#LSBNAMES='-l'  # Uncomment for LSB name support in /etc/cron.d/
+
+. /lib/lsb/init-functions
+if [ -z $WAKAME_HOME ]; then
+    [ -f /etc/default/wakame ] && . /etc/default/wakame
+    export GEM_HOME WAKAME_HOME WAKAME_ENV
+fi
+
+start_master()
+{
+    curl -s -f --retry 3 http://169.254.169.254/2008-02-01/user-data | grep "node=agent" > /dev/null && {
+	log_daemon_msg "Canceled to start master process for Wakame"
+        exit 0
+    }
+    # Prepare the parent directory for the pidfile.
+    typeset pid_dir=`dirname $PIDFILE`
+    if su -c "test -w $pid_dir" $USER; then
+      :
+    else
+      mkdir -p $pid_dir
+      chown "$USER:$GROUP" $pid_dir
+    fi
+    
+    start-stop-daemon --start --quiet --pidfile $PIDFILE --name wakame-master --startas $WAKAME_HOME/bin/wakame-master -- -u $USER $LSBNAMES
+    
+}
+stop_master()
+{
+    start-stop-daemon --stop --quiet --pidfile $PIDFILE --name wakame-master
+}
+
+case "$1" in
+start)	log_daemon_msg "Starting master process for Wakame" "wakame-master"
+	start_master
+        log_end_msg $?
+	;;
+stop)	log_daemon_msg "Stopping master process for Wakame" "wakame-master"
+	stop_master
+        log_end_msg $?
+        ;;
+restart) log_daemon_msg "Restarting master process for Wakame" "wakame-master" 
+	stop_master
+	start_master
+        log_end_msg $?
+        ;;
+reload|force-reload) log_daemon_msg "Reloading configuration files for " ""
+        log_end_msg 0
+        ;;
+*)	log_action_msg "Usage: /etc/init.d/wakame-master {start|stop|restart|reload|force-reload}"
+        exit 2
+        ;;
+esac
+exit 0

data/app_generators/wakame/wakame_generator.rb

@@ -0,0 +1,124 @@
+require 'rbconfig'
+
+class WakameGenerator < RubiGen::Base
+
+  DEFAULT_SHEBANG = File.join(Config::CONFIG['bindir'],
+                              Config::CONFIG['ruby_install_name'])
+
+  # Installation skeleton.  Intermediate directories are automatically
+  # created so don't sweat their absence here.
+  BASEDIRS = %w(
+    bin
+    cluster
+    cluster/actors
+    cluster/actions
+    cluster/triggers
+    cluster/resources
+    cluster/commands
+    config
+    config/init.d
+    config/environments
+    lib
+    tasks
+    vendor
+  )
+
+  default_options   :shebang => DEFAULT_SHEBANG,
+  :bin_name    => nil,
+  :import_path => nil,
+  :version     => '0.0.1'
+
+
+  attr_reader :gem_name, :module_name, :project_name
+  attr_reader :version, :version_str
+
+  # extensions/option
+  attr_reader :bin_names_list
+
+  def initialize(runtime_args, runtime_options = {})
+    super(config_args_and_runtime_args(runtime_args), runtime_options)
+    usage if args.empty?
+    @destination_root = File.expand_path(args.shift)
+    @gem_name = base_name
+    @module_name  = gem_name.gsub('-','_').camelize
+    @project_name = @gem_name
+    extract_options
+  end
+
+  def manifest
+    # Use /usr/bin/env if no special shebang was specified
+    script_options = { :chmod => 0755, :shebang => options[:shebang] == DEFAULT_SHEBANG ? nil : options[:shebang] }
+    windows        = (RUBY_PLATFORM =~ /dos|win32|cygwin/i) || (RUBY_PLATFORM =~ /(:?mswin|mingw)/)
+    source_dir     = source_path('.').sub(%r{/\.\Z}, '')
+
+    record do |m|
+      # Root directory and all subdirectories.
+      m.directory ''
+      BASEDIRS.each { |path| m.directory path }
+
+      m.file_copy_each %w(Rakefile README)
+      m.file_copy_each %w(config/boot.rb config/cluster.rb config/environments/common.rb config/environments/stand_alone.rb  config/environments/ec2.rb)
+      m.file_copy_each %w(config/init.d/wakame-master config/init.d/wakame-agent)
+
+      m.dependency "install_rubigen_scripts", [destination_root, :wakame]
+
+      %w(wakame-master wakame-agent wakameadm).each do |script|
+        m.template "bin/#{script}", "bin/#{script}", script_options
+      end
+      
+    end
+  end
+
+  protected
+  def banner
+    <<-EOS
+Usage: #{File.basename $0} /path/to/your/app [options]
+EOS
+  end
+
+  def add_options!(opts)
+    opts.separator ''
+    opts.separator 'Options:'
+    opts.on("-b", "--bin-name=BIN_NAME[,BIN_NAME2]", String,
+            "Sets up executable scripts in the bin folder.",
+            "Default: none") { |x| options[:bin_name] = x }
+    opts.on("-i", "--install=generator", String,
+            "Installs a generator called install_<generator>.",
+            "For example, '-i cucumber' runs the install_cucumber generator.",
+            "Can be used multiple times for different generators.",
+            "Cannot be used for generators that require argumnts.",
+            "Default: none") do |generator|
+      options[:install] ||= []
+      options[:install] << generator
+    end
+    opts.on("-p", "--project=PROJECT", String,
+            "Rubyforge project name for the gem you are creating.",
+            "Default: same as gem name") { |x| options[:project] = x }
+    opts.on("-r", "--ruby=path", String,
+            "Path to the Ruby binary of your choice (otherwise scripts use env, dispatchers current path).",
+            "Default: #{DEFAULT_SHEBANG}") { |x| options[:shebang] = x }
+    opts.on("-v", "--version", "Show the #{File.basename($0)} version number and quit.")
+  end
+
+  def extract_options
+    @version           = options[:version].to_s.split(/\./)
+    @version_str       = @version.join('.')
+
+    @bin_names_list     = (options[:bin_name] || "").split(',')
+    @project_name       = options[:project] if options.include?(:project)
+    @install_generators = options[:install] || []
+  end
+  
+  # first attempt to merge config args (single string) and runtime args
+  def config_args_and_runtime_args(runtime_args)
+    newgem_config = File.expand_path(File.join(ENV['HOME'], '.newgem.yml'))
+    if File.exists?(newgem_config)
+      config = YAML.load(File.read(newgem_config))
+      if config_args = (config["default"] || config[config.keys.first])
+        return config_args.split(" ") + runtime_args
+      end
+    end
+    runtime_args
+  end
+
+end

data/bin/wakame

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+# -*- ruby -*-
+require 'rubygems' rescue ENV['NO_RUBYGEMS']
+if %w(--version -v).include? ARGV.first
+  require File.dirname(__FILE__) + "/../lib/wakame"
+  puts "#{File.basename($0)} #{Wakame::VERSION}"
+  exit(0)
+end
+
+#app_path = ARGV.first
+
+require 'rubigen'
+require 'rubigen/scripts/generate'
+RubiGen::Base.use_application_sources! 
+RubiGen::Base.append_sources RubiGen::PathSource.new(:application,
+                                                     File.join(File.dirname(__FILE__), "../app_generators"))
+#p RubiGen::Base.sources
+RubiGen::Scripts::Generate.new.run(ARGV, :generator => 'wakame')