wakame 0.4.0

data/lib/wakame/vm_manipulator.rb

@@ -0,0 +1,186 @@
+
+require 'wakame'
+
+module Wakame
+  module VmManipulator
+    def self.create
+      Wakame.new_([self.to_s, Wakame.config.vm_environment ].join('::'))
+    end
+
+    class Base
+      def start_instance(attr)
+      end
+
+      def stop_instance(instance_id)
+      end
+      
+      # expected_status : :online, :offline, :failed
+      def check_status(instance_id, expected_status=:online)
+      end
+
+      # Expected common keys/attributes when returned.
+      # local_ipv4 : Internal IP address which is assigned to the VM instance.
+      # local_hostname : Internal hostname assigned to the VM instance.
+      def fetch_local_attrs
+        
+      end
+    end
+
+    class StandAlone < Base
+      INSTANCE_ID='__standalone__'
+
+      def start_instance(attr)
+        # Nothing to be done
+        {:instance_id =>INSTANCE_ID}
+      end
+
+      def stop_instance(instance_id)
+        # Nothing to be done
+      end
+
+      def check_status(instance_id, expected_status=:online)
+        # Always running
+        expected_status == :online
+      end
+
+      def self.fetch_local_attrs
+        attrs = {:instance_id=>INSTANCE_ID, :local_ipv4=>'127.0.0.1', :local_hostname=>'localhost', :ami_id=>'localimage'}
+        attrs
+      end
+    end
+
+    class EC2 < Base
+
+      require 'EC2'
+
+      def initialize()
+        @ec2 = ::EC2::Base.new(:access_key_id => Wakame.config.aws_access_key, :secret_access_key => Wakame.config.aws_secret_key )
+      end
+      
+      def start_instance(image_id, attr={})
+        res = @ec2.run_instances(:image_id => image_id,
+                                 :availability_zone => attr[:availability_zone],
+                                 :group_id => attr[:security_groups],
+                                 :instance_type => attr[:instance_type],
+                                 :user_data => attr[:user_data]
+                                 )
+        {:instance_id => res.instancesSet.item[0].instanceId}
+      end
+
+      def stop_instance(instance_id)
+        @ec2.terminate_instances(:instance_id=>instance_id)
+      end
+
+      def check_status(instance_id, expected_status=:online)
+        res = @ec2.describe_instances(:instance_id => instance_id)
+
+        status = res.reservationSet.item[0].instancesSet.item[0].instanceState
+        # status is returned in a hash structure. i.e. {'name'=>'running', 'code'=>'16' }
+        Wakame.log.debug("VM (#{instance_id}) status: #{status['name']}")
+        return case status['name']
+               when "running"
+                 expected_status == :online
+               when "terminated"
+                 expected_status == :offline
+               when "rebooting"
+                 expected_status == :offline
+               when "starting"
+                 expected_status == :offline
+               when "pending"
+                 expected_status == :offline
+               else
+                 raise "Unknown status from AWS: #{status['name']}"
+               end
+      end
+
+      def associate_address(instance_id, ip_addr)
+        res = @ec2.associate_address(:instance_id=>instance_id, :public_ip=>ip_addr)
+        # {"requestId"=>"000ac66b-4a9c-43be-8176-b1b96ed6d4b7", "return"=>"true", "xmlns"=>"http://ec2.amazonaws.com/doc/2008-12-01/"}
+        res['return'] == 'true'
+      end
+      def disassociate_address(ip_addr)
+        res = @ec2.disassociate_address(:public_ip=>ip_addr)
+        # {"requestId"=>"2f38c8bb-4b1a-4df3-9f30-fa17317c89c4", "return"=>"true", "xmlns"=>"http://ec2.amazonaws.com/doc/2008-12-01/"}
+        res['return'] == 'true'
+      end
+
+      # volume
+      def describe_volume(vol_id)
+        res = @ec2.describe_volumes(:volume_id=>vol_id)
+        if res['volumeSet']['item'][0]['attachmentSet']
+          res['volumeSet']['item'][0]['attachmentSet']['item'][0]
+        else
+          res['volumeSet']['item'][0]
+        end
+      end
+# >> @ec2.attach_volume(:instance_id => 'i-1fa1cd76', :volume_id => "vol-1f927176", :device=>'/dev/sde')
+#      => {"attachTime"=>"2009-04-17T05:46:18.000Z", "status"=>"attaching", "device"=>"/dev/sde", "requestId"=>"0fd3797b-b4f9-476b-8cb2-3e7401c6fae2", "instanceId"=>"i-1fa1cd76", "volumeId"=>"vol-1f927176", "xmlns"=>"http://ec2.amazonaws.com/doc/2008-12-01/"}
+      def attach_volume(instance_id, vol_id, vol_dev)
+        res = @ec2.attach_volume(:instance_id=>instance_id, :volume_id=>vol_id, :device=>vol_dev)
+      end
+      def detach_volume(vol_id)
+        res = @ec2.detach_volume(:volume_id=>vol_id)
+      end
+
+      # volume
+# >> @ec2.describe_volumes(:volume_id => "vol-c58360ac")
+#      => {"volumeSet"=>{"item"=>[{"status"=>"available", "size"=>"1", "snapshotId"=>nil, "availabilityZone"=>"us-east-1a", "attachmentSet"=>nil, "createTime"=>"2009-04-16T09:56:01.000Z", "volumeId"=>"vol-c58360ac"}]}, "requestId"=>"0e6d0923-eba8-425a-b939-87c7fe8e835e", "xmlns"=>"http://ec2.amazonaws.com/doc/2008-12-01/"}
+      def describe_volume(vol_id)
+        res = @ec2.describe_volumes(:volume_id=>vol_id)
+        res['volumeSet']['item'][0]
+      end
+      def create_volume(availability_zone, size)
+        res = @ec2.create_volume(:availability_zone=>availability_zone, :size=>size)
+      end
+# >> @ec2.create_volume(:availability_zone=>"us-east-1b", :snapshot_id=>"snap-27c1324e")
+#      => {"status"=>"creating", "size"=>"1", "snapshotId"=>"snap-27c1324e", "requestId"=>"f3a0ddbf-9eb8-4594-b43e-8486459a0168", "availabilityZone"=>"us-east-1b", "createTime"=>"2009-04-17T05:44:58.000Z", "volumeId"=>"vol-1f927176", "xmlns"=>"http://ec2.amazonaws.com/doc/2008-12-01/"}
+      def create_volume_from_snapshot(availability_zone, snapshot_id)
+        res = @ec2.create_volume(:availability_zone=>availability_zone, :snapshot_id=>snapshot_id)
+      end
+      def delete_volume(vol_id)
+        res = @ec2.delete_volume(:volume_id=>vol_id)
+      end
+
+      # snapshot
+      def describe_snapshot(snapshot_id)
+        res = @ec2.describe_snapshots(:snapshot_id=>snapshot_id)
+        res['snapshotSet']['item'][0]
+      end
+
+# >> @ec2.create_snapshot(:volume_id => 'vol-c58360ac')
+#      => {"status"=>"pending", "snapshotId"=>"snap-18c13271", "requestId"=>"9d1d586a-44b7-4edd-b94a-aaccb54e888d", "progress"=>nil, "startTime"=>"2009-04-16T10:13:37.000Z", "volumeId"=>"vol-c58360ac", "xmlns"=>"http://ec2.amazonaws.com/doc/2008-12-01/"}
+      def create_snapshot(vol_id)
+        res = @ec2.create_snapshot(:volume_id=>vol_id)
+      end
+      def delete_snapshot(snapshot_id)
+        res = @ec2.delete_snapshot(:snapshot_id=>snapshot_id)
+      end
+
+      module MetadataService
+        def query_metadata_uri(key)
+          require 'open-uri'
+          open("http://169.254.169.254/2008-02-01/meta-data/#{key}") { |f|
+            return f.readline
+          }
+        end
+        module_function :query_metadata_uri
+        public :query_metadata_uri
+
+        def fetch_local_attrs
+          attrs = {}
+          %w[instance-id instance-type local-ipv4 local-hostname public-hostname public-ipv4 ami-id].each { |key|
+            rkey = key.tr('-', '_')
+            attrs[rkey.to_sym]=query_metadata_uri(key)
+          }
+          attrs
+        end
+        module_function :fetch_local_attrs
+        public :fetch_local_attrs
+        
+      end
+
+      include MetadataService
+    end
+
+  end
+end

data/lib/wakame.rb

@@ -0,0 +1,59 @@
+
+require 'rubygems'
+
+# For debugging
+Thread.abort_on_exception = true 
+
+module Wakame
+  require 'jeweler/version_helper'
+  VERSION=Jeweler::VersionHelper.new(File.expand_path('../', File.dirname(__FILE__))).to_s
+
+  autoload :Agent, 'wakame/agent'
+  autoload :Master, 'wakame/master'
+  autoload :Daemonize, 'wakame/daemonize'
+  autoload :Util, 'wakame/util'
+  autoload :Event, 'wakame/event'
+  autoload :Service, 'wakame/service'
+  autoload :Rule, 'wakame/rule'
+  autoload :Graph, 'wakame/graph'
+  autoload :Monitor, 'wakame/monitor'
+  autoload :Actor, 'wakame/actor'
+  autoload :Configuration, 'wakame/configuration'
+  autoload :Logger, 'wakame/logger'
+  autoload :Packets, 'wakame/packets'
+  autoload :AMQPClient, 'wakame/amqp_client'
+  autoload :EventDispatcher, 'wakame/event_dispatcher'
+  autoload :Scheduler, 'wakame/scheduler'
+  autoload :Command, 'wakame/command'
+  autoload :CommandQueue, 'wakame/command_queue'
+  autoload :Template, 'wakame/template'
+  autoload :Trigger, 'wakame/trigger'
+  autoload :Action, 'wakame/action'
+  autoload :RuleEngine, 'wakame/rule_engine'
+  #autoload :Initializer, 'wakame/initializer' # Do not autoload this class since the constant is used for the flag in bootstrap.
+
+  def gen_id(str=nil)
+    Util.gen_id(str)
+  end
+
+  def log
+    Logger.log
+  end
+
+  module_function :gen_id, :log
+
+  class << self
+    def config
+      Initializer.instance.configuration
+    end
+
+    def environment
+      config.environment
+    end
+
+    def new_(class_or_str)
+      Util.new_(class_or_str)
+    end
+  end
+
+end

data/tasks/ec2.rake

@@ -0,0 +1,127 @@
+#!/usr/bin/env ruby
+
+#require 'wakame'
+require 'timeout'
+
+#AWS_ACCESS_KEY=ENV['AMAZON_ACCESS_KEY_ID'] || Wakame.config.aws_access_key
+#AWS_SECRET_KEY=ENV['AMAZON_SECRET_ACCESS_KEY'] || Wakame.config.aws_secret_key
+AWS_ACCESS_KEY=ENV['AMAZON_ACCESS_KEY_ID']
+AWS_SECRET_KEY=ENV['AMAZON_SECRET_ACCESS_KEY']
+
+def create_ec2
+  require 'EC2'
+  ec2 = EC2::Base.new(:access_key_id =>AWS_ACCESS_KEY, :secret_access_key =>AWS_SECRET_KEY)
+end
+
+def request_metadata_url(key)
+  require 'open-uri'
+  open("http://169.254.169.254/2008-09-01/meta-data/#{key}") { |f|
+    return f.readline
+  }
+end
+
+namespace :ec2 do
+  desc "Automate the EC2 image bundle procedure.(ec2-bundle-vol + ec2-upload-bundle + ec2-register)"
+  task :bundle, :manifest_path do |t, args|
+    raise 'This task requires root privilege.' unless Process.uid == 0
+    raise 'Required key files counld not be detected: /mnt/cert.pem or /mnt/pk.pem'  unless File.exist?('/mnt/cert.pem') && File.exist?('/mnt/pk.pem')
+
+    #sh("/etc/init.d/rabbitmq-server stop") rescue puts $!
+
+    bundle_tmpdir='/mnt/wakame-bundle'
+    # If the arg was not set, it tries to overwrite the running image.
+    manifest_path= args.manifest_path || request_metadata_url('ami-manifest-path')
+
+    #manifest_path.sub!(/.manifest.xml\Z/, '')
+    if manifest_path =~ %r{\A([^/]+)/(.+)\.manifest\.xml\Z}
+      #s3bucket = manifest_path[0, manifest_path.index('/') - 1]
+      #manifest_prefix = manifest_path[manifest_path.index('/')]
+      s3bucket = $1
+      manifest_basename = File.basename($2)
+      manifest_path = "#{s3bucket}/#{manifest_basename}.manifest.xml"
+      #puts "#{manifest_path}"
+    else
+      fail "Given manifest path is not valid: #{manifest_path}"
+    end
+
+    puts "Manifest Path: #{manifest_path}"
+    
+    ec2 = create_ec2()
+
+    ami_id = request_metadata_url('ami-id')
+
+    instance_id = request_metadata_url('instance-id')
+    res = ec2.describe_instances(:instance_id=>instance_id)
+    account_no = res['reservationSet']['item'][0]['ownerId']
+
+    res = ec2.describe_images(:image_id=>ami_id)
+    arch = res['imagesSet']['item'][0]['architecture']
+
+    begin
+      FileUtils.rm_rf(bundle_tmpdir) if File.exist?(bundle_tmpdir)
+      FileUtils.mkpath(bundle_tmpdir)
+
+      sh("ec2-bundle-vol --batch -d '#{bundle_tmpdir}' -p '#{manifest_basename}' -c /mnt/cert.pem -k /mnt/pk.pem -u '#{account_no}' -r '#{arch}'")
+      sh("ec2-upload-bundle -d '#{bundle_tmpdir}' -b '#{s3bucket}' -m '#{File.join(bundle_tmpdir, manifest_basename + '.manifest.xml')}' -a '#{AWS_ACCESS_KEY}' -s '#{AWS_SECRET_KEY}'")
+      res = ec2.register_image(:image_location=>manifest_path)
+      puts "New AMI ID for #{manifest_path}: #{res['imageId']}"
+    ensure
+      FileUtils.rm_rf(bundle_tmpdir) if File.exist?(bundle_tmpdir)
+    end
+  end
+
+
+  desc "Initiate the mysql master volume using Amazon EBS"
+  task :mysqlsetupvol, :size do |t, args|
+    raise 'This task requires root privilege.' unless Process.uid == 0
+
+    ATTACH_DEV='/dev/sdw'
+    TMP_MNT='/mnt/mysql-tmp'
+    vol_size = (args.size || '1').to_s
+    zone = request_metadata_url('placement/availability-zone')
+    instance_id = request_metadata_url('instance-id')
+
+    ec2 = create_ec2
+    res = ec2.create_volume(:size=>vol_size, :availability_zone=>zone)
+    vol_id = res['volumeId']
+    
+    timeout(10) {
+      begin
+        res = ec2.describe_volumes(:volume_id=>vol_id)
+        next if res['volumeSet']['item'][0]['status'] == 'available'
+        sleep 0.5
+        retry
+      end
+    }
+
+    begin
+      res = ec2.attach_volume(:instance_id=>instance_id, :volume_id=>vol_id, :device=>ATTACH_DEV)
+      
+      timeout(10) {
+        begin
+          res = ec2.describe_volumes(:volume_id=>vol_id)
+          next if res['volumeSet']['item'][0]['status'] == 'in-use' && File.blockdev?(ATTACH_DEV)
+          sleep 0.5
+          retry
+        end
+      }
+      
+      sh("echo 'y' | mkfs.ext3 -q #{ATTACH_DEV}")
+      
+      begin
+        FileUtils.mkpath(TMP_MNT) unless File.exist?(TMP_MNT)
+        sh("mount #{ATTACH_DEV} #{TMP_MNT}")
+        sleep 1.0
+        sh("/usr/bin/mysql_install_db --datadir=#{TMP_MNT}")
+      ensure
+        sh("umount #{TMP_MNT}")
+        sleep 1.0
+      end
+
+    ensure
+      ec2.detach_volume(:volume_id=>vol_id, :instance_id=>instance_id) rescue puts $!
+    end
+
+    puts "Initialized EBS Volume: #{vol_id} "
+  end
+end

data/tests/cluster.json

@@ -0,0 +1,3 @@
+{
+
+}

data/tests/conf/a

@@ -0,0 +1 @@
+aaaaaa

data/tests/conf/b

@@ -0,0 +1 @@
+bbbbbbb

data/tests/conf/c

@@ -0,0 +1 @@
+ccccccc

data/tests/setup_agent.rb

@@ -0,0 +1,39 @@
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+WAKAME_ROOT="#{File.dirname(__FILE__)}/.."
+WAKAME_FRAMEWORK_ROOT="#{File.dirname(__FILE__)}/.."
+WAKAME_ENV=:StandAlone
+
+require 'rubygems' rescue nil
+
+require 'test/unit'
+
+require 'wakame'
+require 'wakame/initializer'
+
+#require "#{WAKAME_ROOT}/config/boot"
+#Wakame::Bootstrap.boot_agent!
+
+Wakame::Initializer.run(:process_agent)
+
+
+class DummyAgent
+  attr_reader :publish_count
+
+  attr_accessor :actor_registry, :monitor_registry
+
+  def initialize(&blk)
+    blk.call(self) if blk
+    @publish_count = 0
+  end
+  
+  def agent_id
+    'test_id'
+  end
+
+  def publish_to(*args)
+    @publish_count += 1
+  end
+
+end

data/tests/setup_master.rb

@@ -0,0 +1,28 @@
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+WAKAME_ROOT="#{File.dirname(__FILE__)}/.."
+WAKAME_FRAMEWORK_ROOT="#{File.dirname(__FILE__)}/.."
+WAKAME_ENV=:StandAlone
+
+require 'rubygems' rescue nil
+
+require 'test/unit'
+
+require 'wakame'
+require 'wakame/initializer'
+
+require 'ext/eventmachine'
+
+#require "#{WAKAME_ROOT}/config/boot"
+#Wakame::Bootstrap.boot_agent!
+
+#Wakame::Initializer.run(:process_master)
+Wakame::Initializer.run(:setup_load_paths)
+Wakame::Initializer.run(:setup_logger)
+      
+
+class MockCluster < Wakame::Service::ServiceCluster
+end
+
+Wakame.config.cluster_class = 'MockCluster'

data/tests/test_actor.rb

@@ -0,0 +1,54 @@
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+$:.unshift(File.dirname(__FILE__))
+
+require 'setup_agent.rb'
+
+require 'rubygems'
+
+require 'test/unit'
+
+require 'wakame'
+require 'wakame/packets'
+require 'wakame/agent'
+require 'wakame/actor'
+require 'wakame/actor/service_monitor'
+
+class TestActor < Test::Unit::TestCase
+  include Wakame
+
+
+  def test_actor_registry
+    reg = ActorRegistry.new
+    reg.register(Actor::ServiceMonitor.new)
+    assert(reg.actors.keys.include?('/wakame/actor/service_monitor'))
+    reg.unregister('/wakame/actor/service_monitor')
+    assert(reg.actors.size == 0)
+  end
+
+  def test_service_monitor
+    require 'wakame/monitor/service'
+
+    EM.run {
+      agent = DummyAgent.new { |me|
+        me.actor_registry = ActorRegistry.new
+        me.monitor_registry = MonitorRegistry.new
+        mon = Wakame::Monitor::Service.new
+        mon.agent = me
+        me.monitor_registry.register(mon, '/service')
+      }
+
+      d = Dispatcher.new(agent)
+      svcmon = Actor::ServiceMonitor.new
+      svcmon.agent = agent
+
+      agent.actor_registry.register(svcmon, '/service_monitor')
+      d.handle_request({:path=>'/service_monitor/register', :args=>['aaaa', 'ls -l /tmp']})
+      EM.next_tick {
+        assert(agent.monitor_registry.find_monitor('/service').checkers.keys.include?('aaaa'))
+        EM.stop
+      }
+    }
+  end
+
+end