tengine_resource 0.5.13

data/lib/tengine/resource/drivers/resource_control_driver.rb

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+require 'active_support/core_ext/hash/keys'
+
+# リソース制御ドライバ
+driver :resource_control_driver do
+
+  on :'仮想サーバ起動リクエストイベント' do
+    prop = event.properties.dup
+ 
+    raise "malformed event (packet corruption?), provider not found. #{event.inspect}" unless pid = prop.delete("provider_id")
+    provider = Tengine::Resource::Provider.find(pid)
+
+    name = provider._type
+    raise "logical bug or DB corruption, unknown class #{name}" unless klass = ObjectSpace.each_object(Class).select {|i| i.name == name }.first
+
+    [["physical_server", Tengine::Resource::PhysicalServer],
+     ["virtual_server_image", Tengine::Resource::VirtualServerImage],
+     ["virtual_server_type", Tengine::Resource::VirtualServerType],
+    ].each do |(i, j)|
+      if k = prop.delete("#{i}_id")
+        prop[i] = j.find(k)
+      end
+    end
+
+    provider.becomes(klass).create_virtual_servers prop.symbolize_keys
+  end
+
+  on :'仮想サーバ停止リクエストイベント' do
+    prop = event.properties.dup
+ 
+    raise "malformed event (packet corruption?), provider not found. #{event.inspect}" unless pid = prop.delete("provider_id")
+    provider = Tengine::Resource::Provider.find(pid)
+
+    name = provider._type
+    raise "logical bug or DB corruption, unknown class #{name}" unless klass = ObjectSpace.each_object(Class).select {|i| i.name == name }.first
+
+    raise "malformed event (packet corruption?), no server to stop. #{event.inspect}" unless servers = prop.delete("virtual_servers")
+
+    servers.map! do |i|
+      Tengine::Resource::VirtualServer.find(i)
+    end
+
+    provider.becomes(klass).terminate_virtual_servers servers
+  end
+
+  on :'Tengine::Resource::VirtualServer.created.tengine_resource_watchd'        # 仮想サーバ登録通知イベント
+  on :'Tengine::Resource::VirtualServer.updated.tengine_resource_watchd'        # 仮想サーバ変更通知イベント
+  on :'Tengine::Resource::VirtualServer.destroyed.tengine_resource_watchd'      # 仮想サーバ削除通知イベント
+  on :'Tengine::Resource::PhysicalServer.created.tengine_resource_watchd'       # 物理サーバ登録通知イベント
+  on :'Tengine::Resource::PhysicalServer.updated.tengine_resource_watchd'       # 物理サーバ変更通知イベント
+  on :'Tengine::Resource::PhysicalServer.destroyed.tengine_resource_watchd'     # 物理サーバ削除通知イベント
+  on :'Tengine::Resource::VirtualServerImage.created.tengine_resource_watchd'   # 仮想サーバイメージ登録通知イベント
+  on :'Tengine::Resource::VirtualServerImage.updated.tengine_resource_watchd'   # 仮想サーバイメージ変更通知イベント
+  on :'Tengine::Resource::VirtualServerImage.destroyed.tengine_resource_watchd' # 仮想サーバイメージ削除通知イベント
+  on :'Tengine::Resource::VirtualServerType.created.tengine_resource_watchd'    # 仮想サーバタイプ登録通知イベント
+  on :'Tengine::Resource::VirtualServerType.updated.tengine_resource_watchd'    # 仮想サーバタイプ変更通知イベント
+  on :'Tengine::Resource::VirtualServerType.destroyed.tengine_resource_watchd'  # 仮想サーバタイプ削除通知イベント
+end

data/lib/tengine/resource/net_ssh.rb

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+require 'tempfile'
+require 'net/ssh'
+require 'active_support/core_ext/hash/keys'
+
+class << Net::SSH
+  alias __tengine_resource_net_ssh_backed_up_start__ start
+  private :__tengine_resource_net_ssh_backed_up_start__
+
+  # Extended Net::SSH.start
+  #
+  # SYNOPSIS:
+  #
+  #  A:  Net::SSH.start(hostname, credential) {|ctx| ... }
+  #  B:  Net::SSH.start(hostname, user, credential) {|ctx| ... }
+  #  C:  Net::SSH.start(hostname, credential, other_opts) {|ctx| ... }
+  #  D:  Net::SSH.start(hostname, user, other_opts) {|ctx| ... }
+  #  E:  Net::SSH.start(hostname, other_opts) {|ctx| ... }
+  #
+  # ARGUMENTS:
+  #
+  # @param [String]                        hostname   Secure Shell host to connect to.
+  # @param [String]                        user       Account to use.
+  # @param [Tengine::Resource::Credential] credential Credential info.
+  # @param [Hash]                          other_opts Keyword arguments.
+  #
+  # DESCRIPTION:
+  #
+  # Orchestate those given arguments to start SSH connection.
+  #
+  # RETURNS:
+  #
+  # Yields or returns a Net::SSH::Connection.
+  #
+  def start host, obj1, obj2=nil, &block
+    user = nil
+    hash = nil
+
+    case obj1
+    when String
+      user = obj1
+    when Hash
+      hash = obj1.symbolize_keys
+    when Tengine::Resource::Credential
+      hash = obj1.auth_values.symbolize_keys
+    else
+      raise TypeError, "#{obj1.class} not expected (expected String)"
+    end
+
+    hash ||= Hash.new
+    case obj2
+    when NilClass
+      # OK, takes nothing
+    when Hash
+      hash.merge!(obj2.symbolize_keys) {|k, v1, v2|
+        raise ArgumentError, "#{k} specified twice in both credential and hash arguments"
+      }
+    when Tengine::Resource::Credential
+      hash.merge! obj2.auth_values.symbolize_keys {|k, v1, v2|
+        raise ArgumentError, "#{k} specified twice in both credential and hash arguments"
+      }
+    else
+      raise TypeError, "#{obj1.class} not expected (expected #{String})"
+    end
+
+    u2 = hash.delete(:username)
+    k2 = hash.delete(:private_keys)
+
+    raise ArgumentError, "username specified twice in both ordinal and optional arguments" if user and u2
+    user ||= u2
+    raise ArgumentError, "username mandatory" unless user
+
+    argh = %w[
+      auth_methods
+      compression
+      compression_level
+      config
+      encryption
+      forward_agent
+      global_known_hosts_file
+      hmac
+      host_key
+      host_key_alias
+      host_name
+      kex
+      keys
+      key_data
+      keys_only
+      logger
+      paranoid
+      passphrase
+      password
+      port
+      properties
+      proxy
+      rekey_blocks_limit
+      rekey_limit
+      rekey_packet_limit
+      timeout
+      user
+      user_known_hosts_file
+      verbose
+    ].map(&:intern).inject(Hash.new) do |r, k|
+      r[k] = hash.delete(k) if hash.has_key? k
+      r
+    end
+
+    raise ArgumentError, "unknown optional argument(s): #{hash.keys.join(', ')}" unless hash.empty?
+
+    if k2
+      k2 = [k2] unless k2.is_a? Array
+      Dir.mktmpdir(nil, File.expand_path("../../../../tmp", __FILE__)) do |dir|
+        begin
+          k3 = k2.map do |k|
+            fp = Tempfile.new("pk", dir)
+            fp.write(k)
+            fp.chmod(0400)
+            fp.flush
+            fp # no close
+          end
+          k4 = k3.map {|i| File.expand_path(i.path) }
+          argh[:keys] ||= []
+          argh[:keys].concat k4
+
+          return __tengine_resource_net_ssh_backed_up_start__ host, user, argh, &block
+        ensure
+          k3.each {|i| i.close(:real) }
+        end
+      end
+    else
+      return __tengine_resource_net_ssh_backed_up_start__ host, user, argh, &block
+    end
+  end
+end

data/lib/tengine/resource/observer.rb

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+require 'mongoid'
+
+require 'yaml'
+require 'tengine_event'
+require 'tengine/support/yaml_with_erb'
+
+# http://mongoid.org/docs/callbacks/observers.html
+class Tengine::Resource::Observer < Mongoid::Observer
+  include Tengine::Event::ModelNotifiable
+
+  prefix = "tengine/resource/"
+  observe *%w[physical_server virtual_server virtual_server_image virtual_server_type].map{|name| :"#{prefix}#{name}" }
+
+  def event_sender
+    @event_sender = Tengine::Event.default_sender
+  end
+
+  SUFFIX = "tengine_resource_watchd".freeze
+
+  def event_type_name_suffix
+    SUFFIX
+  end
+
+end

data/lib/tengine/resource/physical_server.rb

@@ -0,0 +1,7 @@
+class Tengine::Resource::PhysicalServer < Tengine::Resource::Server
+  field :cpu_cores, :type => Integer
+  field :memory_size, :type => Integer
+
+  belongs_to :provider, :index => true, :inverse_of => :physical_servers,
+    :class_name => "Tengine::Resource::Provider"
+end

data/lib/tengine/resource/provider.rb

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+require 'mongoid'
+
+class Tengine::Resource::Provider
+  autoload :Ec2,    'tengine/resource/provider/ec2'
+  autoload :Wakame, 'tengine/resource/provider/wakame'
+
+  include Mongoid::Document
+  include Mongoid::Timestamps
+  include Tengine::Core::Validation
+  include Tengine::Core::FindByName
+
+  field :name,             :type => String
+  field :description,      :type => String
+  field :polling_interval, :type => Integer, :default => 10   # プロバイダへの問い合わせ間隔
+  field :retry_interval,   :type => Integer, :default => 10   # プロバイダへの問い合わせリトライ間隔
+  field :retry_count,      :type => Integer, :default => 30   # プロバイダへの問い合わせリトライ回数
+  field :properties,       :type => Hash,    :default => {}
+
+  validates :name, :presence => true, :uniqueness => true, :format => BASE_NAME.options
+  index :name, :unique => true
+
+  with_options(:inverse_of => :provider, :dependent => :destroy) do |c|
+    c.has_many :physical_servers       , :class_name => "Tengine::Resource::PhysicalServer"
+    c.has_many :virtual_servers        , :class_name => "Tengine::Resource::VirtualServer"
+    c.has_many :virtual_server_images  , :class_name => "Tengine::Resource::VirtualServerImage"
+    c.has_many :virtual_server_types   , :class_name => "Tengine::Resource::VirtualServerType"
+  end
+
+  # 仮想サーバタイプの監視
+  def virtual_server_type_watch    ; raise NotImplementedError end
+  # 物理サーバの監視
+  def physical_server_watch        ; raise NotImplementedError end
+  # 仮想サーバの監視
+  def virtual_server_watch         ; raise NotImplementedError end
+  # 仮想サーバイメージの監視
+  def virtual_server_image_watch   ; raise NotImplementedError end
+
+  def update_physical_servers      ; raise NotImplementedError end
+  def update_virtual_servers       ; raise NotImplementedError end
+  def update_virtual_server_imagess; raise NotImplementedError end
+
+  private
+  def update_physical_servers_by(hashs)
+    found_ids = []
+    hashs.each do |hash|
+      server = self.physical_servers.where(:provided_id => hash[:provided_id]).first
+      if server
+        server.update_attributes(:status => hash[:status])
+      else
+        server = self.physical_servers.create!(
+          :provided_id => hash[:provided_id],
+          :name => hash[:name],
+          :status => hash[:status])
+      end
+      found_ids << server.id
+    end
+    self.physical_servers.not_in(:_id => found_ids).update_all(:status => "not_found")
+  end
+
+  def update_virtual_servers_by(hashs)
+    found_ids = []
+    hashs.each do |hash|
+      server = self.virtual_servers.where(:provided_id => hash[:provided_id]).first
+      if server
+        server.update_attributes(hash)
+      else
+        server = self.virtual_servers.create!(hash.merge(:name => hash[:provided_id]))
+      end
+      found_ids << server.id
+    end
+    self.virtual_servers.not_in(:_id => found_ids).destroy_all
+  end
+
+  class << self
+    def find_or_create_by_name!(attrs)
+      result = self.first(:conditions => {:name => attrs[:name]})
+      result ||= self.create!(attrs)
+      result
+    end
+  end
+end

data/lib/tengine/resource/provider/ec2.rb

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+class Tengine::Resource::Provider::Ec2 < Tengine::Resource::Provider
+
+  field :connection_settings, :type => Hash
+
+  def update_physical_servers
+    connect do |conn|
+      # ec2.describe_availability_zones  #=> [{:region_name=>"us-east-1",
+      #                                        :zone_name=>"us-east-1a",
+      #                                        :zone_state=>"available"}, ... ]
+      # http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeAvailabilityZones.html
+      hashs = conn.describe_availability_zones.map do |hash|
+        {
+          :provided_id => hash[:zone_name],
+          :name    => hash[:zone_name],
+          :status => hash[:zone_state],
+        }
+      end
+      update_physical_servers_by(hashs)
+    end
+  end
+
+  def update_virtual_servers
+    connect do |conn|
+      # http://rightscale.rubyforge.org/right_aws_gem_doc/
+      # ec2.describe_instances #=>
+      #   [{:aws_image_id       => "ami-e444444d",
+      #     :aws_reason         => "",
+      #     :aws_state_code     => "16",
+      #     :aws_owner          => "000000000888",
+      #     :aws_instance_id    => "i-123f1234",
+      #     :aws_reservation_id => "r-aabbccdd",
+      #     :aws_state          => "running",
+      #     :dns_name           => "domU-12-34-67-89-01-C9.usma2.compute.amazonaws.com",
+      #     :ssh_key_name       => "staging",
+      #     :aws_groups         => ["default"],
+      #     :private_dns_name   => "domU-12-34-67-89-01-C9.usma2.compute.amazonaws.com",
+      #     :aws_instance_type  => "m1.small",
+      #     :aws_launch_time    => "2008-1-1T00:00:00.000Z"},
+      #     :aws_availability_zone => "us-east-1b",
+      #     :aws_kernel_id      => "aki-ba3adfd3",
+      #     :aws_ramdisk_id     => "ari-badbad00",
+      #      ..., {...}]
+      # http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/index.html?ApiReference-query-DescribeInstances.html
+      hashs = conn.describe_instances.map do |hash|
+        result = {
+          :provided_id => hash.delete(:aws_instance_id),
+          :provided_image_id => hash.delete(:aws_image_id),
+          :status => hash.delete(:aws_state),
+        }
+        hash.delete(:aws_state_code)
+        result[:properties] = hash
+        result[:addresses] = {
+          :dns_name        => hash.delete(:dns_name),
+          :ip_address      => hash.delete(:ip_address),
+          :private_dns_name => hash.delete(:private_dns_name),
+          :private_ip_address => hash.delete(:private_ip_address),
+        }
+        result
+      end
+      update_virtual_servers_by(hashs)
+    end
+  end
+
+  def update_virtual_server_images
+    connect do |conn|
+      hashs = conn.describe_images.map do |hash|
+        { :provided_id => hash.delete(:aws_id), }
+      end
+      update_virtual_server_images_by(hashs)
+    end
+  end
+
+  # @param  [String]                                 name         Name template for created virtual servers
+  # @param  [Tengine::Resource::VirtualServerImage]  image        Virtual server image object
+  # @param  [Tengine::Resource::VirtualServerType]   type         Virtual server type object
+  # @param  [String]                                 physical     Data center name to put virtual machines (availability zone)
+  # @param  [String]                                 description  What this virtual server is
+  # @param  [Numeric]                                min_count    Minimum number of vortial servers to boot
+  # @param  [Numeric]                                max_count    Maximum number of vortial servers to boot
+  # @param  [Array<Strng>]                           group_ids    Array of names of security group IDs
+  # @param  [Strng]                                  key_name     Name of root key to sue
+  # @param  [Strng]                                  user_data    User-specified
+  # @param  [Strng]                                  kernel_id    Kernel image ID
+  # @param  [Strng]                                  ramdisk_id   Ramdisk image ID
+  # @return [Array<Tengine::Resource::VirtualServer>]
+  def create_virtual_servers name, image, type, physical, description, min_count, max_count, group_ids, key_name, user_data = "", kernel_id, ramdisk_id
+    connect {|conn|
+      results = conn.run_instances(
+        image.provided_id,
+        min_count,
+        max_count,
+        group_ids,
+        key_name,
+        user_data,
+        nil, # <- addressing_type
+        type.provided_id,
+        kernel_id,
+        ramdisk_id,
+        physical,
+        nil  # <- block_device_mappings
+      )
+      yield if block_given? # テスト用のブロックの呼び出し
+      results.map.with_index {|hash, idx|
+        provided_id = hash.delete(:aws_instance_id)
+        if server = self.virtual_servers.find(:first, :conditions => {:provided_id => provided_id})
+          server
+        else
+          host_server_provided_id = hash[:aws_availability_zone]
+          host_server_provided_id = physical if host_server_provided_id.nil? || host_server_provided_id.blank?
+          # findではなくfirstで検索しているので、もしhost_server_provided_idで指定されるサーバが見つからなくても
+          # host_serverがnilとして扱われるが、仮想サーバ自身の登録は行われます
+          host_server = (host_server_provided_id && !host_server_provided_id.blank?) ?
+            Tengine::Resource::PhysicalServer.first(:conditions => {:provided_id => host_server_provided_id}) : nil
+          begin
+            self.virtual_servers.create!(
+              :name                 => sprintf("%s%03d", name, idx + 1), # 1 origin
+              :address_order        => address_order,
+              :description          => description,
+              :provided_id          => provided_id,
+              :provided_image_id    => hash.delete(:aws_image_id),
+              :provided_type_id     => hash.delete(:aws_instance_type),
+              :host_server_id       => host_server ? host_server.id : nil,
+              :status               => hash.delete(:aws_state),
+              :properties           => hash,
+              :addresses            => {
+    #             :dns_name           => hash.delete(:dns_name),
+    #             :ip_address         => hash.delete(:ip_address),
+    #             :private_dns_name   => hash.delete(:private_dns_name),
+    #             :private_ip_address => hash.delete(:private_ip_address),
+              })
+          rescue Mongo::OperationFailure => e
+            raise e unless e.message =~ /E11000 duplicate key error/
+            self.virtual_servers.find(:first, :conditions => {:provided_id => provided_id}) or
+              raise "VirtualServer not found for #{provided_id}"
+          rescue Mongoid::Errors::Validations => e
+            raise e unless e.document.errors[:provided_id].any?{|s| s =~ /taken/}
+            self.virtual_servers.find(:first, :conditions => {:provided_id => provided_id}) or
+              raise "VirtualServer not found for #{provided_id}"
+          end
+        end
+      }
+    }
+  end
+
+  def terminate_virtual_servers servers
+    connect do |conn|
+      # http://rightscale.rubyforge.org/right_aws_gem_doc/classes/RightAws/Ec2.html#M000287
+      # http://docs.amazonwebservices.com/AWSEC2/latest/APIReference/ApiReference-query-TerminateInstances.html
+      conn.terminate_instances(servers.map {|i| i.provided_id }).map do |hash|
+        serv = self.virtual_servers.where(:provided_id => hash[:aws_instance_id]).first
+        serv.update_attributes(:status => hash[:aws_current_state_name]) if serv
+        serv
+      end
+    end
+  end
+
+  # 仮想サーバタイプの監視
+  def virtual_server_type_watch
+    # ec2から取得する情報はありません
+  end
+
+  # 物理サーバの監視
+  def physical_server_watch        ; raise NotImplementedError end
+  # 仮想サーバの監視
+  def virtual_server_watch         ; raise NotImplementedError end
+  # 仮想サーバイメージの監視
+  def virtual_server_image_watch   ; raise NotImplementedError end
+
+  private
+  def address_order
+    @@address_order ||= %w"private_ip_address private_dns_name ip_address dns_name".each(&:freeze).freeze
+  end
+
+  def connect
+    klass = (ENV['EC2_DUMMY'] == "true") ? Tengine::Resource::Credential::Ec2::Dummy : RightAws::Ec2
+    connection = klass.new(
+      self.connection_settings[:access_key],
+      self.connection_settings[:secret_access_key],
+      {
+        :logger => Tengine.logger,
+        :region => self.connection_settings[:region]
+      }
+      )
+    yield connection
+  end
+end