tengine_resource 0.5.15 → 1.1.0

data/Gemfile

@@ -1,30 +1,4 @@
 # -*- coding: utf-8 -*-
 source "http://rubygems.org"
 
-# Add dependencies required to use your gem here.
-# Example:
-#   gem "activesupport", ">= 2.3.5"
-
-gem "tengine_support", "~> 0.3.27"
-gem "tengine_core", "~> 0.5.39"
-
-gem "wakame-adapters-tengine", "~> 0.0.0"
-
-# http://rightaws.rubyforge.org/
-# http://github.com/rightscale/right_aws
-gem "right_aws", "~> 2.1.0"
-
-gem "net-ssh", "~> 2.2.1"
-
-# Add dependencies to develop your gem here.
-# Include everything needed to run rake, tests, features, etc.
-group :development do
-  gem "rspec", "~> 2.6.0"
-  gem "factory_girl", "~> 2.1.2"
-  gem "yard", "~> 0.7.2"
-  gem "bundler", "~> 1.1.3"
-  gem "jeweler", "~> 1.6.4"
-  # gem "rcov", ">= 0"
-  gem "simplecov", "~> 0.5.3"
-  gem "ZenTest", "~> 4.6.2"
-end
+gemspec

data/Gemfile.lock

@@ -1,12 +1,20 @@
+PATH
+  remote: .
+  specs:
+    tengine_resource (1.1.0)
+      net-ssh (~> 2.5.2)
+      tengine_core (~> 1.1.0)
+      tengine_support (~> 1.1.0)
+
 GEM
   remote: http://rubygems.org/
   specs:
-    ZenTest (4.6.2)
-    activemodel (3.1.4)
-      activesupport (= 3.1.4)
+    ZenTest (4.8.2)
+    activemodel (3.2.9)
+      activesupport (= 3.2.9)
       builder (~> 3.0.0)
+    activesupport (3.2.9)
       i18n (~> 0.6)
-    activesupport (3.1.4)
       multi_json (~> 1.0)
     amq-client (0.8.7)
       amq-protocol (>= 0.8.4)
@@ -16,91 +24,70 @@ GEM
       amq-client (~> 0.8.7)
       amq-protocol (~> 0.8.4)
       eventmachine
-    bson (1.5.2)
-    bson_ext (1.5.2)
-      bson (= 1.5.2)
-    builder (3.0.0)
-    daemons (1.1.8)
+    autotest (4.4.6)
+      ZenTest (>= 4.4.1)
+    builder (3.0.4)
+    daemons (1.1.9)
     diff-lcs (1.1.3)
-    eventmachine (0.12.10)
-    factory_girl (2.1.2)
-      activesupport
-    git (1.2.5)
-    i18n (0.6.0)
-    jeweler (1.6.4)
-      bundler (~> 1.0)
-      git (>= 1.2.5)
-      rake
-    json (1.6.5)
-    macaddr (1.5.0)
-      systemu (>= 2.4.0)
-    mongo (1.5.2)
-      bson (= 1.5.2)
-    mongoid (2.3.5)
-      activemodel (~> 3.1.0)
-      mongo (~> 1.3)
+    eventmachine (1.0.0)
+    i18n (0.6.1)
+    macaddr (1.6.1)
+      systemu (~> 2.5.0)
+    mongoid (3.0.15)
+      activemodel (~> 3.1)
+      moped (~> 1.1)
+      origin (~> 1.0)
       tzinfo (~> 0.3.22)
-    multi_json (1.0.4)
-    net-ssh (2.2.2)
+    moped (1.3.1)
+    multi_json (1.3.6)
+    net-ssh (2.5.2)
+    origin (1.0.11)
     rake (0.9.2.2)
-    right_aws (2.1.0)
-      right_http_connection (>= 1.2.5)
-    right_http_connection (1.3.0)
-    rspec (2.6.0)
-      rspec-core (~> 2.6.0)
-      rspec-expectations (~> 2.6.0)
-      rspec-mocks (~> 2.6.0)
-    rspec-core (2.6.4)
-    rspec-expectations (2.6.0)
-      diff-lcs (~> 1.1.2)
-    rspec-mocks (2.6.0)
-    selectable_attr (0.3.15)
+    rdiscount (1.6.8)
+    rspec (2.10.0)
+      rspec-core (~> 2.10.0)
+      rspec-expectations (~> 2.10.0)
+      rspec-mocks (~> 2.10.0)
+    rspec-core (2.10.1)
+    rspec-expectations (2.10.0)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.10.1)
+    selectable_attr (0.3.17)
       i18n
-    simplecov (0.5.4)
-      multi_json (~> 1.0.3)
+    simplecov (0.6.4)
+      multi_json (~> 1.0)
       simplecov-html (~> 0.5.3)
     simplecov-html (0.5.3)
-    systemu (2.5.0)
-    tengine_core (0.5.39)
-      activemodel (~> 3.1.0)
-      activesupport (~> 3.1.0)
-      bson (~> 1.5.2)
-      bson_ext (~> 1.5.2)
+    systemu (2.5.2)
+    tengine_core (1.1.0)
+      activemodel (>= 3.1.0)
+      activesupport (>= 3.1.0)
       daemons (~> 1.1.4)
-      mongo (~> 1.5.2)
-      mongoid (~> 2.3.3)
+      mongoid (~> 3.0.0)
       selectable_attr (~> 0.3.15)
-      tengine_event (~> 0.4.9)
-      tengine_support (~> 0.3.27)
-    tengine_event (0.4.9)
+      tengine_event (~> 1.1.0)
+      tengine_support (~> 1.1.0)
+    tengine_event (1.1.0)
       activesupport (>= 3.0.0)
       amqp (~> 0.8.0)
-      tengine_support (>= 0.3.27)
+      tengine_support (~> 1.1.0)
       uuid (~> 2.3.4)
-    tengine_support (0.3.27)
+    tengine_support (1.1.0)
       activesupport (>= 3.0.0)
-    tzinfo (0.3.33)
+    tzinfo (0.3.35)
     uuid (2.3.5)
       macaddr (~> 1.0)
-    wakame-adapters-tengine (0.0.1)
-      json
-      right_aws (= 2.1.0)
-      rspec
-    yard (0.7.5)
+    yard (0.8.2.1)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  ZenTest (~> 4.6.2)
-  bundler (~> 1.1.3)
-  factory_girl (~> 2.1.2)
-  jeweler (~> 1.6.4)
-  net-ssh (~> 2.2.1)
-  right_aws (~> 2.1.0)
-  rspec (~> 2.6.0)
-  simplecov (~> 0.5.3)
-  tengine_core (~> 0.5.39)
-  tengine_support (~> 0.3.27)
-  wakame-adapters-tengine (~> 0.0.0)
-  yard (~> 0.7.2)
+  autotest
+  bundler
+  rake (~> 0.9.2.2)
+  rdiscount
+  rspec (~> 2.10.0)
+  simplecov (~> 0.6.4)
+  tengine_resource!
+  yard (~> 0.8.1)

data/README.rdoc

@@ -3,7 +3,7 @@
 Description goes here.
 
 == Contributing to tengine_resource
- 
+
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
 * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
 * Fork the project
@@ -13,7 +13,7 @@ Description goes here.
 * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
 
 == License
-tengine_event is distributed under the MPL2.0 or LGPLv3 or the dual license of MPL2.0/LGPLv3
+tengine_resource is distributed under the MPL2.0 or LGPLv3 or the dual license of MPL2.0/LGPLv3
 
 == Copyright
 

data/bin/create_indexes_for_tengine_resource

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+# Use to create indexes. Sample is below.
+#  create_indexes_for_tengine_resource localhost:27017/tengine_production
+#
+# Default options
+#  hostname localhost
+#  port     27017
+#  database tengine_production
+#
+require 'tengine_support'
+require 'tengine_resource'
+
+include Tengine::Support::Mongoid
+
+target = File.expand_path('../lib', File.dirname(__FILE__))
+Tengine::Support::Mongoid.create_indexes(target, ARGV.shift)

data/lib/tengine/resource/config/resource.rb

@@ -4,13 +4,10 @@ require 'tengine/resource/config'
 
 require 'yaml'
 require 'optparse'
-require 'active_support/memoizable'
 
 require 'tengine/support/yaml_with_erb'
 
 class Tengine::Resource::Config::Resource < Tengine::Support::Config::Definition::Suite
-  # memoize については http://wota.jp/ac/?date=20081025#p11 などを参照してください
-  extend ActiveSupport::Memoizable
 
   class << self
     def default_hash

data/lib/tengine/resource/credential.rb

@@ -3,8 +3,6 @@ require 'mongoid'
 require 'selectable_attr'
 
 class Tengine::Resource::Credential
-  autoload :Ec2, 'tengine/resource/credential/ec2'
-
   include Mongoid::Document
   include Mongoid::Timestamps
   include Tengine::Core::SelectableAttr
@@ -24,22 +22,23 @@ class Tengine::Resource::Credential
   # EC2での認証については以下などを参照してください。
   #     http://builder.japan.zdnet.com/member/u502383/blog/2008/08/08/entry_27012840/
   selectable_attr :auth_type_cd do
-    entry "01", :ssh_password  , "SSHパスワード認証"   , :for_launch => false
-    entry "02", :ssh_public_key, "SSH公開鍵認証"       , :for_launch => false
-    # entry "03", :ec2_access_key, "EC2 アクセスキー認証", :for_launch => true
+    entry "01", :ssh_password  , "SSHパスワード認証"
+    entry "02", :ssh_public_key, "SSH公開鍵認証"
+    # entry "03", :ec2_access_key, "EC2 アクセスキー認証"
     # entry "04", :ec2_x509_cert, "EC2 X.509認証"
-    # entry "05", :tama, "Tama", :for_launch => true
+    # entry "05", :tama, "Tama"
+    entry "06", :ssh_public_key_file, "SSH公開鍵認証(ファイル)"
   end
 
   validates :name, :presence => true, :uniqueness => true, :format => BASE_NAME.options
   validates :auth_type_cd, :presence => true
 
-  index :name, :unique => true
+  index({ name: 1 }, { unique: true })
 
-  index([ [:_id, Mongo::ASCENDING], [:auth_type_cd, Mongo::ASCENDING], ])
-  index([ [:_id, Mongo::ASCENDING], [:auth_type_cd, Mongo::DESCENDING], ])
-  index([ [:_id, Mongo::ASCENDING], [:description, Mongo::ASCENDING], ])
-  index([ [:_id, Mongo::ASCENDING], [:description, Mongo::DESCENDING], ])
+  index _id: 1, auth_type_cd: 1
+  index _id: 1, auth_type_cd: -1
+  index _id: 1, description: 1
+  index _id: 1, description: -1
 
   before_validation :prepare_auth_values_default # auth_valuesの各値がnilならデフォルト値を設定します
   validate{|c| c.validate_auth_values}
@@ -98,11 +97,11 @@ class Tengine::Resource::Credential
       AuthField.new(:passphrase  , :secret, :optional => true),
     ].freeze,
 
-    # {:access_key => "xxxxx", :secret_access_key =>"xxxxx"}
-    :ec2_access_key => [
-      AuthField.new(:access_key, :string),
-      AuthField.new(:secret_access_key, :string),
-      AuthField.new(:default_region, :string, :default => "us-east-1"),
+    # {:username => "goku", :private_key_file =>"xxx", :passphrase => "xxxx"}
+    :ssh_public_key_file => [
+      AuthField.new(:username        , :string),
+      AuthField.new(:private_key_file, :string),
+      AuthField.new(:passphrase      , :secret, :optional => true),
     ].freeze,
   }.freeze
 
@@ -147,7 +146,7 @@ class Tengine::Resource::Credential
 
   class << self
     def find_or_create_by_name!(attrs = {}, &block)
-      result = self.first(:conditions => {:name => attrs[:name]})
+      result = self.where({:name => attrs[:name]}).first
       result ||= self.create!(attrs)
       result
     end

data/lib/tengine/resource/net_ssh.rb

@@ -42,6 +42,8 @@ class << Net::SSH
     when Hash
       hash = obj1.symbolize_keys
     when Tengine::Resource::Credential
+      # obj1がCredentialでobj2が指定されていない場合は、objがinvalid?ならエラー
+      raise ArgumentError, obj1.errors.full_messages.join if !obj1.valid? && obj2.nil?
       hash = obj1.auth_values.symbolize_keys
     else
       raise TypeError, "#{obj1.class} not expected (expected String)"
@@ -65,6 +67,7 @@ class << Net::SSH
 
     u2 = hash.delete(:username)
     k2 = hash.delete(:private_keys)
+    f2 = hash.delete(:private_key_file)
 
     raise ArgumentError, "username specified twice in both ordinal and optional arguments" if user and u2
     user ||= u2
@@ -107,28 +110,37 @@ class << Net::SSH
 
     raise ArgumentError, "unknown optional argument(s): #{hash.keys.join(', ')}" unless hash.empty?
 
-    if k2
-      k2 = [k2] unless k2.is_a? Array
+    assign_options(argh, k2, f2) do
+      return __tengine_resource_net_ssh_backed_up_start__(host, user, argh, &block)
+    end
+  end
+
+  def assign_options(options, private_keys, private_key_file)
+    if private_keys
+      private_keys = [private_keys] unless private_keys.is_a?(Array)
       Dir.mktmpdir(nil, File.expand_path("../../../../tmp", __FILE__)) do |dir|
         begin
-          k3 = k2.map do |k|
+          pk_files = private_keys.map do |k|
             fp = Tempfile.new("pk", dir)
             fp.write(k)
             fp.chmod(0400)
             fp.flush
             fp # no close
           end
-          k4 = k3.map {|i| File.expand_path(i.path) }
-          argh[:keys] ||= []
-          argh[:keys].concat k4
-
-          return __tengine_resource_net_ssh_backed_up_start__ host, user, argh, &block
+          options[:keys] ||= []
+          options[:keys].concat(pk_files.map {|i| File.expand_path(i.path) })
+          yield
         ensure
-          k3.each {|i| i.close(:real) }
+          pk_files.each {|i| i.close(:real) }
         end
       end
+    elsif private_key_file
+      options[:keys] ||= []
+      options[:keys] << private_key_file
+      yield
     else
-      return __tengine_resource_net_ssh_backed_up_start__ host, user, argh, &block
+      yield
     end
   end
+
 end

data/lib/tengine/resource/physical_server.rb

@@ -4,4 +4,12 @@ class Tengine::Resource::PhysicalServer < Tengine::Resource::Server
 
   belongs_to :provider, :index => true, :inverse_of => :physical_servers,
     :class_name => "Tengine::Resource::Provider"
+
+  class << self
+    def by_provided_id(provided_id)
+      return nil if provided_id.blank?
+      where({:provided_id => provided_id}).first
+    end
+
+  end
 end

data/lib/tengine/resource/provider.rb

@@ -2,9 +2,6 @@
 require 'mongoid'
 
 class Tengine::Resource::Provider
-  autoload :Ec2,    'tengine/resource/provider/ec2'
-  autoload :Wakame, 'tengine/resource/provider/wakame'
-
   include Mongoid::Document
   include Mongoid::Timestamps
   include Tengine::Core::Validation
@@ -18,7 +15,7 @@ class Tengine::Resource::Provider
   field :properties,       :type => Hash,    :default => {}
 
   validates :name, :presence => true, :uniqueness => true, :format => BASE_NAME.options
-  index :name, :unique => true
+  index({ name: 1 }, { unique: true })
 
   with_options(:inverse_of => :provider, :dependent => :destroy) do |c|
     c.has_many :physical_servers       , :class_name => "Tengine::Resource::PhysicalServer"
@@ -28,55 +25,229 @@ class Tengine::Resource::Provider
   end
 
   # 仮想サーバタイプの監視
-  def virtual_server_type_watch    ; raise NotImplementedError end
+  def synchronize_virtual_server_types    ; raise NotImplementedError end
   # 物理サーバの監視
-  def physical_server_watch        ; raise NotImplementedError end
+  def synchronize_physical_servers        ; raise NotImplementedError end
   # 仮想サーバの監視
-  def virtual_server_watch         ; raise NotImplementedError end
+  def synchronize_virtual_servers         ; raise NotImplementedError end
   # 仮想サーバイメージの監視
-  def virtual_server_image_watch   ; raise NotImplementedError end
+  def synchronize_virtual_server_images   ; raise NotImplementedError end
 
   def update_physical_servers      ; raise NotImplementedError end
   def update_virtual_servers       ; raise NotImplementedError end
   def update_virtual_server_imagess; raise NotImplementedError end
 
+
+  def find_virtual_server_on_duplicaion_error(virtual_server_provided_id)
+    begin
+      yield
+    rescue Moped::Errors::OperationFailure => e
+      raise e unless e.message =~ /E11000 duplicate key error/
+      self.virtual_servers.where({:provided_id => virtual_server_provided_id}).first or
+        raise "VirtualServer not found for #{virtual_server_provided_id}"
+    rescue Mongoid::Errors::Validations => e
+      raise e unless e.document.errors[:provided_id].any?{|s| s =~ /taken/}
+      self.virtual_servers.where({:provided_id => virtual_server_provided_id}).first or
+        raise "VirtualServer not found for #{virtual_server_provided_id}"
+    end
+  end
+
+  class << self
+    def find_or_create_by_name!(attrs)
+      result = self.where({:name => attrs[:name]}).first
+      result ||= self.create!(attrs)
+      result
+    end
+
+    def synchronizers
+      @synchronizers ||= {}
+    end
+
+    def register_synchronizers(hash)
+      synchronizers.update(hash)
+    end
+  end
+
   private
-  def update_physical_servers_by(hashs)
-    found_ids = []
-    hashs.each do |hash|
-      server = self.physical_servers.where(:provided_id => hash[:provided_id]).first
-      if server
-        server.update_attributes(:status => hash[:status])
-      else
-        server = self.physical_servers.create!(
-          :provided_id => hash[:provided_id],
-          :name => hash[:name],
-          :status => hash[:status])
+
+  def synchronize_by(target_name)
+    synchronizer = synchronizers_by(target_name)
+    synchronizer.execute
+  end
+
+  def synchronizers_by(target_name)
+    unless @synchronizers
+      @synchronizers = {}
+      self.class.synchronizers.each do |target_name, klass|
+        @synchronizers[target_name] = klass.new(self, target_name)
       end
-      found_ids << server.id
     end
-    self.physical_servers.not_in(:_id => found_ids).update_all(:status => "not_found")
+    @synchronizers[target_name]
   end
 
-  def update_virtual_servers_by(hashs)
-    found_ids = []
-    hashs.each do |hash|
-      server = self.virtual_servers.where(:provided_id => hash[:provided_id]).first
-      if server
-        server.update_attributes(hash)
+  public
+
+  class Synchronizer
+    class << self
+      def fetch_known_target_method(method_name = nil)
+        @fetch_known_target_methods ||= {}
+        @fetch_known_target_methods[self] = method_name if method_name
+        @fetch_known_target_methods[self]
+      end
+
+      def map(attr_name, prop_name = nil, &block)
+        property_map[attr_name] = prop_name || block
+      end
+
+      def property_map
+        @property_map ||= {}
+        @property_map[self] ||= {}
+      end
+    end
+
+    def fetch_known_target_method
+      self.class.fetch_known_target_method
+    end
+
+    def property_map
+      @property_map ||= self.class.property_map
+    end
+
+    attr_reader :provider, :target_name, :log_prefix
+    def initialize(provider, target_name)
+      @provider, @target_name = provider, target_name
+      @log_prefix = "#{self.class.name} for #{provider.name}"
+    end
+
+    def execute
+      actual_target_hashs = fetch_actual_target_hashs
+
+      provider.reload
+      known_targets = fetch_known_targets
+
+      id_key = property_map[:provided_id].to_s
+      updated_target_hashs = []
+      destroyed_targets = []
+
+      known_targets.each do |known_target|
+        actual_target_hash = actual_target_hashs.detect do |t|
+          (t[id_key] || t[id_key.to_sym]) == known_target.provided_id
+        end
+        if actual_target_hash
+          updated_target_hashs << actual_target_hash
+        else
+          destroyed_targets << known_target
+        end
+      end
+      created_target_hashs = actual_target_hashs - updated_target_hashs
+
+      differential_update(updated_target_hashs)
+      create_by_hashs(created_target_hashs)
+      destroy_targets(destroyed_targets)
+    end
+
+    private
+
+    # APIからの実際のターゲット情報を取得する
+    def fetch_actual_target_hashs
+      Tengine.logger.debug "#{log_prefix} #{fetch_known_target_method} for api (wakame)"
+      result = provider.send(fetch_known_target_method)
+      Tengine.logger.debug "#{log_prefix} #{result.inspect}"
+      result
+    end
+
+    # DBに記録されているターゲット情報を取得する
+    def fetch_known_targets
+      Tengine.logger.debug "#{log_prefix} #{target_name} on provider (#{provider.name})"
+      result = provider.send(target_name)
+      Tengine.logger.debug "#{log_prefix} #{result.inspect}"
+      result
+    end
+
+
+    def differential_update(hashs)
+      hashs.map{|hash| differential_update_by_hash(hash)}
+    end
+
+    def differential_update_by_hash(hash, &block)
+      properties = hash.dup
+      properties.deep_symbolize_keys!
+      provided_id = properties[ property_map[:provided_id] ]
+      Tengine.logger.debug "#{log_prefix} update (#{provided_id})"
+      target = provider.send(target_name).where(:provided_id => provided_id).first
+      unless target
+        raise "target #{target_name.to_s.singularize} not found by using #{map[:provided_id]}: #{provided_id.inspect}. properties: #{properties.inspect}"
+      end
+      attrs = mapped_attributes(properties)
+      attrs.each do |attr, value|
+        target.send("#{attr}=", value)
+      end
+      prop_backup = properties.dup
+      if target.respond_to?(:properties) && target.properties
+        properties.each do |key, val|
+          value =  properties.delete(key)
+          if (val.to_s != value.to_s) or (value.blank?)
+            if target.properties[key.to_sym]
+              target.properties[key.to_sym] = value
+            else
+              target.properties[key.to_s] = value
+            end
+          end
+        end
+      end
+      if block
+        block.call(target, prop_backup)
       else
-        server = self.virtual_servers.create!(hash.merge(:name => hash[:provided_id]))
+        target.save! if target.changed?
       end
-      found_ids << server.id
+    rescue => e
+      Tengine.logger.error "#{log_prefix} [#{e.class}] #{e.message}: failed to update (#{provided_id}): #{hash.inspect}"
+      raise
     end
-    self.virtual_servers.not_in(:_id => found_ids).destroy_all
-  end
 
-  class << self
-    def find_or_create_by_name!(attrs)
-      result = self.first(:conditions => {:name => attrs[:name]})
-      result ||= self.create!(attrs)
+    def create_by_hashs(hashs)
+      hashs.map{|hash| t = create_by_hash(hash); t ? t.id : nil}.compact
+    end
+
+    def create_by_hash(hash)
+      properties = hash.dup
+      properties.deep_symbolize_keys!
+      provided_id = properties[:provided_id]
+      Tengine.logger.debug "#{log_prefix} create #{provided_id}"
+      attrs = attrs_to_create(properties)
+      target = provider.send(target_name).new
+      attrs[:properties] = properties if target.respond_to?(:properties)
+      yield(attrs) if block_given?
+      target.attributes = attrs
+      target.save!
+      target
+    rescue => e
+      Tengine.logger.error "#{log_prefix} [#{e.class}] #{e.message}: failed to create (#{provided_id}): #{hash.inspect}"
+      raise
+    end
+
+    def attrs_to_create(properties)
+      mapped_attributes(properties)
+    end
+
+    def mapped_attributes(properties)
+      result = {}
+      property_map.each do |attr, prop|
+        value = prop.is_a?(Proc) ?
+        prop.call(properties, provider) : # 引数を一つだけ使うこともあるのlambdaではなくProc.newを使う事を期待しています。
+          properties.delete(prop)
+        result[attr] = value
+      end
       result
     end
+
+    def destroy_targets(targets)
+      targets.each do |target|
+        Tengine.logger.debug "#{log_prefix} destroy #{target.provided_id}"
+        target.destroy
+      end
+    end
+
   end
+
 end