wakame-vdc-agents 10.12.0 → 11.06.0

data/lib/dcmgr/logger.rb

@@ -4,7 +4,11 @@ require 'logger'
 module Dcmgr
   module Logger
 
-    @logdev = ::Logger::LogDevice.new(STDOUT)
+    # for passenger, messages in STDOUT are not appeared in
+    # error.log. $> is changed in initializers/logger.rb as per the
+    # server environment. so that here also refers $> instead of STDOUT or
+    # STDERR constant.
+    @logdev = ::Logger::LogDevice.new($>)
 
     def self.default_logdev
       @logdev

data/lib/dcmgr/messaging_client.rb

@@ -0,0 +1,117 @@
+# -*- coding: utf-8 -*-
+
+require 'isono'
+require 'eventmachine'
+
+module Dcmgr
+  # @example Sync RPC call with object method. 
+  # mc = MessagingClient.start
+  # puts mc.request('endpoint', 'func1', xxxx, xxxx)
+  # puts mc.request('endpoint', 'func2', xxx, xxx)
+  #
+  # @example Sync RPC call using delegated object
+  # mc = MessagingClient.start
+  # endpoint = mc.sync_rpc('endpoint')
+  # endpoint.func1(xxxx, xxxx)
+  # endpoint.func2(xxx, xxx)
+  #
+  class MessagingClient < Isono::Node
+    include Logger
+    include Isono
+
+    def self.start(amqp_uri, manifest=nil, &blk)
+      node = self.new(manifest, &blk)
+
+      if EventMachine.reactor_thread?
+        EventMachine.schedule {
+          node.connect(amqp_uri)
+        }
+      else
+        q = ::Queue.new
+        EventMachine.schedule {
+          node.connect(amqp_uri) { |type|
+            q << type
+          }
+        }
+        case q.deq
+        when :success
+        when :error
+          raise "Connection failed: #{amqp_uri}"
+        end
+      end
+
+      node
+    end
+
+    def stop
+      if connected?
+        close {
+          EventMachine.schedule {
+            EventMachine.stop
+          }
+        }
+      end
+    end
+    
+    def initialize(m=nil, &blk)
+      m ||= Manifest.new(Dir.pwd) {
+        node_name 'dcmgr'
+        node_instance_id Util.gen_id
+
+        load_module NodeModules::EventChannel
+        load_module NodeModules::RpcChannel
+        load_module NodeModules::JobChannel
+      }
+      m.instance_eval(&blk) if blk
+      super(m)
+    end
+
+    class RpcSyncDelegator
+      attr_reader :endpoint
+      
+      def initialize(rpc, endpoint, opts={})
+        @rpc = rpc
+        @endpoint = endpoint
+        @opts = {:timeout=>0.0, :oneshot=>false}.merge(opts)
+      end
+
+      private
+      def method_missing(m, *args)
+        if @opts[:oneshot]
+          oneshot_request(m, *args)
+        else
+          normal_request(m, *args)
+        end
+      end
+
+      def oneshot_request(m, *args)
+        @rpc.request(@endpoint, m, *args) { |req|
+          req.oneshot = true
+        }
+      end
+
+      def normal_request(m, *args)
+        @rpc.request(@endpoint, m, *args)
+      end
+    end
+    
+    def sync_rpc(endpoint, opts={})
+      rpc = NodeModules::RpcChannel.new(self)
+      RpcSyncDelegator.new(rpc, endpoint, opts)
+    end
+
+    def request(endpoint, key, *args, &blk)
+      rpc = NodeModules::RpcChannel.new(self)
+      rpc.request(endpoint, key, *args, &blk)
+    end
+
+    def submit(job_endpoint, key, *args)
+      NodeModules::JobChannel.new(self).submit(job_endpoint, key, *args)
+    end
+
+    def event_publish(evname, opts={})
+      NodeModules::EventChannel.new(self).publish(evname, opts)
+    end
+
+  end
+end

data/lib/dcmgr/models/account.rb

@@ -16,6 +16,7 @@ module Dcmgr::Models
     with_timestamps
 
     one_to_many  :tags, :dataset=>lambda { Tag.filter(:account_id=>self.canonical_uuid); }
+    one_to_one :quota, :class=>Quota, :key=>:account_id
 
     # sti plugin has to be loaded at lower position.
     plugin :subclasses
@@ -30,6 +31,17 @@ module Dcmgr::Models
       self.enabled == ENABLED
     end
 
+    def after_create
+      self.quota = Quota.create
+      super
+    end
+    
+    def before_destroy
+      self.quota.destroy unless self.quota.nil?
+      super
+    end
+    
+
     # STI class variable setter, getter methods.
     class << self
       def default_values
@@ -46,12 +58,13 @@ module Dcmgr::Models
       def uuid(uuid=nil)
         if uuid.is_a?(String)
           uuid = uuid.downcase
-          if uuid !~ /^[a-z0-9]{8}$/
+          unless self.check_trimmed_uuid_format(uuid)
             raise "Invalid syntax of uuid: #{uuid}"
           end
           default_values[:uuid] = uuid
         end
-        default_values[:uuid] || raise("#{self}.uuid is unset. Set the unique number")
+        raise("#{self}.uuid is unset. Set the unique number") unless default_values[:uuid]
+        "#{uuid_prefix}-#{default_values[:uuid]}"
       end
 
       def description(description=nil)
@@ -80,8 +93,19 @@ module Dcmgr::Models
       Account.subclasses.each { |m|
         Account.create(m.default_values.dup)
       }
-    end
 
+      # create shared resource pool tags
+      Dcmgr::Tags::HostPool.create(:account_id=>SystemAccount::SharedPoolAccount.uuid,
+                                   :uuid=>'shhost',
+                                   :name=>"default_shared_hosts")
+      Dcmgr::Tags::NetworkPool.create(:account_id=>SystemAccount::SharedPoolAccount.uuid,
+                                      :uuid=>'shnet',
+                                      :name=>"default_shared_networks")
+      Dcmgr::Tags::StoragePool.create(:account_id=>SystemAccount::SharedPoolAccount.uuid,
+                                      :uuid=>'shstor',
+                                      :name=>"default_shared_storages")
+    end
+    
     SystemAccount.define_account(:DatacenterAccount) do
       pk 100
       uuid '00000000'

data/lib/dcmgr/models/base_new.rb

@@ -4,9 +4,6 @@ require 'sequel/model'
 
 
 module Dcmgr::Models
-  class InvalidUUIDError < StandardError; end
-  class UUIDPrefixDuplication < StandardError; end
-    
   # Sequal::Model plugin to inject the Taggable feature to the model
   # class.
   #
@@ -187,6 +184,11 @@ module Dcmgr::Models
         raise InvalidUUIDError, "Invalid uuid or unsupported uuid: #{p_uuid} in #{self}"
       end
 
+      # Checks the general uuid syntax
+      def check_trimmed_uuid_format(uuid)
+        uuid.match(/^[a-z0-9 ]*$/) && uuid.length <= 8
+      end
+
       # Checks the uuid syntax if it is for the Taggable class.
       def check_uuid_format(uuid)
         uuid =~ /^#{self.uuid_prefix}-/
@@ -397,11 +399,18 @@ module Dcmgr::Models
   class BaseNew < Sequel::Model
 
     LOCK_TABLES_KEY='__locked_tables'
+
+    def self.default_row_lock_mode=(mode)
+      raise ArgumentError unless [nil, :share, :update].member?(mode)
+      @default_row_lock_mode = mode
+    end
     
-    def self.lock!
+    def self.lock!(mode=nil)
+      raise ArgumentError unless [nil, :share, :update].member?(mode)
+      mode ||= @default_row_lock_mode
       locktbls = Thread.current[LOCK_TABLES_KEY]
       if locktbls
-        locktbls[self.db.uri.to_s + @dataset.first_source_alias.to_s]=1
+        locktbls[self.db.uri.to_s + @dataset.first_source_alias.to_s]=mode
       end
     end
     
@@ -414,8 +423,9 @@ module Dcmgr::Models
     
     def self.dataset
       locktbls = Thread.current[LOCK_TABLES_KEY]
-      if locktbls && locktbls[self.db.uri.to_s + @dataset.first_source_alias.to_s]
-        @dataset.opts = @dataset.opts.merge({:lock=>:update})
+      if locktbls && (mode = locktbls[self.db.uri.to_s + @dataset.first_source_alias.to_s])
+        # lock mode: :share or :update
+        @dataset.opts = @dataset.opts.merge({:lock=>mode})
       else
         @dataset.opts = @dataset.opts.merge({:lock=>nil})
       end
@@ -439,6 +449,10 @@ module Dcmgr::Models
       s
     end
 
+    # Returns true if this Model has time stamps
+    def with_timestamps?
+      self.columns.include?(:created_at) && self.columns.include?(:updated_at)
+    end
 
     # Callback when the initial data is setup to the database.
     def self.install_data

data/lib/dcmgr/models/host_pool.rb

@@ -1,18 +1,21 @@
 # -*- coding: utf-8 -*-
 
+require 'isono'
+
 module Dcmgr::Models
   class HostPool < AccountResource
     taggable 'hp'
     with_timestamps
 
-    HYPERVISOR_XEN_34=:'xen-3.4'
-    HYPERVISOR_XEN_40=:'xen-4.0'
-    HYPERVISOR_KVM=:'kvm'
+    HYPERVISOR_XEN_34='xen-3.4'
+    HYPERVISOR_XEN_40='xen-4.0'
+    HYPERVISOR_KVM='kvm'
 
     ARCH_X86=:x86.to_s
     ARCH_X86_64=:x86_64.to_s
 
     SUPPORTED_ARCH=[ARCH_X86, ARCH_X86_64]
+    SUPPORTED_HYPERVISOR=[HYPERVISOR_KVM]
 
     inheritable_schema do
       String :node_id, :size=>80, :null=>true
@@ -35,8 +38,13 @@ module Dcmgr::Models
 
     def validate
       super
-      unless self.node_id =~ /^hva-/
-        errors.add(:node_id, "hva node has to be associated: #{self.node_id}")
+      # for compatibility: hva.xxx or hva-xxxx
+      unless self.node_id =~ /^hva[-.]/
+        errors.add(:node_id, "is invalid ID: #{self.node_id}")
+      end
+
+      if (h = self.class.filter(:node_id=>self.node_id).first) && h.id != self.id
+        errors.add(:node_id, " #{self.node_id} is already been associated to #{h.canonical_uuid} ")
       end
       
       unless SUPPORTED_ARCH.member?(self.arch)
@@ -75,11 +83,18 @@ module Dcmgr::Models
       i.account_id = account.canonical_uuid
       i.image = image
       i.instance_spec = spec
+      i.cpu_cores = spec.cpu_cores
+      i.memory_size = spec.memory_size
+      i.quota_weight = spec.quota_weight
       i.host_pool = self
       i.save
 
       vnic = i.add_nic(network)
       IpLease.lease(vnic, network)
+      
+      #Lease the nat ip in case there is an outside network mapped
+      nat_network = Network.find(:id => vnic[:nat_network_id])
+      IpLease.lease(vnic,nat_network) unless nat_network.nil? 
       i
     end
 
@@ -93,10 +108,15 @@ module Dcmgr::Models
       raise TypeError unless spec.is_a?(InstanceSpec)
       inst_on_hp = self.instances_dataset.lives.all
 
-      (self.offering_cpu_cores > inst_on_hp.inject(0) {|t, i| t += i.spec.cpu_cores } + spec.cpu_cores) &&
-        (self.offering_memory_size > inst_on_hp.inject(0) {|t, i| t += i.spec.memory_size } + spec.memory_size)
+      (self.offering_cpu_cores >= inst_on_hp.inject(0) {|t, i| t += i.cpu_cores } + spec.cpu_cores) &&
+        (self.offering_memory_size >= inst_on_hp.inject(0) {|t, i| t += i.memory_size } + spec.memory_size)
     end
     
+    def to_api_document
+      h = to_hash
+      h.delete(:node_id)
+      h
+    end
     
   end
 end

data/lib/dcmgr/models/image.rb

@@ -14,17 +14,19 @@ module Dcmgr::Models
       Text :source, :null=>false
       String :arch, :size=>10, :null=>false
       Text :description
+      Boolean :is_public, :null=>false, :default=>false
       #Fixnum :parent_image_id
+      String :format, :null=>false, :default=>'raw'
 
       String :state, :size=>20, :null=>false, :default=>:init.to_s
+      index :is_public
     end
 
     # serialize plugin must be defined at the bottom of all class
     # method calls.
     # Possible source column data:
-    # vdc volume:
-    # {:type=>:vdcvol, :account_id=>'a-xxxxx', :snapshot_id=>'snap-xxxxxx'}
-    # {:type=>:http, :uri=>'http://localhost/xxx/xxx'}
+    # {:snapshot_id=>'snap-xxxxxx'}
+    # {:uri=>'http://localhost/xxx/xxx'}
     plugin :serialization
     serialize_attributes :yaml, :source
     
@@ -36,11 +38,37 @@ module Dcmgr::Models
       unless HostPool::SUPPORTED_ARCH.member?(self.arch)
         errors.add(:arch, "Unsupported arch type: #{self.arch}")
       end
+      
+      # validate source
+      md = self.source
+      case self.boot_dev_type
+      when BOOT_DEV_LOCAL
+        errors.add(:source, "Unknown image URI") if md[:uri].nil? || md[:uri] == ''
+      when BOOT_DEV_SAN
+        errors.add(:source, "Unknown snapshot ID") if md[:snapshot_id].nil? || md[:snapshot_id] == '' || VolumeSnapshot[md[:snapshot_id]].nil?
+      end
     end
 
     def to_hash
       super.merge({:source=>self.source.dup, :description=>description.to_s})
     end
+
+    # note on "lookup_account_id":
+    # the source column sometime contains the information which
+    # should not be shown to other accounts. so that the method takes
+    # an argument who is looking into then filters the data in source
+    # column accordingly.
+    def to_api_document(lookup_account_id)
+      h = to_hash
+      if self.account_id == lookup_account_id
+      else
+        if h[:source][:type] == :http
+          # do not show URI for non-owner accounts.
+          h[:source][:uri] = nil
+        end
+      end
+      h
+    end
     
   end
 end

data/lib/dcmgr/models/instance.rb

@@ -17,11 +17,16 @@ module Dcmgr::Models
       String :state, :size=>20, :null=>false, :default=>:init.to_s
       String :status, :size=>20, :null=>false, :default=>:init.to_s
       String :hostname, :null=>false, :size=>32
+      # TODO: remove ssh_key_pair_id column
       String :ssh_key_pair_id
       Fixnum :ha_enabled, :null=>false, :default=>0
+      Float  :quota_weight, :null=>false, :default=>0.0
+      Fixnum :cpu_cores, :null=>false, :unsigned=>true
+      Fixnum :memory_size, :null=>false, :unsigned=>true
       
       Text :user_data, :null=>false, :default=>''
       Text :runtime_config, :null=>false, :default=>''
+      Text :ssh_key_data, :null=>true
 
       Time :terminated_at
       index :state
@@ -36,13 +41,22 @@ module Dcmgr::Models
     one_to_many :volume
     one_to_many :instance_nic
     alias :nic :instance_nic
-    one_to_many :instance_netfilter_groups
     many_to_many :netfilter_groups, :join_table=>:instance_netfilter_groups
+    # TODO: remove ssh_key_pair_id column
     many_to_one :ssh_key_pair
 
     plugin ArchiveChangedColumn, :histories
     
     subset(:lives, {:terminated_at => nil})
+
+    RECENT_TERMED_PERIOD=(60 * 15)
+    # lists the instances which alives and died within
+    # RECENT_TERMED_PERIOD sec.
+    # it was difficult for me to write exprs in virtual row syntax as
+    # per subset(). ;-(
+    def_dataset_method(:alives_and_recent_termed) {
+      filter("terminated_at IS NULL OR terminated_at >= ?", (Time.now.utc - RECENT_TERMED_PERIOD))
+    }
     
     # serialization plugin must be defined at the bottom of all class
     # method calls.
@@ -51,6 +65,8 @@ module Dcmgr::Models
     # {:vnc_port=>11, :telnet_port=>1111}
     plugin :serialization
     serialize_attributes :yaml, :runtime_config
+    # equal to SshKeyPair#to_hash
+    serialize_attributes :yaml, :ssh_key_data
 
     module ValidationMethods
       def self.hostname_uniqueness(account_id, hostname)
@@ -125,14 +141,20 @@ module Dcmgr::Models
         end
         @update_hostname = false
       end
-      
+
+      # sum() returns nil if there is no instance rows.
+      lives_weight = self.class.filter(:account_id=>self.account_id).lives.sum(:quota_weight) || 0.0
+      unless lives_weight <= self.account.quota.instance_total_weight
+        raise "Out of quota limit: #{self.account_id}'s current weight capacity: #{lives_weight} (<= #{self.account.quota.instance_total_weight})"
+      end
+
       super
     end
 
     def before_destroy
       HostnameLease.filter(:account_id=>self.account_id, :hostname=>self.hostname).destroy
       self.instance_nic.each { |o| o.destroy }
-      self.instance_netfilter_groups.each{|o| o.destroy }
+      self.remove_all_netfilter_groups
       self.volume.each { |v|
         v.instance_id = nil
         v.state = :available
@@ -154,14 +176,15 @@ module Dcmgr::Models
     # this is for internal use.
     def to_hash
       h = super
-      h = h.merge({:user_data => user_data.to_s, # Sequel::BLOB -> String
+      h.merge!({:user_data => user_data.to_s, # Sequel::BLOB -> String
                     :runtime_config => self.runtime_config, # yaml -> hash
                     :image=>image.to_hash,
                     :host_pool=>host_pool.to_hash,
                     :instance_nics=>instance_nic.map {|n| n.to_hash },
-                    :instance_spec=>instance_spec.to_hash,
-                    :ips => instance_nic.map { |n| n.ip.ipv4 if n.ip },
-                  })
+                    :ips => instance_nic.map { |n| n.ip.map {|i| unless i.is_natted? then i.ipv4 else nil end} if n.ip }.flatten.compact,
+                    :nat_ips => instance_nic.map { |n| n.ip.map {|i| if i.is_natted? then i.ipv4 else nil end} if n.ip }.flatten.compact,
+              })
+      h.merge!({:instance_spec=>instance_spec.to_hash}) unless instance_spec.nil?
       h[:volume]={}
       if self.volume
         self.volume.each { |v|
@@ -186,12 +209,13 @@ module Dcmgr::Models
     #   :created_at
     #   :state
     #   :status
+    #   :vif => {'vif-xxxxx'=>{:ipv4=>{:address=>'8.8.8.8', :nat_address=>'9.9.9.9.9'}}}
     # }
     def to_api_document
       h = {
         :id => canonical_uuid,
-        :cpu_cores   => instance_spec.cpu_cores,
-        :memory_size => instance_spec.memory_size,
+        :cpu_cores   => cpu_cores,
+        :memory_size => memory_size,
         :image_id    => image.canonical_uuid,
         :created_at  => self.created_at,
         :state => self.state,
@@ -200,19 +224,41 @@ module Dcmgr::Models
         :network => [],
         :volume => [],
         :netfilter_group => [],
+        :vif => [],
       }
-      if self.ssh_key_pair
-        h[:ssh_key_pair] = self.ssh_key_pair.name
+      if self.ssh_key_data
+        h[:ssh_key_pair] = self.ssh_key_data[:name]
       end
 
       if instance_nic
         instance_nic.each { |n|
-          if n.ip
-            h[:network] << {
-              :network_name => n.network.canonical_uuid,
-              :ipaddr => n.ip.ipv4
+          direct_lease_ds = n.direct_ip_lease_dataset
+          next if direct_lease_ds.first.nil?
+          outside_lease_ds = n.nat_ip_lease_dataset
+          
+          h[:network] << {
+            :network_name => n.network.canonical_uuid,
+            :ipaddr => direct_lease_ds.all.map {|lease| lease.ipv4 }.compact,
+            :nat_ipaddr => outside_lease_ds.all.map {|lease| lease.ipv4 }.compact
+          }
+        }
+      end
+
+      if instance_nic
+        instance_nic.each { |vif|
+          ent = {
+            :vif_id=>vif.canonical_uuid,
+          }
+          direct_lease = vif.direct_ip_lease.first
+          if direct_lease.nil?
+          else
+            outside_lease = direct_lease.nat_outside_lease
+            ent[:ipv4] = {
+              :address=> direct_lease.ipv4,
+              :nat_address => outside_lease.nil? ? nil : outside_lease.ipv4,
             }
           end
+          h[:vif] << ent
         }
       end
       
@@ -244,14 +290,6 @@ module Dcmgr::Models
       self.image.arch
     end
 
-    def cpu_cores
-      self.instance_spec.cpu_cores
-    end
-
-    def memory_size
-      self.instance_spec.memory_size
-    end
-
     def config
       self.instance_spec.config
     end
@@ -261,6 +299,7 @@ module Dcmgr::Models
       vendor_id ||= '00:ff:f1'
       nic = InstanceNic.new(:mac_addr=>vendor_id)
       nic.network = network
+      nic.nat_network = network.nat_network
       nic.instance = self
       nic.save
     end
@@ -335,10 +374,20 @@ module Dcmgr::Models
       Volume.lock!
       VolumeSnapshot.lock!
       IpLease.lock!
+      HostnameLease.lock!
+      Network.lock!
     end
 
     def live?
       self.terminated_at.nil?
     end
+
+    def set_ssh_key_pair(ssh_key_pair)
+      raise ArgumentError unless ssh_key_pair.is_a?(SshKeyPair)
+      self.ssh_key_data = ssh_key_pair.to_hash
+      # TODO: remove ssh_key_pair_id column
+      self.ssh_key_pair_id = ssh_key_pair.canonical_uuid
+    end
+    
   end
 end