wakame-vdc-agents 10.11.0 → 10.12.0

data/lib/dcmgr/endpoints/errors.rb

@@ -45,6 +45,10 @@ module Dcmgr
     define_error(:OutOfHostCapacity, 400)
     define_error(:UnknownSshKeyPair, 404)
     define_error(:UndefinedStoragePoolID, 400)
+    define_error(:DetachVolumeFailure, 400)
+    define_error(:AttachVolumeFailure, 400)
+    define_error(:InvalidInstanceState, 400)
+    define_error(:DuplicateHostname, 400)
 
     # netfilter_group
     define_error(:UndefinedNetfilterGroup, 400)

data/lib/dcmgr/helpers/cli_helper.rb

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+
+require 'eventmachine'
+require 'shellwords'
+raise "Shellword is old version." unless Shellwords.respond_to?(:shellescape)
+require 'open4'
+
+module Dcmgr
+  module Helpers
+    module CliHelper
+      class TimeoutError < RuntimeError; end
+      
+      def tryagain(opts={:timeout=>60, :retry=>3}, &blk)
+        timedout = false
+        curthread = Thread.current
+
+        timersig = EventMachine.add_timer(opts[:timeout]) {
+          timedout = true
+          if curthread
+            curthread.raise(TimeoutError.new("timeout"))
+            curthread.pass
+          end
+        }
+
+        count = opts[:retry]
+        begin
+          begin
+            break if blk.call
+          end while !timedout && ((count -= 1) >= 0)
+        rescue TimeoutError => e
+          raise e
+        rescue RuntimeError => e
+          if respond_to?(:logger)
+            logger.debug("Caught Error. To be retrying....: #{e}")
+          end
+          retry if (count -= 1) >= 0
+        ensure
+          curthread = nil
+          EventMachine.cancel_timer(timersig) rescue nil
+        end
+      end
+
+      class CommandError < StandardError
+        attr_reader :stderr, :stdout
+        def initialize(msg, stdout, stderr)
+          super(msg)
+          @stdout = stdout
+          @stderr = stderr
+        end
+      end
+      
+      def sh(cmd, args=[], opts={})
+        opts = opts.merge({:expect_exitcode=>0})
+        cmd = sprintf(cmd, *args.map {|a| Shellwords.shellescape(a.to_s) })
+
+        outbuf = errbuf = ''
+        blk = proc {|pid, stdin, stdout, stderr|
+          stdin.close
+          outbuf = stdout.read
+          errbuf = stderr.read
+        }
+        stat = Open4::popen4(cmd, &blk)
+        if self.respond_to?(:logger)
+          logger.debug("Exec command (pid=#{stat.pid}): #{cmd}")
+          msg = "Command output:"
+          msg << "\nSTDOUT:\n#{outbuf.strip}" if outbuf && outbuf.strip.size > 0
+          msg << "\nSTDERR:\n#{errbuf.strip}" if errbuf && errbuf.strip.size > 0
+          logger.debug(msg)
+        end
+        if stat.exitstatus != opts[:expect_exitcode]
+          raise CommandError.new("Unexpected exit code=#{stat.exitstatus} (expected=#{opts[:expect_exitcode]})", \
+            outbuf, errbuf)
+        end
+        true
+      end
+    end
+
+  end
+end

data/lib/dcmgr/models/account.rb

@@ -3,10 +3,6 @@
 module Dcmgr::Models
   class Account < BaseNew
     taggable 'a'
-    with_timestamps
-    plugin :single_table_inheritance, :uuid, :model_map=>{}
-    plugin :subclasses
-
     # pk has to be overwritten by the STI subclasses.
     unrestrict_primary_key
 
@@ -17,8 +13,14 @@ module Dcmgr::Models
       String :description, :size=>100
       Fixnum :enabled, :default=>ENABLED, :null=>false
     end
+    with_timestamps
 
-    one_to_many  :tags
+    one_to_many  :tags, :dataset=>lambda { Tag.filter(:account_id=>self.canonical_uuid); }
+
+    # sti plugin has to be loaded at lower position.
+    plugin :subclasses
+    plugin :single_table_inheritance, :uuid, :model_map=>{}
+    
 
     def disable?
       self.enabled == DISABLED
@@ -28,13 +30,6 @@ module Dcmgr::Models
       self.enabled == ENABLED
     end
 
-    def to_hash_document
-      h = self.values.dup
-      h[:id] = h[:uuid] = self.canonical_uuid
-      h
-    end
-
-
     # STI class variable setter, getter methods.
     class << self
       def default_values
@@ -98,6 +93,18 @@ module Dcmgr::Models
         self.enabled = Account::ENABLED
       end
     end
+
+    SystemAccount.define_account(:SharedPoolAccount) do
+      pk 101
+      uuid 'shpoolxx'
+      description 'system account for shared resources'
+
+      # SahredPoolAccount is always enabled.
+      def before_save
+        super
+        self.enabled = Account::ENABLED
+      end
+    end
     
   end
 end

data/lib/dcmgr/models/base_new.rb

@@ -50,6 +50,7 @@ module Dcmgr::Models
           column(:uuid, String, :size=>8, :null=>false, :fixed=>true, :unique=>true)
         end
       }
+      model.many_to_many :tags, :dataset=>lambda { Tag.join(TagMapping.table_name, :tag_id=>:id, :uuid=>self.canonical_uuid); }
     end
 
     module InstanceMethods
@@ -65,6 +66,12 @@ module Dcmgr::Models
         self[:uuid] ||= Array.new(8) do UUID_TABLE[rand(UUID_TABLE.size)]; end.join
       end
 
+      # model hook
+      def after_destroy
+        super
+        TagMapping.filter(:uuid=>self.canonical_uuid).delete
+      end
+
       # Returns canonicalized uuid which has the form of
       # "{uuid_prefix}-{uuid}".
       def canonical_uuid
@@ -74,36 +81,56 @@ module Dcmgr::Models
 
       # Put the tag on the object.
       #
-      # This method just delegates the method call of Tag#label().
-      # @params [Models::Tag,String] tag_or_tag_uuid 'tag-xxxx' is expected when the type is string
-      def label_tag(tag_or_tag_uuid)
-        tag = case tag_or_tag_uuid
+      # @params [Models::Tag,String,Symbol] arg1
+      # @params [String,NilClass] arg2
+      # @params [String,NilClass] arg3
+      # 
+      # @example
+      # lable_tag('tag-xxxxx')
+      # t = Tag['tag-xxxx']
+      # label_tag(t)
+      # label_tag(:NetworkPool, 'newname1', 'account_id')
+      def label_tag(arg1, arg2=nil, arg3=nil)
+        tag = case arg1
               when String
-                Tag[tag_or_tag_uuid]
+                Tag[arg1]
+              when Symbol
+                acctid = arg3 || self.respond_to?(:account_id) ? self.account_id : raise("Unknown Account ID")
+                Dcmgr::Tags.const_get(arg1).find_or_create(:account_id=>acctid, :name=>arg2)
               when Tag
-                tag_or_tag_uuid
+                arg1
               else
-                raise ArgumentError, "Invalid type: #{tag_or_tag_uuid.class}"
+                raise ArgumentError, "Invalid type: #{arg1.class}"
               end
-        
-        tag.label(self.uuid)
+        raise "Root Tag class can not be used" unless tag.class < Tag
+        tag.label(self.canonical_uuid)
       end
 
       # Remove the labeled tag from the object
       #
-      # This method just delegates the method call of Tag#unlabel().
-      # @params [Models::Tag,String] tag_or_tag_uuid 'tag-xxxx' is expected when the type is string
-      def unlabel_tag(tag_or_tag_uuid)
-        tag = case tag_or_tag_uuid
+      # @params [Models::Tag,String,Symbol] arg1
+      # @params [String,NilClass] arg2
+      # @params [String,NilClass] arg3
+      #
+      # @example
+      # unlable_tag('tag-xxxxx')
+      # t = Tag['tag-xxxx']
+      # unlabel_tag(t)
+      # unlabel_tag(:NetworkPool, 'newname1', 'account_id')
+      def unlabel_tag(arg1, arg2=nil, arg3=nil)
+        tag = case arg1
               when String
-                Tag[tag_or_tag_uuid]
+                Tag[arg1]
+              when Symbol
+                acctid = arg3 || self.respond_to?(:account_id) ? self.account_id : raise("Unknown Account ID")
+                Dcmgr::Tags.const_get(arg1).find(:account_id=>acctid, :name=>arg2)
               when Tag
-                tag_or_tag_uuid
+                arg1
               else
-                raise ArgumentError, "Invalid type: #{tag_or_tag_uuid.class}"
+                raise ArgumentError, "Invalid type: #{arg1.class}"
               end
 
-        tag.unlabel(self.uuid)
+        tag.unlabel(self.canonical_uuid)
       end
 
       def to_hash()
@@ -256,7 +283,116 @@ module Dcmgr::Models
     end
     
   end
-  
+
+  # This plugin is to archive the changes on each column of the model
+  # to a history table.
+  # 
+  # plugin ArchiveChangedColumn, :your_history_table
+  #  or
+  # plugin ArchiveChangedColumn
+  # history_dataset = DB[:history_table]
+  #
+  # The history table should have the schema below:
+  # schema do
+  #   Fixnum :id, :null=>false, :primary_key=>true
+  #   String :uuid, :size=>50, :null=>false
+  #   String :attr, :null=>false
+  #   String :vchar_value, :null=>true
+  #   String :blob_value, :null=>true, :text=>true
+  #   Time  :created_at, :null=>false
+  #   index [:uuid, :created_at]
+  #   index [:uuid, :attr]
+  # end
+  module ArchiveChangedColumn
+    def self.configure(model, history_table=nil)
+      model.history_dataset = case history_table
+                              when NilClass
+                                nil
+                              when String,Symbol
+                                model.db.from(history_table)
+                              when Class
+                                raise "Unknown type" unless history_table < Sequel::Model
+                                history_table.dataset
+                              when Sequel::Dataset
+                                history_table
+                              else
+                                raise "Unknown type"
+                              end
+    end
+    
+    module ClassMethods
+      def history_dataset=(ds)
+        @history_ds = ds
+      end
+      
+      def history_dataset
+        @history_ds
+      end
+    end
+
+    module InstanceMethods
+      def history_snapshot(at)
+        raise TypeError unless at.is_a?(Time)
+
+        if self.created_at > at || (!self.terminated_at.nil? && self.terminated_at < at)
+          raise "#{at} is not in the range of the object's life span."
+        end
+        
+        ss = self.dup
+        #  SELECT * FROM (SELECT * FROM `instance_histories` WHERE
+        #  (`uuid` = 'i-ezsrs132') AND created_at <= '2010-11-30 23:08:05'
+        #  ORDER BY created_at DESC) AS a GROUP BY a.attr;
+        ds = self.class.history_dataset.filter('uuid=? AND created_at <= ?', self.canonical_uuid, at).order(:created_at.desc)
+        ds = ds.from_self.group_by(:attr)
+        ds.all.each { |h|
+          if !h[:blob_value].nil?
+            ss.send("#{h[:attr]}=", typecast_value(h[:attr], h[:blob_value]))
+          else
+            ss.send("#{h[:attr]}=", typecast_value(h[:attr], h[:vchar_value]))
+          end
+        }
+        # take care for serialized columns by serialization plugin.
+        ss.deserialized_values.clear if ss.respond_to?(:deserialized_values)
+        
+        ss
+      end
+
+      def before_create
+        return false if super == false
+        store_changes(self.columns)
+        true
+      end
+
+      def before_update
+        return false if super == false
+        store_changes(self.changed_columns)
+        true
+      end
+      
+      private
+      def store_changes(cols_stored)
+        return if cols_stored.nil? || cols_stored.empty?
+        common_rec = {
+          :uuid=>self.canonical_uuid,
+          :created_at => Time.now,
+        }
+        
+        cols_stored.each { |c|
+          hist_rec = common_rec.dup
+          hist_rec[:attr] = c.to_s
+          
+          coldef = self.class.db_schema[c]
+          case coldef[:type]
+          when :text,:blob
+            hist_rec[:blob_value]= (new? ? (self[c] || coldef[:default]) : self[c])
+          else
+            hist_rec[:vchar_value]=(new? ? (self[c] || coldef[:default]) : self[c])
+          end
+          self.class.history_dataset.insert(hist_rec)
+        }
+      end
+    end
+  end
 
   class BaseNew < Sequel::Model
 

data/lib/dcmgr/models/history.rb

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr::Models
+  # History record table for ArchiveChangedColumn plugin
+  class History < BaseNew
+    set_dataset(:histories)
+    
+    inheritable_schema do
+      String :uuid, :size=>50, :null=>false
+      String :attr, :null=>false
+      String :vchar_value, :null=>true
+      String :blob_value, :null=>true, :text=>true
+      Time  :created_at, :null=>false
+      index [:uuid, :created_at]
+      index [:uuid, :attr]
+    end
+
+    plugin :timestamps
+    
+  end
+end

data/lib/dcmgr/models/host_pool.rb

@@ -19,16 +19,15 @@ module Dcmgr::Models
       
       String :arch, :size=>10, :null=>false # :x86, :x86_64
       String :hypervisor, :size=>30, :null=>false
-      Fixnum :network_id, :null=>false
 
       Fixnum :offering_cpu_cores,   :null=>false, :unsigned=>true
       Fixnum :offering_memory_size, :null=>false, :unsigned=>true
-      Fixnum :allow_memory_overcommit, :null=>false, :default=>1
+
+      index :node_id
     end
     
     one_to_many :instances
     many_to_one :node, :class=>Isono::Models::NodeState, :key=>:node_id, :primary_key=>:node_id
-    many_to_one :network
 
     def after_initialize
       super
@@ -66,10 +65,12 @@ module Dcmgr::Models
     # @param [Models::Account] account
     # @param [Models::Image] image
     # @param [Models::InstanceSpec] spec
+    # @param [Models::Network] network
     # @return [Models::Instance] created new Instance object.
-    def create_instance(account, image, spec, &blk)
+    def create_instance(account, image, spec, network, &blk)
       raise ArgumentError unless image.is_a?(Image)
       raise ArgumentError unless spec.is_a?(InstanceSpec)
+      raise ArgumentError unless network.is_a?(Network)
       i = Instance.new &blk
       i.account_id = account.canonical_uuid
       i.image = image
@@ -77,9 +78,8 @@ module Dcmgr::Models
       i.host_pool = self
       i.save
 
-      # TODO: set vnic spec from InstanceSpec
-      vnic = i.add_nic
-      IpLease.lease(vnic)
+      vnic = i.add_nic(network)
+      IpLease.lease(vnic, network)
       i
     end
 

data/lib/dcmgr/models/hostname_lease.rb

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr::Models
+  # hostname table for each instance to ensure uniqueness.
+  class HostnameLease < BaseNew
+
+    inheritable_schema do
+      String :account_id, :null=>false, :size=>50
+      String :hostname, :null=>false, :size=>32
+
+      index [:account_id, :hostname], {:unique=>true}
+    end
+    with_timestamps
+  end
+end

data/lib/dcmgr/models/instance.rb

@@ -16,17 +16,16 @@ module Dcmgr::Models
       Fixnum :instance_spec_id, :null=>false
       String :state, :size=>20, :null=>false, :default=>:init.to_s
       String :status, :size=>20, :null=>false, :default=>:init.to_s
-      String :hostname, :null=>false
+      String :hostname, :null=>false, :size=>32
       String :ssh_key_pair_id
+      Fixnum :ha_enabled, :null=>false, :default=>0
       
       Text :user_data, :null=>false, :default=>''
       Text :runtime_config, :null=>false, :default=>''
 
-      Time   :terminated_at
+      Time :terminated_at
       index :state
       index :terminated_at
-      # can not use same hostname within an account.
-      index  [:account_id, :hostname], {:unique=>true}
     end
     with_timestamps
     
@@ -41,29 +40,116 @@ module Dcmgr::Models
     many_to_many :netfilter_groups, :join_table=>:instance_netfilter_groups
     many_to_one :ssh_key_pair
 
+    plugin ArchiveChangedColumn, :histories
+    
     subset(:lives, {:terminated_at => nil})
     
     # serialization plugin must be defined at the bottom of all class
     # method calls.
     # Possible column data:
     #   kvm:
-    # {:vnc_port=>11}
+    # {:vnc_port=>11, :telnet_port=>1111}
     plugin :serialization
     serialize_attributes :yaml, :runtime_config
 
+    module ValidationMethods
+      def self.hostname_uniqueness(account_id, hostname)
+        HostnameLease.filter(:account_id=>account_id, :hostname=>hostname).empty?
+      end
+    end
+
     def validate
       super
 
-      # TODO: hostname column validation
+      unless self.hostname =~ /\A[0-9a-z][0-9a-z\-]{0,31}\Z/
+        errors.add(:hostname, "Invalid hostname syntax")
+      end
+
+      # uniqueness check for hostname
+      if changed_columns.include?(:hostname)
+        proc_test = lambda {
+          unless ValidationMethods.hostname_uniqueness(self.account_id, self.hostname)
+            errors.add(:hostname, "Duplicated hostname: #{self.hostname}")
+          end
+        }
+        
+        if new?
+          proc_test.call
+        else
+          orig = self.dup.refresh
+          # do nothing if orig.hostname == self.hostname
+          if orig.hostname != self.hostname
+            proc_test.call
+          end
+        end
+        @update_hostname = true
+      end
+      
+      # check runtime_config column
+      case self.hypervisor
+      when HostPool::HYPERVISOR_KVM
+        r1 = self.runtime_config
+        self.host_pool.instances.each { |i|
+          next true if i.id == self.id
+          r2 = i.runtime_config
+          unless r1[:vnc_port] != r2[:vnc_port] && r1[:telnet_port] != r2[:telnet_port]
+            errors.add(:runtime_config, "#{self.canonical_uuid}.runtime_config conflicted with #{i.canonical_uuid}")
+            break
+          end
+        }
+      end
     end
 
     def before_validation
+      self[:user_data] ||= ''
+      self[:hostname] ||= self.uuid
+      self[:hostname] = self[:hostname].downcase
       super
+    end
 
-      self[:user_data] = '' if self.user_data.nil?
-      self[:hostname] = self.uuid if self.hostname.nil?
+    def before_save
+      if @update_hostname
+        if new?
+          HostnameLease.create(:account_id=>self.account_id,
+                               :hostname=>self.hostname)
+        else
+          orig = self.dup.refresh
+          # do nothing if orig.hostname == self.hostname
+          if orig.hostname != self.hostname
+            
+            orig_name = HostnameLease.filter(:account_id=>self.account_id,
+                                             :hostname=>orig.hostname).first
+            orig_name.hostname = self.hostname
+            orig_name.save
+          end
+        end
+        @update_hostname = false
+      end
+      
+      super
     end
 
+    def before_destroy
+      HostnameLease.filter(:account_id=>self.account_id, :hostname=>self.hostname).destroy
+      self.instance_nic.each { |o| o.destroy }
+      self.instance_netfilter_groups.each{|o| o.destroy }
+      self.volume.each { |v|
+        v.instance_id = nil
+        v.state = :available
+        v.save
+      }
+      super
+    end
+
+    # override Sequel::Model#_delete not to delete rows but to set
+    # delete flags.
+    def _delete
+      self.terminated_at ||= Time.now
+      self.state = :terminated if self.state != :terminated
+      self.status = :offline if self.status != :offline
+      self.save
+    end
+    
     # dump column data as hash with details of associated models.
     # this is for internal use.
     def to_hash
@@ -74,11 +160,12 @@ module Dcmgr::Models
                     :host_pool=>host_pool.to_hash,
                     :instance_nics=>instance_nic.map {|n| n.to_hash },
                     :instance_spec=>instance_spec.to_hash,
+                    :ips => instance_nic.map { |n| n.ip.ipv4 if n.ip },
                   })
       h[:volume]={}
       if self.volume
         self.volume.each { |v|
-          h[:volume][v.canonical_uuid] = v.to_hash_document
+          h[:volume][v.canonical_uuid] = v.to_hash
         }
       end
       h
@@ -92,8 +179,8 @@ module Dcmgr::Models
     #   :cpu_cores
     #   :memory_size
     #   :image_id
-    #   :network => {'global1'=>{:ipaddr=>'111.111.111.111'}}
-    #   :volume => {'uuid'=>{:guest_device_name=>,}}
+    #   :network => [{:network_name=>'nw-xxxxxxx', :ipaddr=>'111.111.111.111'}]
+    #   :volume => [{'uuid'=>{:guest_device_name=>,}]
     #   :ssh_key_pair => 'xxxxx',
     #   :netfilter_group => ['rule1', 'rule2']
     #   :created_at
@@ -122,7 +209,7 @@ module Dcmgr::Models
         instance_nic.each { |n|
           if n.ip
             h[:network] << {
-              :network_name => n.ip.network.name,
+              :network_name => n.network.canonical_uuid,
               :ipaddr => n.ip.ipv4
             }
           end
@@ -169,13 +256,11 @@ module Dcmgr::Models
       self.instance_spec.config
     end
 
-    def add_nic(vifname=nil, vendor_id=nil)
-      vifname ||= "vif-#{self[:uuid]}"
+    def add_nic(network, vendor_id=nil)
       # TODO: get default vendor ID based on the hypervisor.
       vendor_id ||= '00:ff:f1'
-      nic = InstanceNic.new({:vif=>vifname,
-                              :mac_addr=>vendor_id
-                            })
+      nic = InstanceNic.new(:mac_addr=>vendor_id)
+      nic.network = network
       nic.instance = self
       nic.save
     end
@@ -202,15 +287,15 @@ module Dcmgr::Models
       self.instance_nic.map { |nic| nic.ip }
     end
 
-    def netfilter_group_instances
-      instances = self.netfilter_groups.map { |g| g.instances }
-
-      instances.flatten!.uniq! if instances.size > 0
-      instances
-    end
+#    def netfilter_group_instances
+#      instances = self.netfilter_groups.map { |g| g.instances }
+#
+#      instances.flatten!.uniq! if instances.size > 0
+#      instances
+#    end
 
     def fqdn_hostname
-      sprintf("%s.%s.%s", self.hostname, self.account.uuid, self.host_pool.network.domain_name)
+      sprintf("%s.%s.%s", self.hostname, self.account.uuid, self.nic.first.network.domain_name)
     end
 
     # Retrieve all networks belong to this instance
@@ -221,7 +306,7 @@ module Dcmgr::Models
       }.map { |nic|
         nic.ip.network
       }.group_by { |net|
-        net.name
+        net.canonical_uuid
       }.values.map { |i|
         i.first
       }
@@ -251,5 +336,9 @@ module Dcmgr::Models
       VolumeSnapshot.lock!
       IpLease.lock!
     end
+
+    def live?
+      self.terminated_at.nil?
+    end
   end
 end