wakame-vdc-dcmgr 11.06.0 → 11.12.0

data/lib/dcmgr/models/tag.rb

@@ -43,15 +43,6 @@ module Dcmgr::Models
   class Tag < AccountResource
     taggable('tag')
 
-    inheritable_schema do
-      Fixnum :type_id, :null=>false
-      String :name, :null=>false
-      String :attributes
-
-      index [:account_id, :type_id, :name], {:unique=>true}
-    end
-    with_timestamps
-
     many_to_one :account
     
     one_to_many :mapped_uuids, :class=>TagMapping do |ds|
@@ -160,10 +151,5 @@ module Dcmgr::Models
     def to_api_document
       to_hash.merge({:type_id=>self.class.to_s.split('::').last})
     end
-
-    def self.declare(account_id, tag_class_name, name)
-      Dcmgr::Tags.const_get(tag_class_name).find_or_create(:account_id=>account_id,
-                                                           :name=>name)
-    end
   end
 end

data/lib/dcmgr/models/tag_mapping.rb

@@ -2,13 +2,7 @@
 
 module Dcmgr::Models
   class TagMapping < BaseNew
-    inheritable_schema do
-      Fixnum :tag_id, :null=>false
-      String :uuid, :null=>false, :size=>20
-      index :tag_id
-      index :uuid
-    end
-    
+
     many_to_one :tag
 
   end

data/lib/dcmgr/models/vlan_lease.rb

@@ -5,17 +5,11 @@ module Dcmgr::Models
   class VlanLease < AccountResource
     taggable 'vlan'
 
-    inheritable_schema do
-      Fixnum :tag_id, :null=>false
-
-      index :tag_id, {:unique=>true}
-    end
-    with_timestamps
-
     one_to_many :networks
 
     def validate
-
+      super
+      
       unless 1 <= self.tag_id.to_i && self.tag_id.to_i <= 4095
         errors.add(:tag_id, "Tag ID is out of range (1-4095)")
       end

data/lib/dcmgr/models/volume.rb

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-require 'statemachine'
 
 module Dcmgr::Models
   class Volume < AccountResource
@@ -21,31 +20,7 @@ module Dcmgr::Models
     STATE_TYPE_DELETING = "deleting"
     STATE_TYPE_DELETED = "deleted"
 
-    inheritable_schema do
-      Fixnum :storage_pool_id, :null=>false
-      String :status, :null=>false, :default=>STATUS_TYPE_REGISTERING
-      String :state, :null=>false, :default=>STATE_TYPE_REGISTERING
-      Fixnum :size, :null=>false
-      Fixnum :instance_id
-      Fixnum :boot_dev, :null=>false, :default=>0
-      String :snapshot_id
-      String :host_device_name
-      String :guest_device_name
-      String :export_path, :null=>false
-#      String :intermediate_path, :null=>false
-      Text :transport_information
-      Time :deleted_at
-      Time :attached_at
-      Time :detached_at
-
-      index :storage_pool_id
-      index :instance_id
-      index :snapshot_id
-      index :deleted_at
-    end
-    with_timestamps
-
-    many_to_one :storage_pool
+    many_to_one :storage_node, :after_set=>:validate_storage_node_assigned
     many_to_one :instance
 
     plugin ArchiveChangedColumn, :histories
@@ -65,25 +40,38 @@ module Dcmgr::Models
     # iscsi:
     # {:iqn=>'iqn.1986-03.com.sun:02:a1024afa-775b-65cf-b5b0-aa17f3476bfc', :lun=>0}
     plugin :serialization, :yaml, :transport_information
+    plugin :serialization, :yaml, :request_params
     
-    class DiskError < RuntimeError; end
+    class CapacityError < RuntimeError; end
     class RequestError < RuntimeError; end
 
-    def before_create
-      sp = self.storage_pool
+    def validate_storage_node_assigned(sp)
+      unless sp.is_a?(StorageNode)
+        raise "unknown class: #{sp.class}"
+      end
       volume_size = sp.volumes_dataset.lives.sum(:size).to_i
       # check if the sum of available volume and new volume is under
       # the limit of offering capacity.
       total_size = sp.offering_disk_space - volume_size.to_i
       if self.size > total_size
-        raise DiskError, "out of disk space"
+        raise CapacityError, "Allocation exceeds storage node blank size: #{}"
       end
-
-      # TODO: Here may not be the right place for capacity validation.
-      per_account_totoal = self.class.filter(:account_id=>self.account_id).lives.sum(:size).to_i
-      if self.account.quota.volume_total_size < per_account_totoal + self.size.to_i
-        raise DiskError, "Out of account quota: #{self.account.quota.volume_total_size}, #{self.size.to_i}, #{per_account_totoal}"
+    end
+    
+    def validate
+      # do not run validation if the row is maked as deleted.
+      return true if self.deleted_at
+
+      if new?
+        # TODO: Here may not be the right place for capacity validation.
+        per_account_total = self.class.filter(:account_id=>self.account_id).lives.sum(:size).to_i
+        if self.account.quota.volume_total_size < per_account_total + self.size.to_i
+          raise CapacityError, "Allocation exceeds account's quota: #{self.account.quota.volume_total_size}, #{self.size.to_i}, #{per_account_total}"
+        end
       end
+
+      errors.add(:size, "Invalid volume size.") if self.size == 0
+      
       super
     end
 
@@ -125,7 +113,7 @@ module Dcmgr::Models
 
     def merge_pool_data
       v = self.to_hash
-      v.merge(:storage_pool=>storage_pool.to_hash)
+      v.merge(:storage_node=>storage_node.to_hash)
     end
 
     def to_hash
@@ -157,9 +145,33 @@ module Dcmgr::Models
 
     def create_snapshot(account_id)
       vs = VolumeSnapshot.create(:account_id=>account_id,
-                                 :storage_pool_id=>self.storage_pool_id,
+                                 :storage_node_id=>self.storage_node_id,
                                  :origin_volume_id=>self.canonical_uuid,
                                  :size=>self.size)
     end
+
+    # override Sequel::Model#delete not to delete rows but to set
+    # delete flags.
+    def delete
+      self.deleted_at ||= Time.now
+      self.state = :deleted if self.state != :deleted
+      self.status = :offline if self.status != :offline
+      self.save
+    end
+
+    def snapshot
+      VolumeSnapshot[self.snapshot_id]
+    end
+
+    def self.entry_new(account, size, params, &blk)
+      # Mash is passed in some cases.
+      raise ArgumentError unless params.class == ::Hash
+      v = self.new &blk
+      v.account_id = account.canonical_uuid
+      v.size = size
+      v.request_params = params
+      v
+    end
+      
   end
 end

data/lib/dcmgr/models/volume_snapshot.rb

@@ -11,22 +11,11 @@ module Dcmgr::Models
     STATE_TYPE_DELETING = "deleting"
     STATE_TYPE_DELETED = "deleted"
 
-    inheritable_schema do
-      Fixnum :storage_pool_id, :null=>false
-      String :origin_volume_id, :null=>false
-      Fixnum :size, :null=>false
-      Fixnum :status, :null=>false, :default=>0
-      String :state, :null=>false, :default=>STATE_TYPE_REGISTERING
-      String :destination_key, :null=>false 
-      Time   :deleted_at
-      index :storage_pool_id
-      index  :deleted_at
-    end
-    with_timestamps
-
-    many_to_one :storage_pool
+    many_to_one :storage_node
     plugin ArchiveChangedColumn, :histories
 
+    subset(:alives, {:deleted_at => nil})
+    
     RECENT_TERMED_PERIOD=(60 * 15)
     # lists the volumes are available and deleted within
     # RECENT_TERMED_PERIOD sec.
@@ -46,15 +35,16 @@ module Dcmgr::Models
         :origin_volume_id => self.origin_volume_id,
         :destination_id => self.destination,
         :destination_name => self.display_name, 
+        :backing_store => self.storage_node.storage_type,
         :created_at => self.created_at,
         :deleted_at => self.deleted_at,
       }
     end
 
     # create volume inherite from this snapshot for the account.
-    # limitation: inherit volume is created on same storage_pool.
+    # limitation: inherit volume is created on same storage_node.
     def create_volume(account_id)
-      storage_pool.create_volume(account_id, self.size, self.canonical_uuid)
+      storage_node.create_volume(account_id, self.size, self.canonical_uuid)
     end
 
     def display_name
@@ -68,7 +58,7 @@ module Dcmgr::Models
     end
     
     def snapshot_filename
-      "#{self.canonical_uuid}.zsnap"
+      "#{self.canonical_uuid}.snap"
     end
     
     def destination
@@ -83,6 +73,14 @@ module Dcmgr::Models
       vs.state = :deleting
       vs.save_changes
     end
+
+    # override Sequel::Model#delete not to delete rows but to set
+    # delete flags.
+    def delete
+      self.state = :deleted if self.state != :deleted
+      self.deleted_at ||= Time.now
+      self.save
+    end
     
     def update_destination_key(account_id, destination_key)
       self.destination_key = destination_key

data/lib/dcmgr/node_modules/hva_collector.rb

@@ -46,11 +46,11 @@ module Dcmgr
         nil
       end
 
-      def get_netfilter_groups_of_instance(instance_id)
+      def get_security_groups_of_instance(instance_id)
         inst = Models::Instance[instance_id]
         raise "UnknownInstanceID" if inst.nil?
 
-        inst.netfilter_groups.map { |g| g.to_hash }
+        inst.security_groups.map { |g| g.to_hash }
       end
 
       #Returns an array containing the ip addresses of all instances in the same security group.
@@ -63,8 +63,10 @@ module Dcmgr
         raise "UnknownInstanceID" if inst.nil?
         raise "Unknown ip set." unless [:inside,:all,:outside].member?(set)
 
-        inst.netfilter_groups.compact.map { |netfilter_group|
-          netfilter_group.instances_dataset.lives.all.compact.map { |instance|
+        inst.security_groups.compact.map { |security_group|
+          # do not include 'stopped' or 'scheduling' instances as they
+          # are not allocated IP address.
+          security_group.instances_dataset.runnings.all.compact.map { |instance|
               instance.ips.compact.map { |ip|
                 case set
                 when :all
@@ -79,23 +81,32 @@ module Dcmgr
           }.flatten.uniq.compact
       end
 
-#      def get_instances_of_account_netfilter_group(account_id, netfilter_group_id)
-      def get_instances_of_account_netfilter_group(account_id, netfilter_group_name)
-        ng_map = Models::NetfilterGroup.find(:account_id => account_id, :name => netfilter_group_name)
-        raise "UnknownNetfilterGroupID" if ng_map.nil?
-        inst_maps = ng_map.instances_dataset.lives.all.map { |inst| inst.to_hash }.flatten.uniq.compact
-        inst_maps
-      end
-
       def get_network(network_id)
         network = Models::Network[network_id]
         raise "UnknownNetworkID" if network.nil?
         network.to_hash
       end
 
+      def get_nic(nic_uuid)
+        nic = Models::Taggable.find(nic_uuid)
+        raise "UnknownNIC" if nic.nil?
+        nic.to_hash
+      end
+
+      def get_instance_of_nic(nic_uuid)
+        nic = Models::Taggable.find(nic_uuid)
+        raise "UnknownNIC" if nic.nil?
+
+        instance = nic.instance
+        raise "UnknownInstance" if instance.nil?
+
+        instance.to_hash
+      end
+
       #Returns the current iplease for nic with uuid _nic_uuid_
       def get_iplease_for_nic(nic_uuid)
         nic = Models::Taggable.find(nic_uuid)
+        raise "UnknownNIC" if nic.nil?
         Models::IpLease.find(:instance_nic_id => nic[:id])[:ipv4]
       end
 
@@ -125,13 +136,13 @@ module Dcmgr
 
       def get_dhcp_conf(network_name)
         build_network_segment = proc { |network|
-          gwaddr = network.ipaddress
+          nwaddr = network.ipv4_ipaddress
           h = {
             :uuid => network.canonical_uuid,
-            :ipv4_first => gwaddr.network.first.to_s,
-            :ipv4_last  => gwaddr.network.last.to_s,
-            :ipv4_gw=>network.ipv4_gw,
-            :netmask => gwaddr.network.prefix.to_ip,
+            :ipv4_first => nwaddr.first.to_s,
+            :ipv4_last  => nwaddr.last.to_s,
+            :ipv4_gw=> network.ipv4_gw,
+            :netmask => nwaddr.prefix.to_ip,
             :prefix => network.prefix,
             :dns_server=> network.dns_server,
             :domain_name => network.domain_name,
@@ -149,8 +160,8 @@ module Dcmgr
               :ipaddr   => ip.ipv4
             }
             h[:addr2host] << {
-              :hostname => ip.instance_nic.instance.fqdn_hostname,
-              :ipaddr   => network.nat_network.nil? ? ip.ipv4 : ip.nat_outside_lease.ipv4 
+              :hostname => ip.instance_nic.fqdn_hostname,
+              :ipaddr   => ip.instance_nic.nat_network_id.nil? ? ip.ipv4 : ip.nat_outside_lease.ipv4 
             }
           }
           
@@ -168,6 +179,7 @@ module Dcmgr
         else
           raise "Unknown network name: #{network_name}"
         end
+
         h = {}
         network_set.each {|n|
           h[n.canonical_uuid] = build_network_segment.call(n)
@@ -175,30 +187,59 @@ module Dcmgr
         h
       end
 
-      def get_instances_of_netfilter_group(netfilter_group_id)
-        g = Models::NetfilterGroup[netfilter_group_id]
-        raise "UnknownNetfilterGroupID" if g.nil?
-        g.instances.map {|i| i.to_hash }.flatten.uniq.compact
+      def get_instances_of_security_group(security_group_uuid)
+        sg_map = Models::SecurityGroup[security_group_uuid]
+        raise "Unknown security group ID: #{security_group_uuid}" if sg_map.nil?
+        # do not include 'stopped' or 'scheduling' instances as they
+        # are not allocated IP address.
+        inst_maps = sg_map.instances_dataset.runnings.all.map { |inst| inst.to_hash }.flatten.uniq.compact
+        inst_maps
       end
 
       def get_alive_instances(node_id)
-        hp = Models::HostPool.find(:node_id => node_id)
+        hp = Models::HostNode.find(:node_id => node_id)
         if hp.nil?
-          logger.error("The node ID is not bound to HostPool yet: #{node_id}")
+          logger.error("The node ID is not bound to HostNode yet: #{node_id}")
           return []
         end
-        hps = Models::HostPool.where(:account_id => hp.account_id).all
+        hps = Models::HostNode.where(:account_id => hp.account_id).all
         inst_on_hp = hps.map { |hp|
-          inst_on_hp = hp.instances_dataset.lives.all.map { |inst|
+          inst_on_hp = hp.instances_dataset.runnings.all.map { |inst|
             inst_map = inst.to_hash
             # Does the hva have instance?
-            next unless inst_map[:host_pool][:node_id] == node_id
+            next unless inst_map[:host_node][:node_id] == node_id
             inst_map
           }
         }.flatten.uniq.compact
         inst_on_hp
       end
 
+      # Method designed to get all data needed for the netfilter service in a single request
+      def get_netfilter_data(node_id)
+        inst_maps = get_alive_instances(node_id)
+        
+        #Determine which security groups are in use
+        secg_uuids = inst_maps.map { |inst_map|
+          inst_map[:security_groups]
+        }.flatten.uniq
+        
+        secg_maps = Models::SecurityGroup.map { |secg|
+          secg.to_hash if secg_uuids.member? secg.canonical_uuid
+        }.compact
+        
+        # Trim out the data netfilter doesn't need and return a clean hash
+        {
+          :instances => inst_maps.map { |inst_map|
+            {
+            :uuid => inst_map[:uuid],
+            :security_groups => inst_map[:security_groups],
+            :vif => inst_map[:vif].map { |vif| vif.merge({:security_groups => inst_map[:security_groups]}) }# <-- Just putting the security groups in vif for the vnet isolator, gonna properly do this in the database asap
+            }
+          },
+          :security_groups => secg_maps
+        }
+      end
+
     end
   end
 end

data/lib/dcmgr/node_modules/instance_ha.rb

@@ -5,7 +5,7 @@ module Dcmgr
   module NodeModules
     class InstanceHA < Isono::NodeModules::Base
       include Dcmgr::Logger
-      
+
       initialize_hook do
         @thread_pool = Isono::ThreadPool.new(1, 'InstanceHA')
         event = Isono::NodeModules::EventChannel.new(node)
@@ -26,12 +26,19 @@ module Dcmgr
 
       def restart_instance(inst)
         # terminate and cleanup
+
+        # set boot_vol before instance is cleaned up.
+        case inst.image.boot_dev_type
+        when Models::Image::BOOT_DEV_SAN
+          boot_vol = inst.volume.find {|v| v.boot_dev == 1 }
+        end
+
         begin
-          jobrpc.run("kvm-handle.#{inst.host_pool.node_id}", 'cleanup', inst.canonical_uuid)
+          jobrpc.run("hva-handle.#{inst.host_node.node_id}", 'cleanup', inst.canonical_uuid)
         rescue => e
           # termination may fail
         end
-        
+
         # TODO: pick a new host node
         Isono::NodeModules::DataStore.barrier {
           inst.state = :failingover
@@ -39,18 +46,22 @@ module Dcmgr
         }
 
         # start a new backup instance
-        case inst.image.boot_dev_type
-        when Models::Image::BOOT_DEV_SAN
-          boot_vol = inst.volume.find {|v| v.boot_dev == 1 }
-          res = jobrpc.submit("kvm-handle.#{inst.host_pool.node_id}", 'run_vol_store', inst.canonical_uuid, boot_vol.canonical_uuid)
-        when Models::Image::BOOT_DEV_LOCAL
-          res = jobrpc.submit("kvm-handle.#{inst.host_pool.node_id}", 'run_local_store', inst.canonical_uuid)
-        else
-          raise "Unknown boot type"
+        begin
+          case inst.image.boot_dev_type
+          when Models::Image::BOOT_DEV_SAN
+            jobrpc.submit("scheduler", 'schedule_instance_ha', inst.canonical_uuid, boot_vol)
+          when Models::Image::BOOT_DEV_LOCAL
+            jobrpc.submit("scheduler", 'schedule_instance_ha', inst.canonical_uuid, nil)
+          else
+            raise "Unknown boot type"
+          end
+        rescue => e
+          # termination may fail
         end
+
         logger.info("#{inst.canonical_uuid} has been restarted")
       end
-      
+
       private
       def event
         @event ||= Isono::NodeModules::EventChannel.new(node)