wakame-vdc-agents 11.06.0 → 11.12.0

data/lib/dcmgr/scheduler/host_node/specify_node.rb

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr
+  module Scheduler
+    module HostNode
+
+      # Find host node which has the same request_params[:host_node_id].
+      class SpecifyNode < HostNodeScheduler
+        include Dcmgr::Logger
+
+        def schedule(instance)
+          # TODO:
+          #  "host_id" and "host_pool_id" will be obsolete.
+          #  They are used in lib/dcmgr/endpoints/core_api.rb.
+          host_node_uuid = instance.request_params['host_id'] || instance.request_params['host_pool_id'] || instance.request_params['host_node_id']
+          ds = Models::HostNode.online_nodes.filter(:uuid=>Models::HostNode.trim_uuid(host_node_uuid))
+
+          host_node = ds.first
+
+          raise HostNodeSchedulingError if host_node.nil?
+          instance.host_node = host_node
+        end
+      end
+    end
+  end
+end

data/lib/dcmgr/scheduler/network/flat_single.rb

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr
+  module Scheduler
+    module Network
+      # Simple network scheduler
+      # assign IP address from first found network to single interface.
+      class FlatSingle < NetworkScheduler
+        
+        def schedule(instance)
+          # add single interface and set network
+          network = Models::Network.first
+          vif_template = instance.spec.vifs[instance.spec.vifs.keys.first] ||
+            {:index=>0, :bandwidth=>100000}
+          
+          vnic = instance.add_nic(vif_template)
+          vnic.network = network
+          vnic.save
+        end
+      end
+    end
+  end
+end

data/lib/dcmgr/scheduler/network/nat_one_to_one.rb

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr
+  module Scheduler
+    module Network
+      # One internal address + NAT external address to single interface.
+      class NatOneToOne < NetworkScheduler
+        
+        def schedule(instance)
+          network = Models::Network[@options.network_id]
+          nat_network = Models::Network[@options.nat_network_id]
+
+          vif_template = instance.spec.vifs.find{ |name,v| v[:index] == 0 }.last
+
+          vnic = instance.add_nic(vif_template)
+          vnic.network = network
+          vnic.nat_network = nat_network
+          vnic.save
+        end
+      end
+    end
+  end
+end

data/lib/dcmgr/scheduler/network/per_instance.rb

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr
+  module Scheduler
+    module Network
+      # Meta scheduler calls another scheduler specified by user.
+      class PerInstance < NetworkScheduler
+        include Dcmgr::Logger
+        
+        def schedule(instance)
+          sched_opts = @options.to_hash || {}
+          sched_name = instance.request_params['network_scheduler']
+          if sched_name.nil? || sched_name == ''
+            if sched_opts.has_key?(:default)
+              sched_conf = @options.default
+            else
+              raise "Missing network_scheduler parameter from the request." 
+            end
+          else
+            if sched_opts.has_key?(sched_name.to_sym)
+              sched_conf = @options.send(sched_name.to_sym)
+            else
+              raise "Unknown scheduler definition: #{sched_name} for the instance #{instance.canonical_uuid}"
+            end
+          end
+          
+          sched_class = Scheduler.scheduler_class(sched_conf.scheduler, ::Dcmgr::Scheduler::Network)
+          sched = if sched_conf.respond_to?(:options)
+                    sched_class.new(sched_conf.options)
+                  else
+                    sched_class.new
+                   end
+          logger.info("Selected network scheduler: #{sched_name} #{sched_class} for the instance #{instance.canonical_uuid}")
+          sched.schedule(instance)
+        end
+      end
+    end
+  end
+end

data/lib/dcmgr/scheduler/network/vif_template.rb

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr
+  module Scheduler
+    module Network
+      # Setup vnics by following InstanceSpec#vifs template.
+      class VifTemplate < NetworkScheduler
+        
+        def schedule(instance)
+          instance.spec.vifs.each { |name, vif|
+            vnic = instance.add_nic(vif)
+            vnic.network = Models::Network[@options[name]]
+            vnic.save
+          }
+        end
+      end
+    end
+  end
+end

data/lib/dcmgr/scheduler/storage_node/find_first.rb

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr; module Scheduler; module StorageNode
+  class FindFirst < StorageNodeScheduler
+
+    protected
+    def schedule_node(volume)
+      params = volume.request_params
+
+      volume.storage_node = Models::StorageNode.first
+    end
+  end
+end; end; end

data/lib/dcmgr/scheduler/storage_node/least_usage.rb

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr
+  module Scheduler
+    module StorageNode
+
+      # Find storage node which has the largest available disk space.
+      class LeastUsage < StorageNodeScheduler
+        include Dcmgr::Logger
+        
+        def schedule(volume)
+          storage_node = Models::StorageNode.online_nodes.all.find_all { |s|
+            s.free_disk_space >= volume.size
+          }.sort_by { |s|
+            s.free_disk_space
+          }.reverse.first
+          raise StorageNodeSchedulingError if storage_node.nil?
+          volume.storage_node = storage_node
+        end
+      end
+    end
+  end
+end

data/lib/dcmgr/storage_service.rb

@@ -5,40 +5,39 @@ module Dcmgr
 
     @snapshot_repository_config = nil
     
-    def initialize(provider, access_key, secret_key)
+    def initialize(driver, options = {})
+      @driver = driver
       @account = {}
-      @account[:provider] = provider.upcase
-      @account[:access_key] = access_key
-      @account[:secret_key] = secret_key
+      @account[:id] = options[:account_id]
+      @account[:access_key] = options[:access_key]
+      @account[:secret_key] = options[:secret_key]
     end
     
     def self.snapshot_repository_config
       if @snapshot_repository_config.nil?
-        config_file = YAML.load_file(File.join(File.expand_path(DCMGR_ROOT), 'config', 'snapshot_repository.yml'))
+        config_file = YAML.load_file(File.join(File.expand_path('../../../', __FILE__), 'config', 'snapshot_repository.yml'))
         @snapshot_repository_config = config_file
       else
         @snapshot_repository_config
       end
     end
 
-    def self.has_driver?(driver)
-      %w(S3 IIJGIO).include? driver.upcase
-    end
-    
-    def bucket(name)
-      case @account[:provider]
-        when 'S3'
-          @driver = Dcmgr::Drivers::S3Storage.new(name)
-        when 'IIJGIO'
-          @driver = Dcmgr::Drivers::IIJGIOStorage.new(name)
+    def snapshot_storage(bucket, path)
+      case @driver
+        when 'local'
+          @storage = Dcmgr::Drivers::LocalStorage.new(@account[:id], bucket, path)
+        when 's3'
+          @storage = Dcmgr::Drivers::S3Storage.new(@account[:id], bucket, path)
+        when 'iijgio'
+          @storage = Dcmgr::Drivers::IIJGIOStorage.new(@account[:id], bucket, path)
       else
-        raise "#{@account[:provider]} is not a recognized storage provider"
+        raise "#{@driver} is not a recognized storage driver"
       end
 
-      @driver.setenv('SERVICE', @account[:provider].downcase)
-      @driver.setenv('ACCESS_KEY_ID', @account[:access_key])
-      @driver.setenv('SECRET_ACCESS_KEY', @account[:secret_key])
-      @driver
+      @storage.setenv('SERVICE', @driver)
+      @storage.setenv('ACCESS_KEY_ID', @account[:access_key])
+      @storage.setenv('SECRET_ACCESS_KEY', @account[:secret_key])
+      @storage
     end
 
     def self.repository(repository_address)
@@ -47,32 +46,33 @@ module Dcmgr
       end
       tmp = repository_address.split(',')
       destination_key = tmp[0]
-      dest = destination_key.match(/^([a-z0-9_]+)@([a-z0-9_-]+):([a-z0-9_-]+):([a-z0-9_\-\/]+)(snap-[a-z0-9]+\.zsnap)+$/)
-      if dest.nil?
-        raise "Invalid format: #{repository_address}"
-      end 
-          
-      h = { 
-        :destination => dest[1],
-        :driver => dest[2],
-        :bucket => dest[3],
-        :path => dest[4],
-        :filename => dest[5],
+
+      # ex. 'local@local:none:/home/ubuntu/work/repos/git/github.com/wakame-vdc/tmp/snap/a-shpoolxx/snap-gkosnc56.snap'
+      # dest = destination_key.match(/^([a-z0-9_]+)@([a-z0-9_-]+):([a-z0-9_-]+):([a-z0-9._\-\/]+)(snap-[a-z0-9]+\.snap)+$/)
+      results  = destination_key.split(':', 3)
+      accounts = results[0].split('@', 2)
+
+      h = {
+        :destination => accounts[0],
+        :driver => accounts[1],
+        :bucket => results[1],
+        :path => File.dirname(results[2]),
+        :filename => File.basename(results[2]),
         :access_key => tmp[1],
         :secret_key => tmp[2],
-      }   
-    end 
+      }
+    end
 
     def self.repository_address(destination_key)
       format = '%s,%s,%s'
       config_data = self.snapshot_repository_config
       destination = destination_key.split('@')[0]
       
-      if destination == 'local'
-        config = {:access_key => '', :secret_key => ''}
-      else
-        config = config_data[destination]
-      end
+      config = if destination == 'local'
+                 {'access_key' => '', 'secret_key' => ''}
+               else
+                 config_data[destination]
+               end
       
       sprintf(format, *[destination_key, config["access_key"], config["secret_key"]])
     end
@@ -90,9 +90,8 @@ module Dcmgr
         if config.nil?
           raise "Destination isn't exists"
         end
-        store_path = "snapshots/#{account_id}/"
       end
-      sprintf(format, *[destination, config["driver"], config["bucket"], File.join(store_path, filename)])
+      sprintf(format, *[destination, config["driver"], config["bucket"], File.join("#{store_path}/#{account_id}/", filename)])
     end
   end
 end

data/lib/dcmgr/tags.rb

@@ -32,12 +32,12 @@ module Dcmgr::Tags
   
   class HostPool < Models::Tag
     def accept_mapping?(to)
-      to.is_a?(Dcmgr::Models::HostPool)
+      to.is_a?(Dcmgr::Models::HostNode)
     end
 
     def pick(spec)
       mapped_uuids.map { |t|
-        Dcmgr::Models::HostPool[t.uuid]
+        Dcmgr::Models::HostNode[t.uuid]
       }.find_all { |h|
         h.check_capacity(spec)
       }.sort_by { |h|
@@ -48,7 +48,7 @@ module Dcmgr::Tags
   
   class StoragePool < Models::Tag
     def accept_mapping?(to)
-      to.is_a?(Dcmgr::Models::StoragePool)
+      to.is_a?(Dcmgr::Models::StorageNode)
     end
   end
 end

data/lib/dcmgr/version.rb

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 module Dcmgr
-  VERSION_MAJOR="11.06"
+  VERSION_MAJOR="11.12"
   VERSION_MINOR=0
 
   VERSION="#{VERSION_MAJOR}.#{VERSION_MINOR}"

data/lib/dcmgr/vnet.rb

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr
+  module VNet
+
+    # Abstract class for a Cache implementation to extend
+    class Cache
+      # Makes a call to the database and updates the Cache
+      def update
+        raise NotImplementedError
+      end
+      
+      # Returns the cache
+      # if _force_update_ is set to true, the cache will be updated from the database
+      def get(force_update = false)
+        raise NotImplementedError
+      end
+      
+      # Adds a newly started instance to the existing cache
+      def add_instance(inst_map)
+        raise NotImplementedError
+      end
+      
+      # Removes a terminated instance from the existing cache
+      def remove_instance(inst_id)
+        raise NotImplementedError
+      end
+    end
+    
+    # A controller interface to be implemented
+    class Controller
+      def apply_instance(instance)
+        raise NotImplementedError
+      end
+      
+      def remove_instance(instance)
+        raise NotImplementedError
+      end
+      
+      def join_security_group(instance,group)
+        raise NotImplementedError
+      end
+      
+      def leave_security_group(instance,group)
+        raise NotImplementedError
+      end
+      
+      def update_security_group(group)
+        raise NotImplementedError
+      end
+    end
+    
+    class Rule
+      
+    end
+    
+    class Task
+      #Must be an array of rules
+      attr_accessor :rules
+      
+      def initialize
+        @rules = []
+      end
+      
+    end
+  
+    # Abstract class for task managers to extend
+    # A task manager should be able to understand certain rules in a task and be able to apply those
+    class TaskManager
+      
+      def apply_task(task)
+        raise NotImplementedError
+      end
+
+      def apply_tasks(tasks)
+        raise ArgumentError, "tasks must be an Array of Tasks." unless tasks.is_a?(Array)
+        tasks.each { |task|
+          next unless task.is_a?(Task)
+          apply_task(task)
+        }
+      end
+
+      def remove_task(task)
+        raise NotImplementedError
+      end
+
+      #TODO: Change Array to Enumerable
+      def remove_tasks(tasks)
+        raise ArgumentError, "tasks must be an Array of Tasks." unless tasks.is_a?(Array)
+        tasks.each { |task|
+          next unless task.is_a?(Task)
+          remove_task(task)
+        }
+      end
+    end
+
+    # Abstract class that determines how to isolate instances (vnics) from each other
+    class Isolator
+      def determine_friends(me,others)
+        raise notImplementedError
+      end
+    end
+    
+  end
+end

data/lib/dcmgr/vnet/factories.rb

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+
+module Dcmgr
+  module VNet
+    # This file stores the factories. It is their job to read config files etc. to decide which implementation to use.
+    V = Dcmgr::VNet
+    class ControllerFactory
+      def self.create_controller(node)
+        V::Netfilter::NetfilterController.new(node)
+      end
+    end
+    
+    class IsolatorFactory
+      def self.create_isolator
+        V::Isolators::BySecurityGroup.new
+        #V::Isolators::DummyIsolator.new
+      end
+    end
+    
+    class TaskManagerFactory
+      def self.create_task_manager(node)
+        manager = V::Netfilter::VNicProtocolTaskManager.new
+        manager.enable_ebtables = node.manifest.config.enable_ebtables
+        manager.enable_iptables = node.manifest.config.enable_iptables
+        manager.verbose_commands = node.manifest.config.verbose_netfilter
+        
+        manager
+      end
+    end
+    
+    class TaskFactory
+      extend Dcmgr::Helpers::NicHelper
+      include V::Tasks
+
+      def self.create_tasks_for_isolation(vnic,friends,node)
+        tasks = []
+        enable_logging = node.manifest.config.packet_drop_log
+        ipset_enabled = node.manifest.config.use_ipset
+        friend_ips = friends.map {|vnic_map| vnic_map[:ipv4][:address]}
+        
+        tasks << AcceptARPFromFriends.new(vnic[:ipv4][:address],friend_ips,enable_logging,"A arp friend #{vnic[:uuid]}")
+        #tasks << AcceptIpFromFriends(friend_ips)
+        
+        if is_natted? vnic          
+          # Friends don't use NAT, friends talk to each other with their REAL ip addresses.
+          # It's a heart warming scene, really
+          if ipset_enabled
+            # Not implemented yet
+            #tasks << ExcludeFromNatIpSet.new(friend_ips,vnic[:ipv4][:address])
+          else
+            tasks << ExcludeFromNat.new(friend_ips,vnic[:ipv4][:address])
+          end
+        end
+        
+        tasks
+      end
+
+      # Returns the tasks required for applying this security group
+      def self.create_tasks_for_secgroup(secgroup)
+        [SecurityGroup.new(secgroup)]
+      end
+
+      # Returns the tasks that drop all traffic
+      def self.create_drop_tasks_for_vnic(vnic,node)
+        enable_logging = node.manifest.config.packet_drop_log
+
+        #TODO: Add logging to ip drops
+        [DropIpFromAnywhere.new, DropArpForwarding.new(enable_logging,"D arp #{vnic[:uuid]}: ")]
+      end
+      
+      # Creates tasks related to network address translation
+      def self.create_nat_tasks_for_vnic(vnic,node)
+        #friend_ips = friends.map {|vnic_map| vnic_map[:ipv4][:address]}
+        #ipset_enabled = node.manifest.config.use_ipset
+        tasks = []
+        
+        # Nat tasks
+        if is_natted? vnic          
+          # Exclude instances in the same security group form using nat
+          #if ipset_enabled
+            # Not implemented yet
+            #tasks << ExcludeFromNatIpSet.new(friend_ips,vnic[:ipv4][:address])
+          #else
+            #tasks << ExcludeFromNat.new(friend_ips,vnic[:ipv4][:address])
+          #end
+          
+          tasks << StaticNatLog.new(vnic[:ipv4][:address], vnic[:ipv4][:nat_address], "SNAT #{vnic[:uuid]}", "DNAT #{vnic[:uuid]}") if node.manifest.config.packet_drop_log
+          tasks << StaticNat.new(vnic[:ipv4][:address], vnic[:ipv4][:nat_address], clean_mac(vnic[:mac_addr]))
+        end
+        
+        tasks << TranslateMetadataAddress.new(vnic[:uuid],vnic[:ipv4][:network][:metadata_server],vnic[:ipv4][:network][:metadata_server_port] || 80) unless vnic[:ipv4][:network][:metadata_server].nil?
+        
+        tasks
+      end
+
+      #Returns the netfilter tasks required for this vnic
+      # The _friends_ parameter is an array of vnic_maps that should not be isolated from _vnic_
+      def self.create_tasks_for_vnic(vnic,friends,security_groups,node)
+        tasks = []
+
+        host_addr = Isono::Util.default_gw_ipaddr
+        enable_logging = node.manifest.config.packet_drop_log
+        ipset_enabled = node.manifest.config.use_ipset
+        
+        # Drop all traffic that isn't explicitely accepted
+        tasks += self.create_drop_tasks_for_vnic(vnic,node)
+        
+        # General data link layer tasks
+        tasks << AcceptARPToHost.new(host_addr,vnic[:ipv4][:address],enable_logging,"A arp to_host #{vnic[:uuid]}: ")
+        tasks << AcceptARPFromGateway.new(vnic[:ipv4][:network][:ipv4_gw],enable_logging,"A arp from_gw #{vnic[:uuid]}: ") unless vnic[:ipv4][:network][:ipv4_gw].nil?
+        tasks << DropIpSpoofing.new(vnic[:ipv4][:address],enable_logging,"D arp sp #{vnic[:uuid]}: ")
+        tasks << DropMacSpoofing.new(clean_mac(vnic[:mac_addr]),enable_logging,"D ip sp #{vnic[:uuid]}: ")
+        tasks << AcceptArpBroadcast.new(host_addr,enable_logging,"A arp bc #{vnic[:uuid]}: ")
+        
+        # General ip layer tasks
+        tasks << AcceptIcmpRelatedEstablished.new
+        tasks << AcceptTcpRelatedEstablished.new
+        tasks << AcceptUdpEstablished.new
+        tasks << AcceptAllDNS.new
+        tasks << AcceptWakameDHCPOnly.new(vnic[:ipv4][:network][:dhcp_server]) unless vnic[:ipv4][:network][:dhcp_server].nil?
+        # Accept OUTGOING traffic from instances to anywhere in the network
+        #tasks << AcceptIpToAnywhere.new
+        
+        # VM isolation based
+        tasks += self.create_tasks_for_isolation(vnic,friends,node)
+        tasks += self.create_nat_tasks_for_vnic(vnic,node)
+        
+        # Accept ip traffic from the gateway that isn't blocked by other tasks
+        tasks << AcceptIpFromGateway.new(vnic[:ipv4][:network][:ipv4_gw]) unless vnic[:ipv4][:network][:ipv4_gw].nil?
+        
+        # Security group tasks
+        security_groups.each { |secgroup|
+          tasks += self.create_tasks_for_secgroup(secgroup)
+        }
+        
+        tasks
+      end
+    end
+      
+  end
+end