dopv 0.11.0

data/lib/dopv/infrastructure/providers/openstack.rb

@@ -0,0 +1,308 @@
+require 'fog'
+
+module Dopv
+  module Infrastructure
+    class OpenStack < Base
+      extend Forwardable
+
+      def_delegator :@plan, :flavor, :flavor_name
+
+      def initialize(node_config, data_disks_db)
+        super(node_config, data_disks_db)
+
+
+        @compute_connection_opts = {
+          :provider                => 'openstack',
+          :openstack_username      => provider_username,
+          :openstack_api_key       => provider_password,
+          :openstack_project_name  => provider_tenant,
+          :openstack_domain_id     => provider_domain_id,
+          :openstack_auth_url      => provider_url,
+          :openstack_endpoint_type => provider_endpoint_type,
+          :connection_options      => {
+            :ssl_verify_peer => false,
+            #:debug_request   => true
+          }
+        }
+
+        @network_connection_opts = @compute_connection_opts
+        @volume_connection_opts  = @compute_connection_opts
+
+        @node_creation_opts = {
+          :name            => nodename,
+          :image_ref       => template.id,
+          :flavor_ref      => flavor.id,
+          :config_drive    => use_config_drive?,
+          :security_groups => security_groups
+        }
+      end
+
+      private
+
+      def provider_tenant
+        @provider_tenant ||= infrastructure_properties.tenant
+      end
+
+      def provider_domain_id
+        @provider_domain_id ||= infrastructure_properties.domain_id
+      end
+
+      def provider_endpoint_type
+        @provider_endpoint_type ||= infrastructure_properties.endpoint_type
+      end
+
+      def use_config_drive?
+        @use_config_drive ||= infrastructure_properties.use_config_drive?
+      end
+
+      def security_groups
+        @security_groups ||= infrastructure_properties.security_groups
+      end
+
+      def network_provider
+        Dopv::log.info("Node #{nodename}: Creating network provider.") unless @network_provider
+        @network_provider ||= @network_connection_opts ? ::Fog::Network.new(@network_connection_opts) : nil
+      end
+
+      def flavor(filters={})
+        @flavor ||= compute_provider.flavors(filters).find { |f| f.name == flavor_name }
+        raise ProviderError, "No such flavor #{flavor_name}" unless @flavor
+        @flavor
+      end
+
+      def network(name, filters={})
+        net = network_provider.networks(filters).find { |n| n.name == name || n.id == name }
+        raise ProviderError, "No such network #{name}" unless net
+        net
+      end
+
+      def subnet(name, filters={})
+        net = network_provider.subnets(filters).find { |s| s.name == name || s.id == name }
+        raise ProviderError, "No such subnetwork #{name}" unless net
+        net
+      end
+
+      def assign_security_groups(node_instance)
+        unless security_groups.empty?
+          Dopv::log.info("Node #{nodename}: Assigning security groups.")
+          config_sgs = security_groups.dup
+          node_instance.security_groups.uniq { |g| g.id }.each do |sg|
+            # Remove the security group from configuration if it is already
+            # assigned to an instance.
+            if config_sgs.delete(sg.name)
+              Dopv::log.debug("Node #{nodename}: Already assigned to security group #{sg.name}.")
+              next
+            end
+            # Remove the security group assignment if it isn't in the
+            # configuration.
+            unless config_sgs.include?(sg.name)
+              Dopv::log.debug("Node #{nodename}: Removing unneeded security group #{sg.name}.")
+              compute_provider.remove_security_group(node_instance.id, sg.name)
+              wait_for_task_completion(node_instance)
+            end
+          end
+          # Add remaining security groups defined in config array.
+          config_sgs.each do |sg_name|
+            begin
+              Dopv::log.debug("Node #{nodename}: Adding security group #{sg_name}.")
+              compute_provider.add_security_group(node_instance.id, sg_name)
+              wait_for_task_completion(node_instance)
+            rescue
+              raise ProviderError, "An error occured while assigning security group #{sg_name}"
+            end
+          end
+          node_instance.reload
+        end
+        node_instance.security_groups
+      end
+
+      def node_instance_stopped?(node_instance)
+        !node_instance.ready?
+      end
+
+      def wait_for_task_completion(node_instance)
+        node_instance.wait_for { ready? }
+      end
+
+      def create_node_instance
+        Dopv::log.info("Node #{nodename}: Creating node instance.")
+
+        @node_creation_opts[:nics] = add_node_network_ports
+        @node_creation_opts[:user_data_encoded] = [cloud_config].pack('m')
+
+        Dopv::log.debug("Node #{nodename}: Spawning node instance.")
+        instance = compute_provider.servers.create(@node_creation_opts)
+        wait_for_task_completion(instance)
+        instance.reload
+
+        assign_security_groups(instance)
+
+        instance
+      end
+
+      def destroy_node_instance(node_instance, destroy_data_volumes=false)
+        remove_node_floating_ips(node_instance)
+        remove_node_network_ports(node_instance)
+        super(node_instance, destroy_data_volumes)
+      end
+
+      def start_node_instance(node_instance)
+      end
+
+      def stop_node_instance(node_instance)
+        super(node_instance)
+        node_instance.wait_for { !ready? }
+      end
+
+      def add_node_volume(node_instance, config)
+        volume = super(
+          compute_provider, {
+            :name => config.name,
+            :display_name => config.name,
+            :size => config.size.gibibytes.to_i,
+            :volume_type => config.pool,
+            :description => config.name
+          }
+        )
+        volume.wait_for { ready? }
+        attach_node_volume(node_instance, volume.reload)
+        volume.reload
+      end
+
+      def destroy_node_volume(node_instance, volume)
+        volume_instance = detach_node_volume(node_instance, volume)
+        volume_instance.destroy
+        node_instance.volumes.all({}).reload
+      end
+
+      def attach_node_volume(node_instance, volume)
+        volume_instance = node_instance.volumes.all({}).find { |v| v.id = volume.id }
+        node_instance.attach_volume(volume_instance.id, nil)
+        volume_instance.wait_for { volume_instance.status.downcase == "in-use" }
+        volume_instance
+      end
+
+      def detach_node_volume(node_instance, volume)
+        volume_instance = node_instance.volumes.all({}).find { |v| v.id = volume.id }
+        node_instance.detach_volume(volume_instance.id)
+        volume_instance.wait_for { volume_instance.status.downcase == "available" }
+        volume_instance
+      end
+
+      def record_node_data_volume(volume)
+        super(
+          :name => volume.name,
+          :id   => volume.id,
+          :pool => volume.type == 'None' ? nil : volume.type,
+          :size => volume.size * 1073741824 # Returned in gibibytes
+        )
+      end
+
+      def fixed_ip(subnet_id, ip_address)
+        rval = { :subnet_id => subnet_id }
+        [:dhcp, :none].include?(ip_address) ? rval : rval.merge(:ip_address => ip_address)
+      end
+
+      def add_node_network_port(attrs)
+        ::Dopv::log.debug("Node #{nodename}: Adding network port #{attrs[:name]}.")
+        network_provider.ports.create(attrs)
+      end
+
+      def add_node_network_ports
+        ::Dopv::log.info("Node #{nodename}: Adding network ports.")
+        ports_config = {}
+        interfaces_config.each do |i|
+          s = subnet(i.network)
+          port_name = "#{nodename}_#{s.network_id}"
+          if ports_config.has_key?(port_name)
+              ports_config[port_name][:fixed_ips] << fixed_ip(s.id, i.ip)
+          else
+            ports_config[port_name] = {
+              :network_id => s.network_id,
+              :fixed_ips => [fixed_ip(s.id, i.ip)]
+            }
+          end
+        end
+        @network_ports = ports_config.map { |k,v| add_node_network_port(v.merge(:name => k)) }
+        @network_ports.collect { |p| {:net_id => p.network_id, :port_id => p.id} } # Net ID is required in Liberty++
+      end
+
+      def remove_node_network_ports(node_instance)
+        ::Dopv::log.warn("Node #{nodename}: Removing network ports.")
+        @network_ports ||= network_provider.ports.select { |p| p.device_id == node_instance.id } rescue {}
+        @network_ports.each { |p| p.destroy rescue nil } # TODO: dangerous, rewrite
+        @network_ports = {}
+      end
+
+      def add_node_floating_ip(attrs)
+        ::Dopv::log.debug("Node #{nodename}: Adding floating IP to #{attrs[:nicname]}.")
+        network_provider.floating_ips.create(attrs)
+      end
+
+      def add_node_floating_ips(node_instance)
+        ::Dopv::log.info("Node #{nodename}: Adding floating IPs.")
+        @network_ports ||= network_provider.ports.select { |p| p.device_id == node_instance.id }
+        interfaces_config.each do |i|
+          if i.floating_network
+            floating_network = network(i.floating_network)
+            subnetwork = subnet(i.network)
+            port = @network_ports.find { |p| p.fixed_ips.find { |f| f["subnet_id"] == subnetwork.id } }
+            attrs = {
+              :floating_network_id => floating_network.id,
+              :port_id => port.id,
+              :fixed_ip_address => port.fixed_ips.first["ip_address"],
+              :nicname => i.name
+            }
+            add_node_floating_ip(attrs)
+          end
+        end
+      end
+      alias_method :add_node_nics, :add_node_floating_ips
+
+      def remove_node_floating_ips(node_instance)
+        ::Dopv::log.warn("Node #{nodename}: Removing floating IPs.")
+        if node_instance
+          floating_ips = network_provider.floating_ips.select do |f|
+            node_instance.floating_ip_addresses.include?(f.floating_ip_address)
+          end
+          floating_ips.each { |f| f.destroy rescue nil } # TODO: dangerous, rewrite
+        end
+      end
+      alias_method :remove_node_nics, :remove_node_floating_ips
+
+      def cloud_config
+        config = "#cloud-config\n"        \
+          "hostname: #{hostname}\n"       \
+          "fqdn: #{fqdn}\n"               \
+          "manage_etc_hosts: True\n"      \
+          "ssh_pwauth: True\n"
+
+        if root_password
+          config <<                       \
+            "chpasswd:\n"                 \
+            "  list: |\n"                 \
+            "    root:#{root_password}\n" \
+            "  expire: False\n"
+        end
+
+        unless root_ssh_pubkeys.empty?
+          config <<                       \
+            "users:\n"                    \
+            "  - name: root\n"            \
+            "    ssh_authorized_keys:\n"
+          root_ssh_pubkeys.each { |k| config << "      - #{k}\n" }
+        end
+
+        config <<
+          "runcmd:\n" \
+          "  - service network restart\n"
+
+        config
+      end
+
+      def get_node_ip_addresses(node_instance)
+        (node_instance.ip_addresses + [node_instance.floating_ip_address]).flatten.uniq.compact
+      end
+    end
+  end
+end

data/lib/dopv/infrastructure/providers/ovirt.rb

@@ -0,0 +1,228 @@
+require 'fog'
+require 'uri'
+require 'open-uri'
+
+module Dopv
+  module Infrastructure
+    class Ovirt < Base
+      def initialize(plan, data_disks_db)
+        super(plan, data_disks_db)
+
+        @compute_connection_opts = {
+          :provider           => 'ovirt',
+          :ovirt_username     => provider_username,
+          :ovirt_password     => provider_password,
+          :ovirt_url          => provider_url,
+          :ovirt_ca_cert_file => provider_ca_cert_file
+        }
+
+        @node_creation_opts = {
+          :name               => nodename,
+          :template           => template.id,
+          :cores              => cores,
+          :memory             => memory.bytes,
+          :storage            => storage.bytes,
+          :cluster            => cluster.id,
+          :ha                 => keep_ha?,
+          :clone              => full_clone?,
+          :storagedomain_name => infrastructure_properties.default_pool
+        }
+      end
+
+      private
+
+      def compute_provider
+        unless @compute_provider
+          super
+          ::Dopv::log.debug("Node #{nodename}: Recreating client with proper datacenter.")
+          @compute_connection_opts[:ovirt_datacenter] = datacenter[:id]
+          @compute_provider = ::Fog::Compute.new(@compute_connection_opts)
+        end
+        @compute_provider
+      end
+
+      def get_node_instance
+        super({:without_details => true})
+      end
+
+      def wait_for_task_completion(node_instance)
+        node_instance.wait_for { !locked? }
+      end
+
+      def create_node_instance
+        node_instance = super
+
+        # For each disk, set up wipe after delete flag
+        node_instance.volumes.each do |v|
+          ::Dopv::log.debug("Node #{nodename}: Setting wipe after delete for disk #{v.alias}.")
+          update_node_volume(node_instance, v, {:wipe_after_delete => true})
+        end
+
+        node_instance
+      end
+
+      def customize_node_instance(node_instance)
+        ::Dopv::log.info("Node #{nodename}: Customizing node.")
+        customization_opts = {
+          :hostname => fqdn,
+          :dns => dns.name_servers,
+          :domain => dns.search_domains,
+          :user => 'root',
+          :password => root_password,
+          :ssh_authorized_keys => root_ssh_pubkeys
+        }
+
+        customization_opts[:nicsdef] = interfaces_config.collect do |i|
+          nic = {}
+          nic[:nicname] = i.name
+          nic[:on_boot] = 'true'
+          nic[:boot_protocol] = case i.ip
+                                when :dhcp
+                                  'DHCP'
+                                when :none
+                                  'NONE'
+                                else
+                                  'STATIC'
+                                end
+          unless [:dhcp, :none].include?(i.ip)
+            nic[:ip] = i.ip
+            nic[:netmask] = i.netmask
+            nic[:gateway] = i.gateway if i.set_gateway?
+          end
+          nic
+        end
+
+        customization_opts
+      end
+
+      def start_node_instance(node_instance)
+        customization_opts = super(node_instance)
+        node_instance.service.vm_start_with_cloudinit(
+          :id => node_instance.id,
+          :user_data => customization_opts
+        )
+      end
+
+      # Redefine until regexp in Fog::Compute::Ovirt::Server#stopped? is fixed
+      def stop_node_instance(node_instance)
+        super
+        node_instance.wait_for { status.downcase == 'down' }
+      end
+
+      def add_node_nic(node_instance, attrs)
+        nic = node_instance.add_interface(attrs)
+        node_instance.interfaces.reload
+        nic
+      end
+
+      def update_node_nic(node_instance, nic, attrs)
+        node_instance.update_interface(attrs.merge({:id => nic.id}))
+        node_instance.interfaces.reload
+      end
+
+      def add_node_nics(node_instance)
+        ::Dopv::log.info("Node #{nodename}: Trying to add interfaces.")
+
+        # Remove all interfaces defined by the template
+        remove_node_nics(node_instance) { |n, i| n.destroy_interface(:id => i.id) }
+
+        # fetch first network for our reservation dance
+        first_network = cluster.networks.first.name
+        # Reserve MAC addresses
+        (1..interfaces_config.size).each do |i|
+          name = "tmp#{i}"
+          ::Dopv::log.debug("Node #{nodename}: Creating interface #{name}.")
+          attrs = {
+            :name         => name,
+            :network_name => first_network,
+            :plugged      => true,
+            :linked       => true
+          }
+          add_node_nic(node_instance, attrs)
+        end
+
+        # Rearrange interfaces by their MAC addresses and assign them into
+        # appropriate networks
+        ic = interfaces_config.reverse
+        node_instance.interfaces.sort_by do |n| n.mac
+          i = ic.pop
+          ::Dopv::log.debug("Node #{nodename}: Configuring interface #{n.name} (#{n.mac}) as #{i.name} in #{i.network}.")
+          attrs = {
+            :name => i.name,
+            :network_name => i.network,
+          }
+          update_node_nic(node_instance, n, attrs)
+        end
+      end
+
+      def add_node_affinity(node_instance, name)
+        affinity_group = compute_provider.affinity_groups.find { |g| g.name == name }
+        raise ProviderError, "No such affinity group #{name}" unless affinity_group
+        ::Dopv::log.info("Node #{nodename}: Adding node to affinity group #{name}.")
+        node_instance.add_to_affinity_group(:id => affinity_group.id)
+      end
+
+      def add_node_volume(node_instance, config)
+        storage_domain = compute_provider.storage_domains.find { |d| d.name == config.pool }
+        raise ProviderError, "No such storage domain #{storage_domain_name}" unless storage_domain
+
+        node_instance.add_volume(
+          {
+            :alias => config.name,
+            :size => config.size.bytes,
+            :bootable => 'false',
+            :wipe_after_delete => 'true',
+            :storage_domain => storage_domain.id
+          }.tap { |h| (h[:format] = 'raw'; h[:sparse] = 'false') unless config.thin? }
+        )
+        wait_for_task_completion(node_instance)
+        node_instance.volumes.find { |v| v.alias == config.name } # TODO: Rewrite with volume.reload if possible
+      end
+
+      def destroy_node_volume(node_instance, volume)
+        node_instance.destroy_volume(:id => volume.id)
+        wait_for_task_completion(node_instance)
+        node_instance.volumes.reload
+      end
+
+      def attach_node_volume(node_instance, volume)
+        node_instance.attach_volume(:id => volume.id)
+        wait_for_task_completion(node_instance)
+        node_instance.volumes.reload
+      end
+
+      def detach_node_volume(node_instance, volume)
+        node_instance.detach_volume(:id => volume.id)
+        wait_for_task_completion(node_instance)
+        node_instance.volumes.reload
+      end
+
+      def record_node_data_volume(volume)
+        super(
+          :name => volume.alias,
+          :id   => volume.id,
+          :pool => volume.storage_domain,
+          :size => volume.size
+        )
+      end
+
+      def provider_ca_cert_file
+        uri = infrastructure.endpoint
+        local_ca_file = "#{TMP}/#{uri.host}_#{uri.port}_ca.crt"
+        remote_ca_file = "#{uri.scheme}://#{uri.host}:#{uri.port}/ca.crt"
+        unless File.exists?(local_ca_file)
+          begin
+            open(remote_ca_file, :ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE) do |remote_ca|
+              local_ca = open(local_ca_file, 'w')
+              local_ca.write(remote_ca.read)
+              local_ca.close
+            end
+          rescue
+            raise ProviderError, "Cannot download CA certificate from #{provider_url}"
+          end
+        end
+        local_ca_file
+      end
+    end
+  end
+end