dopv 0.11.0

data/lib/dopv/cli/command_add.rb

@@ -0,0 +1,37 @@
+module Dopv
+  module Cli
+
+    def self.command_add(base)
+      base.class_eval do
+
+        desc 'Add a new plan file to the plan store'
+        arg_name 'plan_file'
+
+        command :add do |c|
+          c.desc 'Update the plan if it already exists in plan store'
+          c.switch [:update, :u], :negatable => false
+
+          c.action do |global_options, options, args|
+            help_now!('Add takes exactly one argument, a plan file.') if
+              args.empty? || args.length != 1
+
+            plan_file = args[0]
+
+            exit_now!("The plan file #{plan_file} must be a readable file.") unless
+              File.file?(plan_file) && File.readable?(plan_file)
+
+            begin
+              puts Dopv.add(plan_file)
+            rescue DopCommon::PlanExistsError => e
+              if options[:update]
+                puts Dopv.update_plan(plan_file, {})
+              else
+                raise "#{e}, please use 'dopv update' first, or use -u|--update flag to add this plan forcibly."
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dopv/cli/command_export.rb

@@ -0,0 +1,26 @@
+module Dopv
+  module Cli
+
+    def self.command_export(base)
+      base.class_eval do
+
+        desc 'Export the internal data disks state into a local file'
+        arg_name 'plan_name data_disks_file'
+        command :export do |c|
+          c.action do |global_options, options, args|
+            help_now!('Export takes exactly two arguments, plan name and data disks file.') if
+              args.empty? || args.length != 2
+
+            plan_name, data_disks_file = args
+            data_disks_dir = File.dirname(data_disks_file)
+
+            exit_now!("The #{data_disks_dir} must be a directory writable by the process.") unless
+              File.directory?(data_disks_dir) && File.writable?(data_disks_dir)
+
+            Dopv.export_state_file(plan_name, data_disks_file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dopv/cli/command_import.rb

@@ -0,0 +1,32 @@
+module Dopv
+  module Cli
+
+    def self.command_import(base)
+      base.class_eval do
+
+        desc 'Import data disks from a file into internal state store of the given plan'
+        arg_name 'plan_name data_disks_file'
+        command :import do |c|
+          c.desc 'Force plan import'
+          c.switch [:f, :force], :negatable => false
+
+          c.action do |global_options, options, args|
+            help_now!('Import takes exactly two arguments, a plan name and data disks file.') if
+              args.empty? || args.length != 2
+
+            plan_name, data_disks_file = args
+
+            exit_now!("The #{data_disks_file} must be a readable file.") unless
+              File.file?(data_disks_file) && File.readable?(data_disks_file)
+
+            if !Dopv.export_state(plan_name).empty? && !options[:force]
+              exit_now!("The internal plan's state is not empty, please use the '-f|--force' flag to overwrite.")
+            end
+
+            Dopv.import_state_file(plan_name, data_disks_file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dopv/cli/command_list.rb

@@ -0,0 +1,18 @@
+module Dopv
+  module Cli
+
+    def self.command_list(base)
+      base.class_eval do
+
+        desc 'List plans stored in the plan store'
+        
+        command :list do |c|
+          c.action do |global_options,options,args|
+            puts Dopv.list
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/dopv/cli/command_remove.rb

@@ -0,0 +1,29 @@
+module Dopv
+  module Cli
+
+    def self.command_remove(base)
+      base.class_eval do
+
+        desc 'Remove existing plan from the plan store'
+        arg_name 'plan_name'
+
+        command :remove do |c|
+          c.desc 'Keep the DOPi state file'
+          c.switch [:k, :keep_dopi_state], :negatable => false
+
+          c.desc 'Remove the DOPv state file (THIS REMOVES THE DISK STATE!)'
+          c.switch [:r, :remove_dopv_state], :negatable => false
+
+          c.action do |global_options, options, args|
+            help_now!('Remove take exactly one argument, a plan name.') if
+              args.empty? || args.length != 1
+
+            plan_name = args[0]
+
+            Dopv.remove(plan_name, !options[:keep_dopi_state], options[:remove_dopv_state])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dopv/cli/command_run.rb

@@ -0,0 +1,38 @@
+module Dopv
+  module Cli
+
+    def self.command_run(base, action)
+      base.class_eval do
+
+        desc "#{action.capitalize} a plan."
+        arg_name 'plan_name'
+
+        command action do |c|
+          if action == :undeploy
+            c.desc 'Remove data disks from the state and cloud provider.'
+            c.switch [:rmdisk, :r], :default_value => false
+          end
+
+          DopCommon::Cli.node_select_options(c)
+
+          c.action do |global_options, options, args|
+            options[:run_for_nodes] = DopCommon::Cli.parse_node_select_options(options)
+
+            help_now!("#{action.capitalize} takes exactly one argument, a plan name.") if
+              args.empty? || args.length > 1
+
+            plan_name = args[0]
+
+            begin
+              case action
+              when :deploy   then Dopv.deploy(plan_name, options)
+              when :undeploy then Dopv.undeploy(plan_name, options)
+              when :refresh  then Dopv.refresh(plan_name, options)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dopv/cli/command_update.rb

@@ -0,0 +1,35 @@
+module Dopv
+  module Cli
+
+    def self.command_update(base)
+      base.class_eval do
+
+        desc 'Update the plan and/or the plan state for a given plan yaml or plan name.'
+        arg_name 'plan_file_or_name'
+
+        command :update do |c|
+          c.desc 'Remove the existing disk information and start with a clean state.'
+          c.switch [:clear, :c], :default_value => false
+
+          c.desc 'Ignore the update and set the state version to the latest version.'
+          c.switch [:ignore, :i], :default_value => false
+
+          c.action do |global_options, options, args|
+            help_now!('Update takes exactly one argument, the plan name or file.') if
+              args.empty? || args.length != 1
+
+            plan = args[0]
+
+            if Dopv.list.include?(plan)
+              Dopv.update_state(plan, options)
+            elsif File.file?(plan) && File.readable?(plan)
+              Dopv.update_plan(plan, options)
+            else
+              exit_now!("No such plan '#{plan}' in the store or the plan file doesn't exist or is unreadable.")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dopv/cli/command_validate.rb

@@ -0,0 +1,30 @@
+module Dopv
+  module Cli
+
+    def self.command_validate(base)
+      base.class_eval do
+
+        desc 'Validate a plan file.'
+        arg_name 'plan_file'
+
+        command :validate do |c|
+          c.action do |global_options, options, args|
+            help_now!('Validate takes excatly one argument, a plan file') if
+              args.empty? || args.length != 1
+
+            plan_file = args[0]
+
+            exit_now!("The #{plan_file} must exist and be a readable file") unless
+              File.file?(plan_file) && File.readable?(plan_file)
+
+            if Dopv.valid?(plan_file)
+              puts('Plan is valid.')
+            else
+              exit_now!('Plan is NOT valid!')
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dopv/infrastructure.rb

@@ -0,0 +1,40 @@
+require 'dopv/infrastructure/providers/base'
+
+module Dopv
+  module Infrastructure
+    TMP = '/tmp'
+
+    PROVIDER_BASE = 'dopv/infrastructure/providers'
+
+    PROVIDER_CLASSES = {
+      :ovirt      => 'Ovirt',
+      :rhev       => 'Ovirt',
+      :openstack  => 'OpenStack',
+      :vsphere    => 'Vsphere',
+      :vmware     => 'Vsphere',
+      :baremetal  => 'BareMetal'
+    }
+
+    def self.load_provider(provider)
+      require "#{PROVIDER_BASE}/#{PROVIDER_CLASSES[provider].downcase}"
+      klass_name = "Dopv::Infrastructure::#{PROVIDER_CLASSES[provider]}"
+      klass_name.split('::').inject(Object) { |res, i| res.const_get(i) }
+    end
+
+    def self.bootstrap_node(plan, state_store)
+      provider = load_provider(plan.infrastructure.provider)
+      provider.bootstrap_node(plan, state_store)
+    end
+
+    def self.destroy_node(plan, state_store, destroy_data_volumes=false)
+      provider = load_provider(plan.infrastructure.provider)
+      provider.destroy_node(plan, state_store, destroy_data_volumes)
+    end
+
+    def self.refresh_node(plan, state_store)
+      provider = load_provider(plan.infrastructure.provider)
+      provider.refresh_node(plan, state_store)
+    end
+
+  end
+end

data/lib/dopv/infrastructure/providers/baremetal.rb

@@ -0,0 +1,12 @@
+require 'fog'
+
+module Dopv
+  module Infrastructure
+    class BareMetal < Base
+      def bootstrap_node
+        ::Dopv::log.info("Node #{nodename}: Bootstrapping node instance (noop).")
+        true
+      end
+    end
+  end
+end

data/lib/dopv/infrastructure/providers/base.rb

@@ -0,0 +1,422 @@
+require 'forwardable'
+require 'uri'
+require 'fog'
+require 'open3'
+require 'dop_common/utils'
+
+module Dopv
+  module Infrastructure
+    class ProviderError < StandardError
+      def exit_code
+        4
+      end
+    end
+
+    class Base
+      extend Forwardable
+      include DopCommon::Utils
+
+      MAX_RETRIES = 5
+
+      attr_reader :data_disks_db
+      def_delegators :@plan, :nodename, :fqdn, :hostname, :domainname, :dns
+      def_delegators :@plan, :timezone
+      def_delegators :@plan, :full_clone?, :image, :cores, :memory, :storage, :flavor
+      def_delegators :@plan, :infrastructure, :infrastructure_properties
+      def_delegator :@plan, :interfaces, :interfaces_config
+      def_delegator :@plan, :data_disks, :volumes_config
+      def_delegators :@plan, :credentials
+      def_delegators :@plan, :hooks
+
+      def self.bootstrap_node(plan, state_store)
+        new(plan, state_store).bootstrap_node
+      end
+
+      def self.destroy_node(plan, state_store, destroy_data_volumes=false)
+        new(plan, state_store).destroy_node(destroy_data_volumes)
+      end
+
+      def self.refresh_node(plan, state_store)
+        new(plan, state_store).refresh_node
+      end
+
+      def initialize(plan, state_store)
+        @compute_provider = nil
+        @plan = plan
+        @state_store = state_store
+        @data_disks_db = Dopv::PersistentDisk::DB.new(state_store, nodename)
+      end
+
+      def bootstrap_node
+        begin
+          unless get_node_instance
+            execute_hook(:pre_create_vm, true)
+            node_instance = create_node_instance
+            add_node_nics(node_instance)
+            add_node_data_volumes(node_instance)
+            add_node_affinities(node_instance)
+            start_node_instance(node_instance)
+            execute_hook(:post_create_vm, true)
+            record_node_instance(node_instance)
+          else
+            ::Dopv::log.warn("Node #{nodename}: Already exists.")
+            # TODO: Ask Marcel what would be a purpose/use case of this
+            execute_hook(:pre_create_vm, false)
+            execute_hook(:post_create_vm, false)
+          end
+        rescue Exception => e
+          ::Dopv::log.error("Node #{nodename}: #{e}")
+          destroy_node_instance(node_instance)
+          raise ProviderError, "Node #{nodename}: #{e}."
+        end
+      end
+
+      def destroy_node(destroy_data_volumes=false)
+        node_instance = get_node_instance
+        if node_instance
+          execute_hook(:pre_destroy_vm, true)
+          destroy_node_instance(node_instance, destroy_data_volumes)
+          execute_hook(:post_destroy_vm, true)
+          erase_node_instance(node_instance)
+        else
+          # TODO: Ask Marcel what would be a purpose/use case of this
+          execute_hook(:pre_destroy_vm, false)
+          execute_hook(:post_destroy_vm, false)
+        end
+      end
+
+      def refresh_node
+        node_instance = get_node_instance
+        if node_instance
+          record_node_instance(node_instance)
+        else
+          erase_node_instance(node_instance)
+        end
+      end
+
+      private
+
+      def provider_username
+        @provider_username ||= infrastructure.credentials.username
+      end
+
+      def provider_password
+        @provider_passowrd ||= infrastructure.credentials.password
+      end
+
+      def provider_url
+        @provider_url ||= infrastructure.endpoint.to_s
+      end
+
+      def provider_host
+        @provider_host ||= infrastructure.endpoint.host
+      end
+
+      def provider_port
+        @provider_port ||= infrastructure.endpoint.port
+      end
+
+      def provider_scheme
+        @provider_scheme ||= infrastructure.endpoint.scheme
+      end
+
+      def provider_ssl?
+        provider_scheme == 'https'
+      end
+
+      def root_password
+        cred = credentials.find { |c| c.type == :username_password && c.username == 'root' } if
+          @root_password.nil?
+        @root_password ||= cred.nil? ? nil : cred.password
+      end
+
+      def root_ssh_pubkeys
+        cred = credentials.find_all { |c| c.type == :ssh_key && c.username == 'root' } if
+          @root_ssh_pubkeys.nil?
+        @root_ssh_pubkey ||= cred.empty? ? [] : cred.collect { |k| k.public_key }.uniq
+      end
+
+      def administrator_password
+        cred = credentials.find { |c| c.type == :username_password && c.username == 'Administrator' } if
+          @administrator_password.nil?
+        @administrator_password ||= cred.nil? ? nil : cred.password
+      end
+
+      def administrator_fullname
+        'Administrator'
+      end
+
+      def keep_ha?
+        @keep_ha ||= infrastructure_properties.keep_ha?
+      end
+
+      def compute_provider
+        Dopv::log.info("Node #{nodename}: Creating compute provider.") unless @compute_provider
+        @compute_provider ||= @compute_connection_opts ? ::Fog::Compute.new(@compute_connection_opts) : nil
+      end
+
+      def datacenter(filters={})
+        @datacenter ||= compute_provider.datacenters(filters).find do |d|
+          if d.is_a?(Hash) && d.has_key?(:name)
+            d[:name] == infrastructure_properties.datacenter
+          elsif d.respond_to?(:name)
+            d.name == infrastructure_properties.datacenter
+          else
+            raise ProviderError, "Unsupported datacenter class #{d.class}"
+          end
+        end
+        raise ProviderError, "No such data center #{infrastructure_properties.datacenter}" unless @datacenter
+        @datacenter
+      end
+
+      def cluster(filters={})
+        @cluster ||= compute_provider.clusters(filters).find { |c| c.name == infrastructure_properties.cluster }
+        raise ProviderError, "No such cluster #{infrastructure_properties.cluster}" unless @cluster
+        @cluster
+      end
+
+      def template(filters={})
+        raise ProviderError, "No template defined" unless image
+        @template ||= if compute_provider.respond_to?(:templates)
+                         compute_provider.templates.all(filters).find { |t| t.name == image }
+                       elsif compute_provider.respond_to?(:images)
+                         compute_provider.images.all(filters).find { |t| t.name == image }
+                       else
+                         raise ProviderError, "The provider does not to have template/image collection"
+                       end
+        raise ProviderError, "No such template #{image}" unless @template
+        @template
+      end
+
+      def get_node_instance(filters = {})
+        retries = 0
+        compute_provider.servers.all(filters).find { |n| n.name == nodename }
+      rescue => e
+        errmsg = "Node #{nodename}: An error occured while searching for a node: #{e}."
+        retries += 1
+        if retries <= MAX_RETRIES
+          Dopv.log.warn("#{errmsg} Retrying (##{retries}).")
+          sleep 1
+          retry
+        else
+          raise ProviderError, "#{errmsg}. Bailing out"
+        end
+      end
+
+      def node_instance_ready?(node_instance)
+        node_instance.ready?
+      end
+
+      def node_instance_stopped?(node_instance)
+        node_instance.stopped?
+      end
+
+      def wait_for_task_completion(node_instance)
+      end
+
+      def create_node_instance
+        Dopv::log.info("Node #{nodename}: Creating node instance.")
+        node_instance = compute_provider.servers.create(@node_creation_opts)
+        wait_for_task_completion(node_instance)
+        node_instance
+      end
+
+      def destroy_node_instance(node_instance, destroy_data_volumes=false)
+        if node_instance
+          stop_node_instance(node_instance)
+
+          volumes = data_disks_db.volumes
+          volumes.each do |v|
+            if destroy_data_volumes
+              ::Dopv::log.warn("Node #{nodename} Destroying data volume #{v.name}.")
+              begin
+                destroy_node_volume(node_instance, v)
+              rescue
+                ::Dopv::log.error("Could not destroy data volume #{v.name}. Please fix manually.")
+              end
+              erase_node_data_volume(v)
+            else
+              ::Dopv::log.debug("Node #{nodename} Detaching data volume #{v.name}.")
+              begin
+                detach_node_volume(node_instance, v)
+              rescue
+                ::Dopv::log.warn("Could not detach data volume #{v.name}.")
+              end
+            end
+          end
+
+          ::Dopv::log.warn("Node #{nodename}: Destroying node.")
+          node_instance.destroy rescue nil
+        end
+      end
+
+      def reload_node_instance(node_instance)
+        node_instance.reload
+      end
+
+      def customize_node_instance(node_instance)
+      end
+
+      def start_node_instance(node_instance)
+        stop_node_instance(node_instance)
+        ::Dopv::log.info("Node #{nodename}: Starting node.")
+        customize_node_instance(node_instance)
+      end
+
+      def stop_node_instance(node_instance)
+        reload_node_instance(node_instance)
+        unless node_instance_stopped?(node_instance)
+          ::Dopv::log.info("Node #{nodename}: Stopping node.")
+          wait_for_task_completion(node_instance)
+          node_instance.stop
+          reload_node_instance(node_instance)
+        end
+      end
+
+      def add_node_nic(node_instance, attrs)
+        nic = node_instance.interfaces.create(attrs)
+        node_instance.interfaces.reload
+        nic
+      end
+
+      def update_node_nic(node_instance, nic, attrs)
+        nic.save(attrs)
+        node_instance.interfaces.reload
+      end
+
+      def add_node_nics(node_instance)
+      end
+
+      def remove_node_nics(node_instance)
+        Dopv::log.debug("Node #{nodename}: Removing (possible) network interfaces defined by template.")
+        if block_given?
+          node_instance.interfaces.each { |nic| yield(node_instance, nic) }
+        else
+          node_instance.interfaces.each(&:destroy) rescue nil
+        end
+        node_instance.interfaces.reload
+      end
+
+      def add_node_affinity(node_instance, affinity)
+      end
+
+      def remove_node_affinity(node_instance, affinity)
+      end
+
+      def add_node_volume(node_instance, config)
+        node_instance.volumes.create(config)
+      end
+
+      def update_node_volume(node_instance, volume, attrs)
+        node_instance.update_volume(attrs.merge(:id => volume.id))
+        wait_for_task_completion(node_instance)
+        node_instance.volumes.reload
+        volume
+      end
+
+      def destroy_node_volume(node_instance, volume)
+      end
+
+      def attach_node_volume(node_instance, volume)
+      end
+
+      def detach_node_volume(node_instance, volume)
+      end
+
+      def add_node_data_volumes(node_instance)
+        ::Dopv::log.info("Node #{nodename}: Adding data volumes.")
+
+        ::Dopv::log.debug("Node #{nodename}: Loading data volumes DB.")
+        data_volumes = data_disks_db.volumes
+
+        # Check if persistent disks DB is consistent
+        ::Dopv::log.debug("Node #{nodename}: Checking data volumes DB integrity.")
+        data_volumes.each do |dv|
+          # Disk exists in state DB but not in plan
+          unless volumes_config.find { |cv| dv.name == cv.name }
+            err_msg = "Inconsistent data volumes DB: Volume #{dv.name} exists in DB but not in plan"
+            raise ProviderError, err_msg
+          end
+        end
+        volumes_config.each do |cv|
+          # Disk exists in a plan but it is not recorded in the state DB for a
+          # given node
+          if !data_volumes.empty? && !data_volumes.find { |dv| cv.name == dv.name }
+            ::Dopv::log.warn("Node #{nodename}: Data volume #{cv.name} exists in plan but not in DB.")
+          end
+        end
+
+        # Attach all persistent disks
+        data_volumes.each do |dv|
+          ::Dopv::log.debug("Node #{nodename}: Attaching data volume #{dv.name} [#{dv.id}].")
+          begin
+            attach_node_volume(node_instance, dv)
+          rescue Exception => e
+            err_msg = "An error occured while attaching data volume #{dv.name}: #{e}"
+            raise ProviderError, err_msg
+          end
+        end
+
+        # Create those disks that do not exist in peristent disks DB and
+        # record them into DB
+        volumes_config.each do |cv|
+          unless data_disks_db.volumes.find { |v| v.name == cv.name }
+            ::Dopv::log.debug("Node #{nodename}: Creating disk #{cv.name} [#{cv.size.g} G].")
+            volume = add_node_volume(node_instance, cv)
+            record_node_data_volume(volume) unless volume.nil?
+          end
+        end
+      end
+
+      def record_node_data_volume(volume)
+        ::Dopv::log.debug("Node #{nodename}: Recording data volume #{volume[:name]} into data volumes DB.")
+        data_disks_db << volume.merge(:node => nodename)
+      end
+
+      def erase_node_data_volume(volume)
+        ::Dopv::log.debug("Node #{nodename}: Erasing data volume #{volume.name} from data volumes DB.")
+        data_disks_db.delete(volume)
+      end
+
+      def add_node_affinity(node_instance, name)
+      end
+
+      def add_node_affinities(node_instance)
+        infrastructure_properties.affinity_groups.each { |a| add_node_affinity(node_instance, a) }
+      end
+
+      def execute_hook(hook_name, state_changed = false)
+        has_changes = state_changed ? 1 : 0
+        hooks.send(hook_name).each do |prog|
+          prog_name = File.basename(prog)
+          ::Dopv::log.info("Node #{nodename}: Executing #{hook_name}[#{prog_name}].")
+          o, e, s = Open3.capture3(sanitize_env, "#{prog} #{nodename} #{has_changes}", :unsetenv_others => true)
+          ::Dopv::log.debug("Node #{nodename}: #{hook_name}[#{prog_name}] standard output:\n#{o.chomp}")
+          ::Dopv::log.warn("Node #{nodename}: #{hook_name}[#{prog_name}] non-zero exit status #{s.exitstatus}") unless s.success?
+          ::Dopv::log.debug("Node #{nodename}: #{hook_name}[#{prog_name}] standard error:\n#{e.chomp}") unless e.chomp.empty?
+        end
+      end
+
+      def record_node_instance(node_instance)
+        @state_store.transaction do
+          @state_store[:nodes] ||= {}
+          @state_store[:nodes][nodename] ||= {}
+          @state_store[:nodes][nodename][:ip_addresses] = get_node_ip_addresses(node_instance)
+        end
+      end
+
+      def erase_node_instance(node_instance)
+        @state_store.transaction do
+          if @state_store[:nodes]
+            @state_store[:nodes].delete(nodename)
+          end
+        end
+      end
+
+      def get_node_ip_addresses(node_instance)
+        []
+      end
+    end
+  end
+end
+