opennebula 5.12.13 → 5.13.80.pre

data/lib/distributed_firewall.rb

@@ -0,0 +1,280 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2021, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+module NSXDriver
+
+    ONE_LOCATION = ENV['ONE_LOCATION'] unless defined?(ONE_LOCATION)
+
+    if !ONE_LOCATION
+        RUBY_LIB_LOCATION = '/usr/lib/one/ruby' \
+            unless defined?(RUBY_LIB_LOCATION)
+        GEMS_LOCATION     = '/usr/share/one/gems' \
+            unless defined?(GEMS_LOCATION)
+    else
+        RUBY_LIB_LOCATION = ONE_LOCATION + '/lib/ruby' \
+            unless defined?(RUBY_LIB_LOCATION)
+        GEMS_LOCATION     = ONE_LOCATION + '/share/gems' \
+            unless defined?(GEMS_LOCATION)
+    end
+
+    if File.directory?(GEMS_LOCATION)
+        real_gems_path = File.realpath(GEMS_LOCATION)
+        if !defined?(Gem) || Gem.path != [real_gems_path]
+            $LOAD_PATH.reject! {|l| l =~ /vendor_ruby/ }
+            require 'rubygems'
+            Gem.use_paths(real_gems_path)
+        end
+    end
+
+    $LOAD_PATH << RUBY_LIB_LOCATION
+
+    require 'nsx_rule'
+
+    # Class DistributedFirewall
+    # Naming convention for rules names:
+    # "<sg_id>-<sg_name>-<vm_id>-<vm_deploy_id>-<nic_id>"
+    # <sg_id> : Security Group ID
+    # <sg_name> : Security Group name
+    # <vm_id> : Virtual Machine ID
+    # <vm_deploy_id> : Vmware virtual machine reference ( vm-XXX )
+    # <nic_id> : NIC ID in the virtual machine
+    class DistributedFirewall < NSXComponent
+
+        include NSXDriver::NSXRule
+
+        # ATTRIBUTES
+        attr_reader :one_section_name
+
+        def self.new_child(nsx_client)
+            case nsx_client
+            when NSXTClient
+                NSXTdfw.new(nsx_client)
+            when NSXVClient
+                NSXVdfw.new(nsx_client)
+            else
+                error_msg = "Unknown object type: #{nsx_client}"
+                error = NSXError::UnknownObject.new(error_msg)
+                raise error
+            end
+        end
+
+        # Sections
+        # Creates OpenNebula section if not exists and returns
+        # its section_id. Returns its section_id if OpenNebula
+        # section already exists
+        def init_section; end
+
+        # Get all sections
+        def sections; end
+
+        # Get section by id
+        def section_by_id(section_id); end
+
+        # Get section by name
+        def section_by_name(section_name); end
+
+        # Create new section
+        def create_section(section_name); end
+
+        # Delete section
+        def delete_section(section_id); end
+
+        # Rules
+        # Get all rules
+        def rules; end
+
+        # Get rule by id
+        def rule_by_id; end
+
+        # Get rule by name
+        def rules_by_name; end
+
+        # Create new rule
+        def create_rule; end
+
+        # Update rule
+        def update_rule; end
+
+        # Delete rule
+        def delete_rule; end
+
+        def extract_nic_data(nic, nsx_client, vm)
+            # Network attributes
+            nic_id = nic.xpath('NIC_ID').text
+            nic_name = nil
+            nic_lp = nil
+            network_id = nic.xpath('NETWORK_ID').text
+            network_name = nic.xpath('NETWORK').text
+            network_vcref = nic.xpath('VCENTER_NET_REF').text
+            network_pgtype = nic.xpath('VCENTER_PORTGROUP_TYPE').text
+            network_mac = nic.xpath('MAC').text
+
+            # Virtual Machine devices
+            vm_devices = vm.item.config.hardware.device
+            vm_devices.each do |device|
+                next unless VCenterDriver::Network.nic?(device)
+
+                next if device.macAddress != network_mac
+
+                device_label = device.deviceInfo.label
+                nic_name = "#{vm.item.name}-#{nic_id}-#{device_label}"
+
+                case network_pgtype
+                when NSXConstants::NSXT_LS_TYPE
+                    lpid = device.externalId
+                    nic_lp = LogicalPort.new_child(nsx_client, lpid)
+                    raise "Logical port id: #{lpid} not found" unless nic_lp
+                when NSXConstants::NSXV_LS_TYPE
+                    # lpid is vm instanceUuid.sufix
+                    # sufix is device number but removing first number
+                    suffix = device.key.to_s[1..-1]
+                    lpid = "#{vm.item.config.instanceUuid}.#{suffix}"
+                    nic_lp = lpid
+                else
+                    error_msg = "Network type is: #{network_pgtype} \
+                                    and should be \
+                                    #{NSXConstants::NSXT_LS_TYPE} \
+                                    or #{NSXConstants::NSXV_LS_TYPE}"
+                    error = NSXError::UnknownObject.new(error_msg)
+                    raise error
+                end
+                break
+            end
+
+            {
+                :id => nic_id,
+                :name => nic_name,
+                :network_name => network_name,
+                :network_id => network_id,
+                :network_vcref => network_vcref,
+                :lp => nic_lp
+            }
+        end
+
+        # Create OpenNebula fw rules for an instance (given a template)
+        def create_rules(deploy_id, template, only_attached)
+            template_xml = Nokogiri::XML(template)
+
+            # OpenNebula host
+            host_name = template_xml
+                        .xpath('//HISTORY_RECORDS/HISTORY[last()]/HOSTNAME')
+                        .text
+            one_host = VCenterDriver::VIHelper
+                       .find_by_name(OpenNebula::HostPool, host_name)
+            rc = one_host.info
+            if OpenNebula.is_error?(rc)
+                err_msg = rc.message
+                raise err_msg
+            end
+            host_id = one_host['ID']
+
+            # OpenNebula VM
+            one_vm = VCenterDriver::VIHelper
+                     .one_item(OpenNebula::VirtualMachine, deploy_id)
+
+            vm_data = {
+                :id => template_xml.xpath('/VM/ID').text,
+                :deploy_id => deploy_id
+            }
+
+            # vCenter VirtualMachine
+            vi_client = VCenterDriver::VIClient.new_from_host(host_id)
+            vm = VCenterDriver::VirtualMachine
+                 .new_one(vi_client, deploy_id, one_vm)
+
+            # Search NSX Nics
+            # First try to search only new attached NSX Nics
+            nsx_nics = NSXDriver::LogicalSwitch.nsx_nics(template_xml,
+                                                         only_attached)
+            # If there is no NSX Nics
+            return if nsx_nics.empty?
+
+            # Create rules for each NSX Nic
+            nsx_nics.each do |nic|
+                # Extract NIC data
+                nic_data = extract_nic_data(nic, @nsx_client, vm)
+                # Get all Security Groups belonging to each NIC.
+                sec_groups = nic.xpath('SECURITY_GROUPS').text.split(',')
+                sec_groups.each do |sec_group|
+                    sg_rules_array = []
+                    # Get all rules belonging to this Security Group.
+                    xp = "//SECURITY_GROUP_RULE[SECURITY_GROUP_ID=#{sec_group}]"
+                    sg_rules = template_xml.xpath(xp)
+                    sg_rules.each do |sg_rule|
+                        # Create rules spec
+                        rule_data = extract_rule_data(sg_rule)
+                        rule_spec = rule_spec(rule_data,
+                                              vm_data,
+                                              nic_data,
+                                              @nsx_client)
+                        sg_rules_array.push(rule_spec)
+                    end
+                    # Create NSX rules
+                    sg_rules_array.each do |sg_spec|
+                        create_rule(sg_spec)
+                    end
+                end
+            end
+        end
+
+        def clear_all_rules(template)
+            template_xml = Nokogiri::XML(template)
+            vm_id = template_xml.xpath('/VM/ID').text
+            vm_deploy_id = template_xml.xpath('/VM/DEPLOY_ID').text
+            regex = "-#{vm_id}-#{vm_deploy_id}-"
+            rules = rules_by_regex(regex, @one_section_id)
+            rules.each do |rule|
+                delete_rule(rule['id'], @one_section_id) if rule
+            end
+        end
+
+        # Remove OpenNebula created fw rules for an instance (given a template)
+        def clear_rules(template, only_detached)
+            template_xml = Nokogiri::XML(template)
+            # OpenNebula Instance IDs
+            vm_id = template_xml.xpath('/VM/ID').text
+            vm_deploy_id = template_xml.xpath('/VM/DEPLOY_ID').text
+
+            # First try to search only new attached NSX Nics
+            nsx_nics = NSXDriver::LogicalSwitch.nsx_nics(template_xml,
+                                                         only_detached)
+
+            # If there is no NSX Nics
+            return if nsx_nics.empty?
+
+            nsx_nics.each do |nic|
+                nic_id = nic.xpath('NIC_ID').text
+                # network_id = nic.xpath('NETWORK_ID').text
+                sec_groups = nic.xpath('SECURITY_GROUPS').text.split(',')
+                sec_groups.each do |sec_group|
+                    # Get Security Group ID and NAME
+                    xp = "//SECURITY_GROUP_RULE[SECURITY_GROUP_ID=#{sec_group}]"
+                    sg_id = template_xml.xpath(xp)[0]
+                                        .xpath('SECURITY_GROUP_ID').text
+                    sg_name = template_xml.xpath(xp)[0]
+                                          .xpath('SECURITY_GROUP_NAME').text
+                    rule_name =  "#{sg_id}-#{sg_name}-#{vm_id}"
+                    rule_name << "-#{vm_deploy_id}-#{nic_id}"
+                    rules = rules_by_name(rule_name, @one_section_id)
+                    rules.each do |rule|
+                        delete_rule(rule['id'], @one_section_id) if rule
+                    end
+                end
+            end
+        end
+
+    end
+
+end

data/lib/file_helper.rb

@@ -0,0 +1,370 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2021, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+require 'fileutils'
+require 'tempfile'
+
+##############################################################################
+# Module VCenterDriver
+##############################################################################
+module VCenterDriver
+
+    ##########################################################################
+    # Class FileHelper
+    ##########################################################################
+    class FileHelper
+
+        def self.sanitize(text)
+            # Bad as defined by wikipedia:
+            # https://en.wikipedia.org/wiki/Filename in
+            # Reserved_characters_and_words
+            # Also have to escape the backslash
+            bad_chars = ['/', '\\', '?', '%', '*', ':',
+                         '|', '"', '<', '>', '.', ' ']
+            bad_chars.each do |bad_char|
+                text.gsub!(bad_char, '_')
+            end
+            text
+        end
+
+        def self.get_img_name(
+            disk,
+            vm_id,
+            _vm_name,
+            instantiate_as_persistent = false
+        )
+            if disk['PERSISTENT'] == 'YES' || disk['TYPE'] == 'CDROM'
+                disk['SOURCE']
+            else
+                disk_id = disk['DISK_ID']
+                if disk['SOURCE']
+                    if instantiate_as_persistent &&
+                       disk['OPENNEBULA_MANAGED'] &&
+                       disk['OPENNEBULA_MANAGED'].upcase == 'NO'
+                        disk['SOURCE'] # Treat this disk as if was persistent
+                    else
+                        image_name = disk['SOURCE'].split('.').first
+                        "#{image_name}-#{vm_id}-#{disk_id}.vmdk"
+                    end
+                else
+                    ds_volatile_dir =
+                        disk['VCENTER_DS_VOLATILE_DIR'] || 'one-volatile'
+                    "#{ds_volatile_dir}/#{vm_id}/one-#{vm_id}-#{disk_id}.vmdk"
+                end
+            end
+        end
+
+        # REMOVE: no need to change...
+        def self.get_img_name_from_path(path, vm_id, disk_id)
+            # NOTE: This will probably fail if the basename contains '.'
+            "#{path.split('.').first}-#{vm_id}-#{disk_id}.vmdk"
+        end
+
+        def self.remote_or_needs_unpack?(file)
+            !remote?(file).nil? || needs_unpack?(file)
+        end
+
+        def self.remote?(file)
+            file.match(%r{^https?://}) || file.match(%r{^s3?://})
+        end
+
+        def self.vmdk?(file)
+            type = `file #{file}`
+
+            type.include? 'VMware'
+        end
+
+        def self.iso?(file)
+            type = `file #{file}`
+
+            type.include? 'ISO'
+        end
+
+        def self.get_type(file)
+            type = `file -b --mime-type #{file}`
+            if $?.exitstatus != 0 # rubocop:disable Style/SpecialGlobalVars
+                STDERR.puts "Can not read file #{file}"
+                exit(-1)
+            end
+            type.strip
+        end
+
+        def self.needs_unpack?(file_path)
+            type = get_type(file_path)
+            type.gsub!(%r{^application/(x-)?}, '')
+            %w[bzip2 gzip tar].include?(type)
+        end
+
+        def self.vcenter_file_info(file_path)
+            if File.directory?(file_path)
+                files = Dir["#{file_path}/*.vmdk"]
+                found = false
+                count = 0
+                last  = nil
+
+                files.each do |f|
+                    if get_type(f).strip == 'text/plain'
+                        file_path = f
+                        found = true
+                        break
+                    else
+                        count += 1
+                        last = f
+                    end
+                end
+
+                if !found
+                    if count == 1
+                        file_path = last
+                        found = true
+                    else
+                        STDERR.puts 'Could not find vmdk'
+                        exit(-1)
+                    end
+                end
+            end
+
+            case get_type(file_path).strip
+            when 'application/octet-stream'
+                {
+                    :type   => :standalone,
+                    :file   => file_path,
+                    :dir    => File.dirname(file_path)
+                }
+            when 'application/x-iso9660-image'
+                {
+                    :type   => :standalone,
+                    :file   => file_path,
+                    :dir    => File.dirname(file_path),
+                    :extension => '.iso'
+                }
+            when 'text/plain'
+                info = {
+                    :type   => :flat,
+                    :file   => file_path,
+                    :dir    => File.dirname(file_path)
+                }
+
+                files_list = []
+                descriptor = File.read(file_path).split("\n")
+                flat_files = descriptor.select {|l| l.start_with?('RW') }
+
+                flat_files.each do |f|
+                    files_list <<
+                        info[:dir] +
+                        '/' +
+                        f
+                        .split(' ')[3]
+                        .chomp
+                        .chomp('"')
+                        .reverse
+                        .chomp('"')
+                        .reverse
+                end
+
+                info[:flat_files] = files_list
+
+                info
+            else
+                STDERR.puts 'Unrecognized file type'
+                exit(-1)
+            end
+        end
+
+        def self.escape_path(path)
+            path.gsub(' ', '%20')
+        end
+
+        def self.unescape_path(path)
+            path.gsub('%20', ' ')
+        end
+
+        # Recursively downloads vmdk related files and returns filenames
+        def self.get_all_filenames_in_descriptor(descriptor_url, ds)
+            descriptor_filename = File.basename descriptor_url.path
+            # Build array of files to download
+            files_to_download = [descriptor_filename]
+            image_source = descriptor_url.host + descriptor_url.path
+            descriptor_content = ds.get_text_file image_source
+            flat_files = descriptor_content.select {|l| l.start_with?('RW') }
+            flat_files.each do |file|
+                # Get the filename from lines of type
+                # RW 2048000 VMFS "filename-flat.vdmdk"
+                file_to_download = file.split(' ')[3][1..-2]
+                files_to_download << file_to_download
+                image_path =
+                    File
+                    .dirname(
+                        descriptor_url.host+descriptor_url.path
+                    )
+                next unless ds.descriptor?(
+                    image_path + '/' + file_to_download
+                )
+
+                files_to_download <<
+                    download_all_filenames_in_descriptor(
+                        image_path + '/' + file_to_download
+                    )
+            end
+
+            files_to_download
+        end
+
+        def self.download_vmdks(files_to_download, url_prefix, temp_folder, ds)
+            # Download files
+            url_prefix += '/'
+
+            VCenterDriver::VIClient.in_silence do
+                files_to_download.each do |file|
+                    ds.download_file(url_prefix + file, temp_folder + file)
+                end
+            end
+        end
+
+        # Receives a VMDK descriptor or file, downloads all
+        # related files, creates a tar.gz and dumps it in stdout
+        def self.dump_vmdk_tar_gz(vcenter_url, ds)
+            image_source = vcenter_url.host + vcenter_url.path
+            if ds.descriptor?(image_source)
+                files_to_download =
+                    get_all_filenames_in_descriptor(
+                        vcenter_url,
+                        ds
+                    )
+
+                descriptor_name = File.basename vcenter_url.path
+                temp_folder = VAR_LOCATION + '/vcenter/' + descriptor_name + '/'
+                unless File.directory?(temp_folder)
+                    FileUtils
+                        .mkdir_p(
+                            temp_folder
+                        )
+                end
+
+                image_path = File.dirname(vcenter_url.host+vcenter_url.path)
+                download_vmdks(files_to_download, image_path, temp_folder, ds)
+
+                # Create tar.gz
+                rs = system(
+                    "cd #{temp_folder} \&& tar czf #{descriptor_name}.tar.gz \
+                    #{files_to_download.join(' ')} > /dev/null 2>&1"
+                )
+                unless rs
+                    FileUtils.rm_rf temp_folder
+                    raise "Error creating tar file for #{descriptor_name}"
+                end
+
+                # Cat file to stdout
+                rs = system("cat #{temp_folder + descriptor_name}.tar.gz")
+                unless rs
+                    FileUtils.rm_rf temp_folder
+                    raise "Error reading tar for #{descriptor_name}"
+                end
+
+                # Delete tar.gz
+                rs = system(
+                    "cd #{temp_folder} \
+                    && rm #{descriptor_name}.tar.gz #{
+                        files_to_download
+                        .join(' ')}"
+                )
+                unless rs
+                    FileUtils.rm_rf temp_folder
+                    raise "Error removing tar for #{descriptor_name}"
+                end
+            else
+                # Setting "." as the source will read from the stdin
+                VCenterDriver::VIClient.in_stderr_silence do
+                    descriptor_name = File.basename vcenter_url.path
+                    file_to_download = [vcenter_url.path]
+                    temp_folder =
+                        VAR_LOCATION + '/vcenter/' + descriptor_name + '/'
+
+                    unless File
+                           .directory?(
+                               temp_folder + File
+                               .dirname(
+                                   vcenter_url
+                                   .path
+                               ) + '/'
+                           )
+                        FileUtils
+                            .mkdir_p(temp_folder + File
+                            .dirname(
+                                vcenter_url
+                                .path
+                            ) + '/')
+                    end
+
+                    download_vmdks(
+                        file_to_download,
+                        vcenter_url.host,
+                        temp_folder,
+                        ds
+                    )
+
+                    temp_folder += File.dirname(vcenter_url.path)
+
+                    # Create tar.gz
+                    rs = system(
+                        "cd #{temp_folder} && tar czf #{descriptor_name}.tar.gz\
+                         #{descriptor_name} > /dev/null 2>&1"
+                    )
+                    unless rs
+                        (
+                            FileUtils
+                            .rm_rf(
+                                temp_folder
+                            )
+                            raise "Error creating tar \
+                            file for #{descriptor_name}")
+                    end
+
+                    # Cat file to stdout
+                    rs = system(
+                        "cat #{temp_folder + '/' + descriptor_name}.tar.gz"
+                    )
+                    unless rs
+                        (
+                            FileUtils
+                            .rm_rf(
+                                temp_folder
+                            )
+                            raise "Error reading tar for #{descriptor_name}")
+                    end # rubocop:disable Style/Semicolon
+
+                    # Delete tar.gz
+                    rs = system(
+                        "cd #{temp_folder} \
+                        && rm #{descriptor_name}.tar.gz #{descriptor_name}"
+                    )
+                    unless rs
+                        (
+                            FileUtils
+                            .rm_rf(
+                                temp_folder
+                            )
+                            raise "Error \ removing tar for #{descriptor_name}")
+                    end # rubocop:disable Style/Semicolon
+                end
+            end
+        end
+
+    end
+    # class FileHelper
+
+end
+# module VCenterDriver