vagrant-xapi 0.0.1 → 1.0.0

data/lib/vagrant-xapi/services/hosts.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+require_relative "base"
+
+module VagrantPlugins
+  module Xapi
+    module Services
+      # The hosts of the pool as the master reports them: address, role and
+      # TLS certificate. The certificates come from host.get_server_certificate,
+      # so through the verified connection to the master.
+      class Hosts
+        HostInfo = Struct.new(:uuid, :name, :address, :role, :enabled, :fingerprint, :certificate,
+                              keyword_init: true)
+
+        def initialize(ctx)
+          @client = ctx.client
+        end
+
+        # @return [Array<HostInfo>] the master first, then by name
+        def list
+          master = @client.master_ref
+          @client.call("host.get_all_records").map do |ref, rec|
+            pem = certificate(ref)
+            HostInfo.new(uuid: rec["uuid"], name: rec["name_label"], address: rec["address"],
+                         role: ref == master ? "master" : "member", enabled: rec["enabled"] != false,
+                         fingerprint: pem && fingerprint(pem), certificate: pem)
+          end.sort_by { |h| [h.role == "master" ? 0 : 1, h.name.to_s] }
+        end
+
+        # SHA-256 of the DER certificate, as tls_fingerprint takes it.
+        def self.fingerprint(pem)
+          OpenSSL::Digest::SHA256.hexdigest(OpenSSL::X509::Certificate.new(pem).to_der).scan(/../).join(":").upcase
+        end
+
+        private
+
+        def fingerprint(pem)
+          self.class.fingerprint(pem)
+        rescue OpenSSL::X509::CertificateError
+          nil
+        end
+
+        # nil when the host has none or the role may not read it.
+        def certificate(ref)
+          pem = @client.call("host.get_server_certificate", ref).to_s
+          pem.empty? ? nil : pem
+        rescue Api::XapiError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-xapi/services/image_upload.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+
+require_relative "base"
+require_relative "disk_image"
+require_relative "lookup"
+
+module VagrantPlugins
+  module Xapi
+    module Services
+      # Uploads a disk image into an existing VDI through /import_raw_vdi, on
+      # the host that has the VDI's SR attached.
+      #
+      # qcow2: whether XAPI can import it is asked once per session with an
+      # empty probe PUT on a small scratch VDI. XAPI rejects an unknown format
+      # before it reads the body; a supported format gets a 200. If qcow2 is
+      # not supported (or the probe gives no answer), or the image has
+      # compressed clusters or features XAPI's import does not take, the image
+      # is converted to VHD on the local machine with the converter (qemu-img).
+      class ImageUpload
+        include Helpers
+
+        PROBE_SIZE = 1024 * 1024
+
+        # @param converter [#call, nil] converter.call(src, dst): qcow2 -> VHD
+        def initialize(ctx, converter: nil)
+          @ctx = ctx
+          @client = ctx.client
+          @lookup = Lookup.new(ctx)
+          @converter = converter
+        end
+
+        # @param format [String] vhd, qcow2 or raw
+        # @return [String] the VDI ref; a new one when the VDI had to be
+        #   recreated for the VHD fallback
+        def upload(vdi, path, format, label:)
+          sr = @client.call("VDI.get_SR", vdi)
+          host = @lookup.transfer_host_for_sr(sr)
+          tmpdir = nil
+          if format == "qcow2"
+            obstacle = DiskImage.qcow2_import_obstacle(path)
+            if obstacle || !qcow2_supported?(sr, host)
+              tmpdir = Dir.mktmpdir("vagrant-xapi-convert-")
+              path = convert(path, tmpdir, reason: obstacle)
+              format = "vhd"
+            end
+          end
+          begin
+            send_image(vdi, path, format, host, label)
+          rescue Api::HttpError, Api::ConnectionError, Api::XapiError => e
+            # The probe said yes, but this host refused qcow2 after all (e.g.
+            # another version during a rolling upgrade), or its import could
+            # not read this image: convert and retry.
+            raise unless format == "qcow2" && qcow2_refused?(e)
+
+            @ctx.logger&.info("qcow2 upload refused (#{e.message}); converting to VHD")
+            # Only an HTTP refusal of the format says something about the host.
+            @client.set_feature(feature_key(host), false) if e.is_a?(Api::HttpError)
+            tmpdir ||= Dir.mktmpdir("vagrant-xapi-convert-")
+            vhd = convert(path, tmpdir, reason: e.is_a?(Api::XapiError) ? :refused : nil, detail: e.message)
+            vdi = recreate(vdi)
+            begin
+              send_image(vdi, vhd, "vhd", host, label)
+            rescue Exception # rubocop:disable Lint/RescueException
+              # The caller only knows the old (destroyed) ref; clean up here.
+              destroy_vdi(vdi)
+              raise
+            end
+          end
+          vdi
+        ensure
+          FileUtils.rm_rf(tmpdir) if tmpdir
+        end
+
+        # Asked once per session and transfer host: pool members may run
+        # different versions.
+        def qcow2_supported?(sr, host)
+          known = @client.features[feature_key(host)]
+          return known unless known.nil?
+
+          scratch = create_vdi(sr, "vagrant-xapi qcow2 probe", PROBE_SIZE, description: "temporary")
+          begin
+            status = @client.http.probe("/import_raw_vdi", { "vdi" => scratch, "format" => "qcow2" }, host: host)
+          ensure
+            destroy_vdi(scratch)
+          end
+          supported = status ? status.between?(200, 299) : false
+          @ctx.logger&.info("qcow2 import probe: HTTP #{status.inspect}; supported=#{supported}")
+          # Only remember a real answer; a timeout is asked again next time.
+          @client.set_feature(feature_key(host), supported) unless status.nil?
+          supported
+        end
+
+        private
+
+        def feature_key(host)
+          :"qcow2_import@#{host || "master"}"
+        end
+
+        # XAPI refuses a format it cannot import with a 4xx before it reads
+        # the body, and may close the connection while the body is still
+        # being sent (broken pipe, reset). Its qcow2 reader failing on this
+        # image is a VDI_IO_ERROR from qcow-stream-tool. Anything else (a
+        # timeout, an unreachable host, 401/403 for the session, 5xx) is no
+        # refusal and is raised as it is.
+        def qcow2_refused?(error)
+          case error
+          when Api::XapiError
+            error.code == "VDI_IO_ERROR" && error.params.join(" ").match?(/qcow/i)
+          when Api::HttpError
+            error.status.between?(400, 499) && ![401, 403].include?(error.status)
+          when Api::TimeoutError
+            false
+          when Api::ConnectionError
+            [Errno::EPIPE, Errno::ECONNRESET].any? { |klass| error.cause.is_a?(klass) }
+          else
+            false
+          end
+        end
+
+        def send_image(vdi, path, format, host, label)
+          @client.http.upload("/import_raw_vdi", { "vdi" => vdi, "format" => format }, path,
+                              host: host,
+                              progress: @ctx.progress_proc(:uploading, file: File.basename(path)),
+                              task_label: label, cancel: @ctx.cancel)
+          @ctx.ui&.clear_progress
+        end
+
+        # A refused upload may have written part of the image; start the VHD
+        # import on a fresh VDI with the same settings.
+        def recreate(vdi)
+          rec = @client.call("VDI.get_record", vdi)
+          destroy_vdi(vdi)
+          create_vdi(rec["SR"], rec["name_label"], rec["virtual_size"],
+                     description: rec["name_description"].to_s, type: rec["type"] || "user",
+                     other_config: rec["other_config"] || {}, tags: rec["tags"] || [])
+        end
+
+        # @param reason [Symbol, nil] why the image is converted: nil (the host
+        #   cannot import qcow2), :compressed, :features or :refused
+        def convert(path, tmpdir, reason: nil, detail: nil)
+          unless @converter
+            raise Error.new(:qemu_img_missing) unless reason
+
+            raise Error.new(:"qemu_img_missing_#{reason}", file: File.basename(path), detail: detail.to_s)
+          end
+
+          dst = File.join(tmpdir, "#{File.basename(path, ".*")}.vhd")
+          @ctx.ui&.info(:converting_image, file: File.basename(path))
+          @converter.call(path, dst)
+          dst
+        end
+
+        # The probe may leave XAPI busy with the VDI for a moment.
+        def destroy_vdi(vdi)
+          10.times do
+            @client.call("VDI.destroy", vdi)
+            return
+          rescue Api::XapiError => e
+            return if %w[HANDLE_INVALID UUID_INVALID].include?(e.code)
+            raise unless %w[VDI_IN_USE OTHER_OPERATION_IN_PROGRESS].include?(e.code)
+
+            sleep 1
+          end
+          @ctx.logger&.warn("could not remove the qcow2 probe VDI")
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-xapi/services/inventory.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "base"
+require_relative "template_cache"
+
+module VagrantPlugins
+  module Xapi
+    module Services
+      # Overview and cleanup of what the plugin left in a pool: box templates
+      # (shared by all users) and orphaned VMs.
+      class Inventory
+        include Helpers
+
+        TemplateInfo = Struct.new(:ref, :uuid, :name, :box, :version, :provider, :sr_uuid, :sr_name, :size,
+                                  :clones, :clone_users, keyword_init: true)
+        VmInfo = Struct.new(:ref, :uuid, :name, :machine, :project, :user, :client, :power_state,
+                            keyword_init: true)
+        # One object carrying a vagrant-xapi marking.
+        #   kind: :vm, :snapshot, :template, :vdi, :network, :lock
+        Marked = Struct.new(:kind, :uuid, :name, :detail, keyword_init: true)
+
+        KIND_ORDER = %i[vm snapshot template vdi network lock].freeze
+
+        def initialize(ctx)
+          @ctx = ctx
+          @client = ctx.client
+        end
+
+        # @return [Array<TemplateInfo>]
+        def templates
+          vms = marked_vms
+          TemplateCache.new(@ctx, digest_cache: nil).all_templates.map do |ref, rec|
+            oc = rec["other_config"] || {}
+            clones = vms.select { |v| v.dig(:oc, Marking::TEMPLATE_UUID) == rec["uuid"] }
+            TemplateInfo.new(
+              ref: ref, uuid: rec["uuid"], name: rec["name_label"],
+              box: oc[Marking::BOX], version: oc[Marking::BOX_VERSION], provider: oc[Marking::BOX_PROVIDER],
+              sr_uuid: oc[Marking::SR],
+              sr_name: sr_name(oc[Marking::SR]),
+              size: disk_vdis(ref).sum { |vdi| ignore_missing { @client.call("VDI.get_virtual_size", vdi).to_i } || 0 },
+              clones: clones.size,
+              clone_users: clones.map { |v| v.dig(:oc, Marking::USER) }.compact.uniq.sort
+            )
+          end.sort_by { |t| [t.box.to_s, version_key(t.version)] }
+        end
+
+        # Templates to remove so that per box only the newest `keep` versions
+        # remain (per SR).
+        def prune_candidates(keep)
+          # By SR UUID: in a pool every host's local SR is called "Local storage".
+          templates.group_by { |t| [t.box, t.provider, t.sr_uuid] }.flat_map do |_key, list|
+            sorted = list.sort_by { |t| version_key(t.version) }.reverse
+            sorted.drop(keep.to_i)
+          end
+        end
+
+        def find_template(name_or_uuid)
+          list = templates
+          hit = list.select { |t| [t.uuid, t.name, "#{t.box} #{t.version}"].include?(name_or_uuid.to_s) }
+          raise Error.new(:template_not_found, name: name_or_uuid.to_s) if hit.empty?
+          raise Error.new(:ambiguous_name, kind: "template", name: name_or_uuid.to_s, count: hit.size) if hit.size > 1
+
+          hit.first
+        end
+
+        # Removing a template is safe while clones exist: the SM layer keeps
+        # the shared base VHD.
+        def remove_template(info)
+          rec = @client.call("VM.get_record", info.ref)
+          raise Error.new(:template_not_marked, name: info.name) unless (rec["other_config"] || {}).key?(Marking::BOX_DIGEST)
+
+          vdis = disk_vdis(info.ref)
+          @client.call("VM.destroy", info.ref)
+          vdis.each { |vdi| ignore_missing { @client.call("VDI.destroy", vdi) } }
+        end
+
+        # VMs of this Vagrant installation: same client_id, or (for VMs
+        # without one) the same user and local hostname.
+        # @return [Array<VmInfo>]
+        def my_vms(user: Marking.user, client: Marking.client_host, client_id: nil)
+          marked_vms.select { |v| mine?(v[:oc], user, client, client_id) }
+                    .map do |v|
+            VmInfo.new(ref: v[:ref], uuid: v[:uuid], name: v[:name], machine: v.dig(:oc, Marking::MACHINE),
+                       project: v.dig(:oc, Marking::PROJECT), user: v.dig(:oc, Marking::USER),
+                       client: v.dig(:oc, Marking::CLIENT), power_state: v[:power_state])
+          end
+        end
+
+        # Everything that carries the plugin's marking, read through XAPI.
+        # With filters, an object is listed when it belongs to any of them:
+        #   projects: VMs, snapshots and internal networks of these project
+        #             directories
+        #   vm_uuids: these VMs, their snapshots and the VDIs they own
+        #   boxes:    templates of these box names
+        #   mine:     client_id of this installation: its VMs (or, for VMs
+        #             without client id, those of this user and hostname),
+        #             their snapshots and disks, and the internal networks of
+        #             their projects; shared templates are not "mine"
+        # @return [Array<Marked>]
+        def marked(projects: [], vm_uuids: [], boxes: [], mine: nil)
+          projects = projects.map(&:to_s)
+          vm_uuids = vm_uuids.map(&:to_s)
+          boxes = boxes.map(&:to_s)
+          filtered = !(projects.empty? && vm_uuids.empty? && boxes.empty? && mine.nil?)
+          result = []
+          records = @client.call("VM.get_all_records")
+
+          my_vms = []
+          my_projects = []
+          if mine
+            records.each_value do |rec|
+              oc = rec["other_config"] || {}
+              next if rec["is_a_template"] || !Marking.marked?(oc)
+              next unless mine?(oc, Marking.user, Marking.client_host, mine)
+
+              my_vms << rec["uuid"]
+              my_projects << oc[Marking::PROJECT]
+            end
+          end
+          vm_uuids |= my_vms
+          network_projects = projects | my_projects
+
+          records.each_value do |rec|
+            oc = rec["other_config"] || {}
+            if rec["is_a_snapshot"] && oc.key?(Marking::SNAPSHOT_OF)
+              next if filtered && !vm_uuids.include?(oc[Marking::SNAPSHOT_OF]) && !projects.include?(oc[Marking::PROJECT])
+
+              result << Marked.new(kind: :snapshot, uuid: rec["uuid"], name: rec["name_label"],
+                                   detail: "of VM #{oc[Marking::SNAPSHOT_OF]}")
+            elsif rec["is_a_template"] && oc.key?(Marking::BOX_DIGEST)
+              next if filtered && !boxes.include?(oc[Marking::BOX])
+
+              result << Marked.new(kind: :template, uuid: rec["uuid"], name: rec["name_label"],
+                                   detail: "box #{oc[Marking::BOX]} #{oc[Marking::BOX_VERSION]}")
+            elsif !rec["is_a_template"] && Marking.marked?(oc)
+              next if filtered && !projects.include?(oc[Marking::PROJECT]) && !vm_uuids.include?(rec["uuid"])
+
+              result << Marked.new(kind: :vm, uuid: rec["uuid"], name: rec["name_label"],
+                                   detail: "#{oc[Marking::MACHINE]} in #{oc[Marking::PROJECT]}")
+            end
+          end
+
+          @client.call("VDI.get_all_records").each_value do |rec|
+            oc = rec["other_config"] || {}
+            owner = oc[Marking::OWNER_VM]
+            if owner
+              next if filtered && !vm_uuids.include?(owner)
+
+              result << Marked.new(kind: :vdi, uuid: rec["uuid"], name: rec["name_label"], detail: "of VM #{owner}")
+            elsif oc.key?(Marking::OF_TEMPLATE) && !filtered
+              result << Marked.new(kind: :vdi, uuid: rec["uuid"], name: rec["name_label"],
+                                   detail: "of template #{oc[Marking::OF_TEMPLATE]}")
+            end
+          end
+
+          @client.call("network.get_all_records").each_value do |rec|
+            oc = rec["other_config"] || {}
+            next unless oc.key?(Marking::NETWORK)
+            next if filtered && !network_projects.include?(oc[Marking::PROJECT])
+
+            result << Marked.new(kind: :network, uuid: rec["uuid"], name: rec["name_label"],
+                                 detail: "project #{oc[Marking::PROJECT]}")
+          end
+
+          unless filtered
+            pool = @client.call("pool.get_all").first
+            @client.call("pool.get_other_config", pool).each do |key, value|
+              next unless key.start_with?("#{Marking::PREFIX}lock.")
+
+              owner = begin
+                JSON.parse(value)["owner"]
+              rescue JSON::ParserError
+                value
+              end
+              result << Marked.new(kind: :lock, uuid: "", name: key.delete_prefix("#{Marking::PREFIX}lock."),
+                                   detail: "held by #{owner}")
+            end
+          end
+
+          result.sort_by { |m| [KIND_ORDER.index(m.kind), m.name.to_s] }
+        end
+
+        private
+
+        def mine?(oc, user, client, client_id)
+          return oc[Marking::CLIENT_ID] == client_id.to_s if client_id && oc.key?(Marking::CLIENT_ID)
+
+          oc[Marking::USER] == user && oc[Marking::CLIENT] == client
+        end
+
+        def marked_vms
+          @marked_vms ||= begin
+            recs = begin
+              @client.call("VM.get_all_records_where", 'field "is_a_template"="false"')
+            rescue Api::XapiError
+              @client.call("VM.get_all_records")
+            end
+            recs.filter_map do |ref, rec|
+              next if rec["is_a_template"] || rec["is_a_snapshot"] || rec["is_control_domain"]
+
+              oc = rec["other_config"] || {}
+              next unless Marking.marked?(oc)
+
+              { ref: ref, uuid: rec["uuid"], name: rec["name_label"], oc: oc, power_state: rec["power_state"] }
+            end
+          end
+        end
+
+        def sr_name(uuid)
+          return nil if uuid.to_s.empty?
+
+          ignore_missing { @client.call("SR.get_name_label", @client.call("SR.get_by_uuid", uuid)) } || uuid
+        end
+
+        def version_key(version)
+          Gem::Version.new(version.to_s)
+        rescue ArgumentError
+          Gem::Version.new("0")
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-xapi/services/ip_detector.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+
+require_relative "base"
+
+module VagrantPlugins
+  module Xapi
+    module Services
+      # Finds the guest's IP from the configured sources, in order:
+      #   :guest_metrics  VM_guest_metrics.networks "<dev>/ipv4/0", "<dev>/ip"
+      #   :static         ip: of the first VIF (set via cloud-init network-config)
+      #   :himn           network.assigned_ips of the HIMN VIF
+      class IpDetector
+        include Helpers
+
+        SOURCES = %i[guest_metrics static himn].freeze
+
+        # @param static_ip [String, nil] ip: of VIF device 0
+        def initialize(ctx, sources:, static_ip: nil, himn_enabled: false)
+          @ctx = ctx
+          @client = ctx.client
+          @sources = Array(sources).map(&:to_sym)
+          @static_ip = static_ip
+          @himn_enabled = himn_enabled
+        end
+
+        # One look, no waiting.
+        # @return [Array(String, Symbol), nil] [ip, source]
+        def detect(vm)
+          @sources.each do |source|
+            ip = case source
+                 when :guest_metrics then from_guest_metrics(vm)
+                 when :static then @static_ip.to_s.empty? ? nil : @static_ip.to_s
+                 when :himn then @himn_enabled ? from_himn(vm) : nil
+                 end
+            return [ip, source] if ip
+          end
+          nil
+        end
+
+        # Waits until an IP shows up or the timeout passes.
+        def wait(vm, timeout:, interval: 2)
+          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+          loop do
+            found = detect(vm)
+            return found if found
+            raise Api::TaskCancelled, "waiting for IP" if @ctx.cancelled?
+
+            power = @client.call("VM.get_power_state", vm)
+            raise Error.new(:vm_stopped_while_waiting, state: power) unless power == "Running"
+
+            if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+              raise Error.new(:ip_timeout, timeout: timeout, sources: @sources.join(", "))
+            end
+
+            sleep interval
+          end
+        end
+
+        def from_guest_metrics(vm, device: 0)
+          metrics = @client.call("VM.get_guest_metrics", vm)
+          return nil if null_ref?(metrics)
+
+          nets = ignore_missing { @client.call("VM_guest_metrics.get_networks", metrics) } || {}
+          candidate = nets["#{device}/ipv4/0"] || nets["#{device}/ip"]
+          usable?(candidate) ? candidate : nil
+        end
+
+        def from_himn(vm)
+          himn_vif = @client.call("VM.get_VIFs", vm).find do |vif|
+            net = @client.call("VIF.get_network", vif)
+            @client.call("network.get_other_config", net)["is_host_internal_management_network"].to_s == "true"
+          end
+          return nil unless himn_vif
+
+          net = @client.call("VIF.get_network", himn_vif)
+          ip = (@client.call("network.get_assigned_ips", net) || {})[himn_vif]
+          # HIMN leases are link-local (169.254.0.0/16) by design.
+          usable?(ip, allow_link_local: true) ? ip : nil
+        end
+
+        # Only a bare IPv4 or IPv6 literal is accepted. The value is written by
+        # the guest agent, so it is untrusted: it ends up in an SSH
+        # ProxyCommand (%h) and in NFS exports.
+        def self.valid_ip?(ip, allow_link_local: false)
+          s = ip.to_s
+          return false unless s.match?(/\A[0-9A-Fa-f:.]{2,45}\z/)
+
+          addr = IPAddr.new(s)
+          return false if addr.loopback? || addr.to_s == "0.0.0.0" || addr.to_s == "::"
+          return false if !allow_link_local && addr.link_local?
+
+          true
+        rescue IPAddr::Error
+          false
+        end
+
+        private
+
+        def usable?(ip, allow_link_local: false)
+          self.class.valid_ip?(ip, allow_link_local: allow_link_local)
+        end
+      end
+    end
+  end
+end