vagrant-xapi 0.0.1 → 1.0.0

data/lib/vagrant-xapi/api/http_transfer.rb

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "uri"
+
+require_relative "errors"
+
+module VagrantPlugins
+  module Xapi
+    module Api
+      # Streams uploads and downloads to XAPI HTTP handlers (/import,
+      # /import_raw_vdi, /export, /export_raw_vdi) with net/http.
+      #
+      # The session travels as a cookie, never in the URL, and no external
+      # process (curl) is involved, so no secret ever shows up in `ps`.
+      #
+      # XAPI answers these handlers with a 302 to another pool host when the
+      # receiving host cannot reach the SR. A PUT cannot follow that after
+      # its body is streamed, so callers pass the right host up front
+      # (`host:` is a host ref; nil means the master). GETs also follow
+      # redirects.
+      #
+      # Never send `Expect: 100-continue`: XAPI does not answer it with
+      # "100 Continue", and Net::HTTP would then wait forever for a body it
+      # never sent.
+      class HttpTransfer
+        MAX_REDIRECTS = 3
+
+        # IO wrapper that reports how many bytes have been read.
+        class ProgressIO
+          def initialize(io, total, progress)
+            @io = io
+            @total = total.to_f
+            @progress = progress
+            @done = 0
+            @last = -1
+          end
+
+          def read(length = nil, outbuf = nil)
+            data = @io.read(length, outbuf)
+            if data
+              @done += data.bytesize
+              report
+            end
+            data
+          end
+
+          def size
+            @total.to_i
+          end
+
+          private
+
+          def report
+            return unless @progress && @total.positive?
+
+            pct = (@done / @total * 100).floor
+            return if pct == @last
+
+            @last = pct
+            @progress.call(@done, @total.to_i)
+          end
+        end
+
+        def initialize(client)
+          @client = client
+        end
+
+        # PUT a local file to an XAPI handler. When task_label is given, a task
+        # is created, passed as task_id, and waited for; its Result is returned.
+        def upload(path, query, file, host: nil, progress: nil, task_label: nil, cancel: nil)
+          with_session_retry do
+            upload_once(path, query, file, host: host, progress: progress, task_label: task_label, cancel: cancel)
+          end
+        end
+
+        # Sends an empty PUT and returns only the status code, without waiting
+        # for the handler to finish. Used to ask whether a handler accepts a
+        # parameter (e.g. format=qcow2) before streaming a large file.
+        # @return [Integer, nil] nil when there was no answer in time
+        def probe(path, query, host: nil, timeout: 15)
+          status = probe_once(path, query, host: host, timeout: timeout)
+          return status unless status == 401
+
+          # An expired session, not an answer about the parameter.
+          @client.renew_session(@cookie)
+          probe_once(path, query, host: host, timeout: timeout)
+        end
+
+        # GET from an XAPI handler into a local file, following redirects to
+        # other pool hosts. When task_label is given, a task is created,
+        # passed as task_id and waited for once the body is in: the end of the
+        # stream alone does not say that the export on the host succeeded.
+        def download(path, query, dest, host: nil, progress: nil, cancel: nil, task_label: nil)
+          with_session_retry do
+            download_once(path, query, dest, host: host, progress: progress, cancel: cancel, task_label: task_label)
+          end
+        end
+
+        private
+
+        # XAPI's HTTP handlers answer an expired or evicted session (e.g. the
+        # oldest of a user with too many) with 401. Log in again once and
+        # start the transfer over, as Client#call does on SESSION_INVALID.
+        def with_session_retry
+          renewed = false
+          begin
+            yield
+          rescue HttpError => e
+            raise unless e.status == 401 && !renewed
+
+            renewed = true
+            @client.renew_session(@cookie)
+            retry
+          end
+        end
+
+        # The session cookie for a request, remembered so that a 401 renews
+        # exactly that session.
+        def cookie
+          @cookie = @client.session_cookie
+        end
+
+        def upload_once(path, query, file, host:, progress:, task_label:, cancel:)
+          waiting = false
+          task_ref = task_label ? @client.create_task(task_label) : nil
+          query = query.merge("task_id" => task_ref) if task_ref
+          size = File.size(file)
+          transport = @client.transport_for(host)
+
+          File.open(file, "rb") do |io|
+            req = Net::HTTP::Put.new(build_path(path, query))
+            req["Cookie"] = cookie
+            req["Content-Length"] = size.to_s
+            req["Content-Type"] = "application/octet-stream"
+            req["User-Agent"] = Client::ORIGINATOR
+            req.body_stream = ProgressIO.new(io, size, progress)
+            response = perform(transport, req, path, cancel: cancel, task_ref: task_ref)
+            check!(response, path)
+          end
+
+          return nil unless task_ref
+
+          waiting = true
+          @client.wait_task(task_ref, label: task_label, cancel: cancel) # destroys the task
+        rescue Exception # rubocop:disable Lint/RescueException
+          abandon_task(task_ref) if task_ref && !waiting
+          raise
+        end
+
+        def probe_once(path, query, host:, timeout:)
+          transport = @client.transport_for(host)
+          http = transport.new_connection(read_timeout: timeout)
+          begin
+            req = Net::HTTP::Put.new(build_path(path, query))
+            req["Cookie"] = cookie
+            req["User-Agent"] = Client::ORIGINATOR
+            req["Connection"] = "close"
+            req.body = ""
+            # The block runs once the status and headers are in; returning
+            # from it skips reading the body, which XAPI may never close.
+            http.request(req) { |response| return response.code.to_i } # rubocop:disable Lint/NonLocalExitFromIterator
+          ensure
+            begin
+              http.finish if http.started?
+            rescue StandardError
+              nil
+            end
+          end
+        rescue TimeoutError, *Transport::TIMEOUT_ERRORS
+          # No answer in time: the caller converts instead (and asks again
+          # next time). A refused or unreachable host is a real error below.
+          nil
+        rescue OpenSSL::SSL::SSLError => e
+          raise TlsError.new(transport.host, e.message)
+        rescue *Transport::NETWORK_ERRORS => e
+          raise ConnectionError.new(transport.host, e.message)
+        end
+
+        def download_once(path, query, dest, host:, progress:, cancel:, task_label:)
+          waiting = false
+          task_ref = task_label ? @client.create_task(task_label) : nil
+          query = query.merge("task_id" => task_ref) if task_ref
+          transport = @client.transport_for(host)
+          target = build_path(path, query)
+          MAX_REDIRECTS.times do
+            location = get_into(transport, target, path, dest, progress, cancel)
+            unless location
+              if task_ref
+                waiting = true
+                @client.wait_task(task_ref, label: task_label, cancel: cancel) # destroys the task
+              end
+              return dest
+            end
+
+            uri = URI.parse(location)
+            transport = redirect_transport(uri)
+            target = uri.request_uri
+          end
+          raise HttpError.new(302, path, "too many redirects")
+        rescue Exception # rubocop:disable Lint/RescueException
+          abandon_task(task_ref) if task_ref && !waiting
+          raise
+        end
+
+        # @return [String, nil] redirect location, or nil when done
+        def get_into(transport, target, path, dest, progress, cancel)
+          http = transport.new_connection(read_timeout: @client.task_timeout)
+          begin
+            req = Net::HTTP::Get.new(target)
+            req["Cookie"] = cookie
+            req["User-Agent"] = Client::ORIGINATOR
+            http.request(req) do |response|
+              return response["Location"] if response.is_a?(Net::HTTPRedirection) && response["Location"] # rubocop:disable Lint/NonLocalExitFromIterator
+
+              check!(response, path)
+              total = response["Content-Length"].to_i
+              done = 0
+              File.open(dest, "wb") do |f|
+                response.read_body do |chunk|
+                  raise TaskCancelled, path if cancel&.call
+
+                  f.write(chunk)
+                  done += chunk.bytesize
+                  progress&.call(done, total)
+                end
+              end
+              # Net::HTTP ignores an early end of the stream by default.
+              if response["Content-Length"] && done != total
+                raise ProtocolError, "the download of #{path} ended after #{done} of #{total} bytes"
+              end
+            end
+            nil
+          ensure
+            begin
+              http.finish if http.started?
+            rescue StandardError
+              nil
+            end
+          end
+        rescue OpenSSL::SSL::SSLError => e
+          raise TlsError.new(transport.host, e.message)
+        rescue *Transport::NETWORK_ERRORS => e
+          raise ConnectionError.new(transport.host, e.message)
+        end
+
+        # A redirect points at another pool host by address; find its ref so
+        # its certificate can be pinned. hostname, not host: an IPv6 address
+        # comes without its brackets, as host.get_address returns it.
+        def redirect_transport(uri)
+          host_ref = @client.call("host.get_all").find do |ref|
+            @client.call("host.get_address", ref) == uri.hostname
+          end
+          raise HttpError.new(302, uri.path, "redirect to unknown host #{uri.host}") unless host_ref
+
+          @client.transport_for(host_ref)
+        end
+
+        def perform(transport, req, path, cancel:, task_ref:)
+          http = transport.new_connection(read_timeout: @client.task_timeout)
+          watcher = start_cancel_watcher(http, cancel, task_ref)
+          begin
+            http.request(req)
+          ensure
+            watcher&.kill
+            begin
+              http.finish if http.started?
+            rescue StandardError
+              nil
+            end
+          end
+        rescue OpenSSL::SSL::SSLError => e
+          raise TaskCancelled, path if cancel&.call
+
+          raise TlsError.new(transport.host, e.message)
+        rescue *Transport::NETWORK_ERRORS => e
+          raise TaskCancelled, path if cancel&.call
+
+          raise ConnectionError.new(transport.host, e.message)
+        end
+
+        # On Ctrl-C, cancel the task and close the socket so the PUT stops.
+        def start_cancel_watcher(http, cancel, task_ref)
+          return nil unless cancel
+
+          Thread.new do
+            sleep 0.5 until cancel.call
+            cancel_task(task_ref)
+            begin
+              http.finish if http.started?
+            rescue StandardError
+              nil
+            end
+          end
+        end
+
+        def cancel_task(task_ref)
+          return unless task_ref
+
+          @client.call("task.cancel", task_ref)
+        rescue Error
+          nil
+        end
+
+        # A task we created but will not wait for: cancel and remove it.
+        def abandon_task(task_ref)
+          cancel_task(task_ref)
+          @client.call("task.destroy", task_ref)
+        rescue Error
+          nil
+        end
+
+        def check!(response, path)
+          return if response.is_a?(Net::HTTPSuccess)
+
+          body = begin
+            response.body
+          rescue StandardError
+            nil
+          end
+          raise HttpError.new(response.code, path, body)
+        end
+
+        def build_path(path, query)
+          q = query.compact.map { |k, v| "#{URI.encode_www_form_component(k)}=#{URI.encode_www_form_component(v)}" }
+          q.empty? ? path : "#{path}?#{q.join("&")}"
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-xapi/api/redactor.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "set"
+
+module VagrantPlugins
+  module Xapi
+    module Api
+      # Keeps track of secrets (passwords, session ids) and removes them from
+      # any string before it is logged or shown. Process-wide, thread-safe.
+      module Redactor
+        MASK = "[REDACTED]"
+        @secrets = Set.new
+        @mutex = Mutex.new
+
+        class << self
+          def register(secret)
+            return if secret.nil?
+
+            s = secret.to_s
+            return if s.empty?
+
+            @mutex.synchronize { @secrets << s }
+          end
+
+          def unregister(secret)
+            @mutex.synchronize { @secrets.delete(secret.to_s) }
+          end
+
+          def scrub(value)
+            case value
+            when String
+              out = value.dup
+              secrets.each { |s| out = out.gsub(s, MASK) }
+              # session_id never belongs in a URL or a log line, nor do
+              # credentials in a URL (e.g. an xva_url).
+              out = out.gsub(/(session_id=)[^&\s"';]+/, "\\1#{MASK}")
+              out.gsub(%r{(\w+://)[^/\s:@]+:[^/\s@]+@}, "\\1#{MASK}@")
+            when Array
+              value.map { |v| scrub(v) }
+            when Hash
+              value.each_with_object({}) { |(k, v), h| h[k] = scrub(v) }
+            else
+              value
+            end
+          end
+
+          def secrets
+            @mutex.synchronize { @secrets.to_a.sort_by { |s| -s.length } }
+          end
+
+          # For tests.
+          def reset!
+            @mutex.synchronize { @secrets.clear }
+          end
+        end
+      end
+
+      # Minimal logger wrapper: everything passes through the Redactor first.
+      # Wraps any object with debug/info/warn/error (e.g. a Log4r::Logger);
+      # defaults to a no-op logger so the client works without Vagrant.
+      class SafeLogger
+        def initialize(inner = nil)
+          @inner = inner
+        end
+
+        %i[debug info warn error].each do |level|
+          define_method(level) do |msg = nil, &blk|
+            return unless @inner
+
+            msg = blk.call if msg.nil? && blk
+            @inner.public_send(level, Redactor.scrub(msg.to_s))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-xapi/api/session_registry.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "digest"
+
+require_relative "client"
+
+module VagrantPlugins
+  module Xapi
+    module Api
+      # One logged-in Client per process per pool and user, shared between
+      # threads (Vagrant runs machines in parallel). All sessions are logged
+      # out when the process exits.
+      module SessionRegistry
+        @clients = {}
+        @mutex = Mutex.new
+        @at_exit_installed = false
+
+        class << self
+          # @param opts [Hash] Client.new options
+          def fetch(**opts)
+            # A digest of the password, so machines with other credentials for
+            # the same host and user do not share a session.
+            key = [opts[:host], opts[:port], opts[:username], Digest::SHA256.hexdigest(opts[:password].to_s),
+                   opts[:tls].to_a.sort.to_s, opts[:api_timeout], opts[:task_timeout]].join("|")
+            client = @mutex.synchronize do
+              install_at_exit
+              @clients[key] ||= Client.new(**opts)
+            end
+            client.login
+            client
+          rescue Error
+            # Do not cache a client whose login failed (wrong password etc.).
+            @mutex.synchronize { @clients.delete(key) if @clients[key] && !@clients[key].logged_in? }
+            raise
+          end
+
+          def logout_all
+            clients = @mutex.synchronize { @clients.values.tap { @clients.clear } }
+            clients.each(&:logout)
+          end
+
+          # For tests.
+          def reset!
+            @mutex.synchronize { @clients.clear }
+          end
+
+          private
+
+          def install_at_exit
+            return if @at_exit_installed
+
+            @at_exit_installed = true
+            at_exit { logout_all }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-xapi/api/task.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module VagrantPlugins
+  module Xapi
+    module Api
+      # Waits for an XAPI task using event.from, reports progress, supports
+      # cancellation (task.cancel) and destroys the task afterwards.
+      class Task
+        EVENT_WAIT = 5.0 # seconds per event.from call
+        # How long the pool may be unreachable while a task runs. Giving up
+        # destroys the task record, but XAPI goes on with the work (e.g. a
+        # VM.copy whose result nobody would then mark or clean up), so a
+        # dropped connection or a read timeout is retried for a while first.
+        CONNECTION_GRACE = 60.0
+        RETRY_PAUSE = 2.0
+
+        # Result of a finished task. XAPI returns results as an XML-RPC-ish
+        # string, e.g. "<value>OpaqueRef:...</value>".
+        class Result
+          attr_reader :raw
+
+          def initialize(raw)
+            @raw = raw.to_s
+          end
+
+          # All OpaqueRefs in the result (VM.import returns several).
+          def refs
+            @raw.scan(/OpaqueRef:[0-9A-Za-z-]+/)
+          end
+
+          def ref
+            refs.first
+          end
+
+          # The first <value> content, or the raw string if it is not wrapped.
+          def value
+            m = @raw.match(%r{<value>(?:<string>)?([^<]*)(?:</string>)?</value>})
+            m ? m[1] : @raw
+          end
+        end
+
+        attr_reader :ref, :label
+
+        def initialize(client, ref, label:, logger: nil)
+          @client = client
+          @ref = ref
+          @label = label
+          @logger = logger || SafeLogger.new
+        end
+
+        # @param progress [#call] called with a Float 0.0..1.0
+        # @param cancel [#call] returns true when the user asked to cancel
+        def wait(timeout:, progress: nil, cancel: nil)
+          deadline = monotonic + timeout
+          token = ""
+          cancelled = false
+          use_events = true
+          last_progress = nil
+          @lost_since = nil
+
+          loop do
+            if cancel&.call && !cancelled
+              cancelled = true
+              @logger.info("cancelling task #{@label}")
+              begin
+                @client.call("task.cancel", @ref)
+              rescue Error => e
+                @logger.debug("task.cancel: #{e.message}")
+              end
+            end
+
+            record = through_connection_loss(deadline) { @client.call("task.get_record", @ref) }
+            next if record.nil?
+
+            status = record["status"].to_s.downcase
+            pct = record["progress"].to_f
+            if progress && pct != last_progress
+              last_progress = pct
+              progress.call(pct)
+            end
+
+            case status
+            when "success"
+              return Result.new(record["result"])
+            when "failure"
+              info = Array(record["error_info"])
+              raise TaskCancelled, @label if cancelled && info.first.to_s =~ /CANCELLED/
+
+              raise XapiError.new(info.first || "TASK_FAILED", info.drop(1), method_name: @label)
+            when "cancelled", "cancelling"
+              raise TaskCancelled, @label if status == "cancelled"
+            end
+
+            if monotonic > deadline
+              begin
+                @client.call("task.cancel", @ref)
+              rescue Error
+                nil
+              end
+              raise TaskTimeout.new(@label, timeout)
+            end
+
+            if use_events
+              begin
+                res = through_connection_loss(deadline) do
+                  @client.call_with_timeout("event.from", EVENT_WAIT + @client.api_timeout,
+                                            ["task/#{@ref}"], token, EVENT_WAIT)
+                end
+                token = res["token"] if res.is_a?(Hash) && res["token"]
+              rescue XapiError => e
+                # EVENTS_LOST or RBAC: fall back to polling.
+                @logger.debug("event.from failed (#{e.code}); polling task instead")
+                use_events = false
+              end
+            else
+              sleep 1
+            end
+          end
+        ensure
+          destroy
+        end
+
+        private
+
+        # The block's result, or nil after a connection error within the
+        # grace period (the caller then polls again). Raises the error once
+        # the pool has been unreachable for CONNECTION_GRACE seconds or the
+        # task's deadline has passed.
+        def through_connection_loss(deadline)
+          result = yield
+          @lost_since = nil
+          result
+        rescue ConnectionError => e
+          @lost_since ||= monotonic
+          raise if monotonic - @lost_since >= CONNECTION_GRACE || monotonic > deadline
+
+          @logger.info("waiting for task #{@label}: #{e.message}; trying again")
+          sleep RETRY_PAUSE
+          nil
+        end
+
+        def destroy
+          @client.call("task.destroy", @ref)
+        rescue Error
+          nil
+        end
+
+        def monotonic
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end