vagrant-xapi 0.0.1 → 1.0.0

data/lib/vagrant-xapi/services/differ.rb

@@ -0,0 +1,382 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "set"
+
+require_relative "base"
+require_relative "content_diff"
+require_relative "diff_disks"
+require_relative "file_diff"
+require_relative "guest_files"
+
+module VagrantPlugins
+  module Xapi
+    module Services
+      # `vagrant xapi diff`: which files differ between a snapshot and the
+      # running guest, two snapshots, or two running guests.
+      #
+      # A snapshot is read by attaching throwaway clones of its disks to the
+      # running VM and mounting them read-only in the guest itself: the
+      # snapshot comes from that guest, so its file systems cross no new trust
+      # boundary (dom0 or the local machine would parse a file system the
+      # guest wrote). The current state is the live tree, which also holds
+      # writes that are not on disk yet. File systems are paired by UUID
+      # (a snapshot with the running guest) or by mount point (two guests).
+      class Differ
+        # Paths of --path that may go into a find command.
+        SAFE_REL = %r{\A[A-Za-z0-9._@+=,:~%/-]+\z}.freeze
+
+        # A file system to list: where it belongs in the comparison, and where
+        # the guest can read it.
+        Tree = Struct.new(:mountpoint, :root, keyword_init: true)
+        # targets: the mount points of the running guest, or with two
+        # machines those of both. A mount point nested in a tree, and what
+        # lies below it, is left out of that tree on every side: a snapshot
+        # file system is mounted on its own, so it shows files the running
+        # guest has hidden below another mount, and a live tree would show the
+        # content of a bind mount of the same device. With two machines, a
+        # path that is a mount point on only one of them would otherwise show
+        # as all added or removed on the other.
+        Side = Struct.new(:files, :trees, :label, :targets, keyword_init: true)
+        # contents (with content): path => { diff: } or { skipped: :binary,
+        # :too_large (with size:) or :unreadable }
+        Result = Struct.new(:changes, :mountpoints, :read_errors, :contents, keyword_init: true)
+        # Changes whose content --content shows: of regular files only.
+        CONTENT_KINDS = %i[added removed changed].freeze
+
+        # @param tmpdir [String] local directory for downloaded listings
+        def initialize(ctx, tmpdir:)
+          @ctx = ctx
+          @client = ctx.client
+          @tmpdir = tmpdir
+        end
+
+        # Snapshot `from` against the running guest, or against snapshot `to`.
+        # @param guest see GuestFiles
+        # @param filter [FileDiff::Filter]
+        # @param content [Integer, nil] also show what changed in files up to
+        #   this many bytes (--content)
+        # @return [Result]
+        def snapshot(vm, guest, from:, filter:, to: nil, checksum: false, content: nil)
+          files = GuestFiles.new(guest, @ctx)
+          disks = DiffDisks.new(@ctx)
+          run_id = SecureRandom.hex(8)
+          # Everything under the lock, so that a run never removes what another
+          # run on this VM is using: its disks, mounts and directory.
+          disks.synchronize(vm) do
+            files.prepare!
+            begin
+              files.layout # what an interrupted run left mounted
+              files.remove_stale!
+              disks.cleanup_leftovers(vm)
+              # Only now: a leftover copy with LUKS or RAID would fail the
+              # storage check of every later diff.
+              live = files.layout
+              files.check_storage!
+              attached = []
+              begin
+                targets = files.mount_targets
+                in_snapshot = Set.new
+                a = Side.new(files: files, label: [:snapshot, from], targets: targets,
+                             trees: snapshot_trees(vm, files, disks, from, "a", live, run_id, attached, in_snapshot))
+                b = if to
+                      side = Side.new(files: files, label: [:snapshot, to], targets: targets,
+                                      trees: snapshot_trees(vm, files, disks, to, "b", live, run_id, attached,
+                                                            in_snapshot))
+                      # One only one snapshot has is named by pair; one
+                      # neither has would stay hidden below its mount point.
+                      only_in_live(live, in_snapshot)
+                      side
+                    else
+                      only_in_live(live, in_snapshot)
+                      # Every snapshot file system was paired with a live one,
+                      # so a missing one is a live mount point that failed the
+                      # character check (already reported).
+                      Side.new(files: files, label: [:live, nil], targets: targets,
+                               trees: live_trees(live.select { |fs| a.trees.any? { |t| t.mountpoint == fs.target } }))
+                    end
+                pair(a, b, report: !to.nil?)
+                compare(a, b, filter.with_internal(files.tmp), checksum, content)
+              ensure
+                files.unmount_all
+                files.deactivate_lvm
+                disks.detach(attached)
+              end
+            ensure
+              files.cleanup
+            end
+          end
+        end
+
+        # Two running guests, file systems paired by mount point.
+        # @param vms [Array(Array(Context, String), Array(Context, String))]
+        #   per machine the context of its pool and its VM
+        # @param names [Array(String, String)] machine names, for messages
+        def machines(guest_from, guest_to, vms:, names:, filter:, checksum: false, content: nil)
+          from_files = GuestFiles.new(guest_from, @ctx)
+          to_files = GuestFiles.new(guest_to, @ctx)
+          # Under the diff locks of both VMs, like a snapshot run, because each
+          # removes what interrupted runs left in its guest.
+          synchronize_all(vms) do
+            from_files.prepare!
+            begin
+              to_files.prepare!
+              begin
+                [from_files, to_files].each do |files|
+                  files.layout # what an interrupted run left mounted
+                  files.remove_stale!
+                end
+                from_live = live_trees(from_files.layout)
+                to_live = live_trees(to_files.layout)
+                # Both sides leave out the mount points of both machines.
+                targets = from_files.mount_targets | to_files.mount_targets
+                a = Side.new(files: from_files, trees: from_live, label: [:machine, names[0]], targets: targets)
+                b = Side.new(files: to_files, trees: to_live, label: [:machine, names[1]], targets: targets)
+                pair(a, b)
+                compare(a, b, filter.with_internal(from_files.tmp, to_files.tmp), checksum, content)
+              ensure
+                to_files.cleanup
+              end
+            ensure
+              from_files.cleanup
+            end
+          end
+        end
+
+        private
+
+        # Runs the block under the diff locks of several VMs, taken in the
+        # order of their names (diff.<vm uuid>), so that two runs that need
+        # the same VMs cannot deadlock: one waits for the other, never both
+        # for each other.
+        # @param vms [Array(Context, String)] per VM the context of its pool
+        def synchronize_all(vms, &block)
+          locks = vms.map { |ctx, vm| DiffDisks.new(ctx).lock(vm) }.uniq(&:name).sort_by(&:name)
+          locks.reverse.reduce(block) { |inner, lock| -> { lock.synchronize(&inner) } }.call
+        end
+
+        # @param in_snapshot [Set<String>, nil] gets the UUID of every file
+        #   system on the snapshot's disks, also of those that are skipped
+        def snapshot_trees(vm, files, disks, name, side, live, run_id, attached, in_snapshot = nil)
+          @ctx.ui&.info(:diff_attaching, name: name.to_s)
+          list = disks.attach_snapshot(vm, name, run_id)
+          attached.concat(list)
+          devices = list.map { |disk| files.probe(disk.device, disk.size) }
+          # After all disks are there: an LVM volume group may span several.
+          files.import_lvm(devices, "#{GuestFiles::VG_PREFIX}#{run_id}_#{side}")
+          by_uuid = live.reject { |fs| fs.uuid.empty? }.to_h { |fs| [fs.uuid, fs] }
+          trees = []
+          seen = Set.new
+          listed = Set.new
+          devices.each do |device|
+            files.partitions(device).each do |part|
+              next if part.fs_type == "swap"
+              # An LV with extents on two copies is listed under both disks.
+              next unless listed.add?(part.name)
+
+              in_snapshot&.add(part.uuid)
+
+              fs = by_uuid[part.uuid]
+              unless fs && files.mountable?(part)
+                @ctx.ui&.warn(:diff_fs_skipped, name: name.to_s, uuid: part.uuid, fs_type: part.fs_type)
+                next
+              end
+              # Two disks with one file system UUID (e.g. a snapshot taken while
+              # a diff had its copy attached): neither can be trusted to be the
+              # one the running guest has, so only the first is used.
+              unless seen.add?(part.uuid)
+                @ctx.ui&.warn(:diff_fs_duplicate, name: name.to_s, uuid: part.uuid)
+                next
+              end
+              root = files.mount(part, "#{side}#{trees.size}", like: fs)
+              trees << Tree.new(mountpoint: fs.target, root: root)
+            end
+          end
+          trees
+        end
+
+        # The live file systems; their mount points go into a command, so only
+        # those that pass the character check.
+        def live_trees(list)
+          list.filter_map do |fs|
+            unless fs.safe?
+              @ctx.ui&.warn(:diff_unsafe_mountpoint, target: FileDiff.escape(fs.target))
+              next
+            end
+            Tree.new(mountpoint: fs.target, root: fs.target)
+          end
+        end
+
+        # Live file systems the snapshot does not have at all (e.g. a disk
+        # added since): they are not compared, and their mount points stay
+        # left out of the tree above them, so they are named. One the
+        # snapshot has but could not mount was named already (diff_fs_skipped).
+        def only_in_live(live, in_snapshot)
+          live.each do |fs|
+            next if in_snapshot.include?(fs.uuid)
+
+            @ctx.ui&.warn(:diff_only_in_live, target: FileDiff.escape(fs.target), name: "")
+          end
+        end
+
+        # Only file systems both sides have are compared; with report, the
+        # others are named.
+        def pair(from, to, report: true)
+          common = from.trees.map(&:mountpoint) & to.trees.map(&:mountpoint)
+          [from, to].each do |side|
+            side.trees.each do |t|
+              next if common.include?(t.mountpoint) || !report
+
+              @ctx.ui&.warn(:"diff_only_in_#{side.label[0]}", target: FileDiff.escape(t.mountpoint),
+                                                              name: side.label[1].to_s)
+            end
+            side.trees = side.trees.select { |t| common.include?(t.mountpoint) }.sort_by(&:mountpoint)
+          end
+          raise Error.new(:diff_nothing_to_compare) if common.empty?
+        end
+
+        def compare(from, to, filter, checksum, content)
+          from_entries, from_errors = list(from, filter.paths)
+          to_entries, to_errors = list(to, filter.paths)
+          [from_entries, to_entries].each { |entries| entries.select! { |path, _| filter.include?(path) } }
+
+          candidates = FileDiff.hash_candidates(from_entries, to_entries, checksum: checksum)
+          hashes_from = {}
+          hashes_to = {}
+          unless candidates.empty?
+            @ctx.ui&.info(:diff_hashing, count: candidates.size)
+            hashes_from = hashes(from, from_entries, candidates)
+            hashes_to = hashes(to, to_entries, candidates)
+          end
+          changes = FileDiff.compare(from_entries, to_entries, hashed: candidates.to_set,
+                                                               hashes_from: hashes_from, hashes_to: hashes_to)
+          Result.new(changes: changes, mountpoints: from.trees.map(&:mountpoint), read_errors: from_errors + to_errors,
+                     contents: content && contents(from, to, from_entries, to_entries, changes, content))
+        end
+
+        # What changed in regular files up to limit bytes, while the snapshot
+        # copies are still mounted.
+        def contents(from, to, from_entries, to_entries, changes, limit)
+          result = {}
+          wanted = {}
+          changes.each do |change|
+            next unless CONTENT_KINDS.include?(change.kind)
+
+            a = change.kind == :added ? nil : from_entries[change.path]
+            b = change.kind == :removed ? nil : to_entries[change.path]
+            sides = [a, b].compact
+            next unless sides.all? { |entry| entry.type == "f" }
+
+            size = sides.map(&:size).max
+            if size > limit
+              result[change.path] = { skipped: :too_large, size: size }
+            else
+              wanted[change.path] = [a, b]
+            end
+          end
+          return result if wanted.empty?
+
+          @ctx.ui&.info(:diff_fetching_content, count: wanted.size)
+          old = fetch(from, wanted.transform_values(&:first).compact, limit)
+          new = fetch(to, wanted.transform_values(&:last).compact, limit)
+          wanted.each { |path, (a, b)| result[path] = content(path, a && old[path], b && new[path], a, b) }
+          result
+        end
+
+        # One file's ContentDiff result from what both sides copied.
+        def content(path, old, new, from_entry, to_entry)
+          copied = [old, new]
+          if (from_entry && old.nil?) || (to_entry && new.nil?)
+            { skipped: :unreadable }
+          elsif copied.include?(:budget)
+            { skipped: :budget }
+          elsif (large = copied.grep(GuestFiles::TooLarge).first)
+            { skipped: :too_large, size: large.size }
+          elsif copied.include?(:binary)
+            { skipped: :binary }
+          else
+            ContentDiff.render(path, old, new)
+          end
+        end
+
+        # One copy run per side, all trees together, so the budget counts
+        # for the whole side.
+        # @param entries [Hash{String => FileDiff::Entry}] full path => entry
+        # @return [Hash{String => String, Symbol}] full path => content, or
+        #   :binary / :budget (GuestFiles#contents)
+        def fetch(side, entries, limit)
+          raise Api::TaskCancelled, "diff" if @ctx.cancelled?
+
+          list = entries.to_a
+          guest_paths = list.map { |_path, entry| FileDiff.join(FileDiff.normalize(side.trees[entry.tree].root), entry.rel) }
+          copied = side.files.contents(guest_paths, @tmpdir, limit: limit, budget: ContentDiff.budget(limit))
+          list.each_with_index.to_h { |(path, _entry), i| [path, copied[i]] }.compact
+        end
+
+        def list(side, paths)
+          @ctx.ui&.info(:"diff_listing_#{side.label[0]}", name: side.label[1].to_s)
+          entries = {}
+          errors = 0
+          side.trees.each_with_index do |tree, i|
+            raise Api::TaskCancelled, "diff" if @ctx.cancelled?
+
+            starts = starts_for(tree, side.targets, paths)
+            next if starts&.empty?
+
+            local, failed = side.files.listing(tree.root, @tmpdir, starts: starts)
+            FileDiff.parse(File.binread(local), tree.mountpoint, i, entries, hidden: nested(tree, side.targets))
+            File.delete(local)
+            errors += failed
+          end
+          if errors.positive?
+            @ctx.ui&.warn(:"diff_read_errors_#{side.label[0]}", count: errors, name: side.label[1].to_s)
+          end
+          [entries, errors]
+        end
+
+        # With --path: where find starts in the tree, relative to its root.
+        # nil lists the whole tree, [] nothing of it. A path belongs to the
+        # tree of the deepest mount point above it. A path that cannot go into
+        # a command lists the whole tree; the filter narrows it afterwards.
+        def starts_for(tree, targets, paths)
+          return nil if paths.empty?
+
+          mountpoint = tree.mountpoint
+          starts = []
+          paths.each do |path|
+            return nil if FileDiff.below?(mountpoint, path)
+            next unless FileDiff.below?(path, mountpoint)
+
+            owner = (targets + [mountpoint]).select { |t| FileDiff.below?(path, t) }.max_by(&:length)
+            next unless owner == mountpoint
+
+            rel = mountpoint == "/" ? path.delete_prefix("/") : path.delete_prefix("#{mountpoint}/")
+            return nil unless SAFE_REL.match?(rel) && rel.split("/").none? { |c| %w[. ..].include?(c) || c.empty? }
+
+            starts << rel
+          end
+          starts.uniq
+        end
+
+        # Mount points of the running guest below the tree's own.
+        def nested(tree, targets)
+          targets.select { |t| t != tree.mountpoint && FileDiff.below?(t, tree.mountpoint) }
+        end
+
+        # @return [Hash{String => String}] full path => SHA-256
+        def hashes(side, entries, candidates)
+          result = {}
+          candidates.group_by { |path| entries[path].tree }.each do |tree, paths|
+            raise Api::TaskCancelled, "diff" if @ctx.cancelled?
+
+            by_rel = paths.to_h { |path| [entries[path].rel, path] }
+            side.files.hashes(side.trees[tree].root, by_rel.keys, @tmpdir).each do |rel, hex|
+              result[by_rel[rel]] = hex
+            end
+          end
+          result
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-xapi/services/disk_image.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module VagrantPlugins
+  module Xapi
+    module Services
+      # Reads format and virtual size straight from a disk image header, so
+      # no qemu-img is needed:
+      #   VHD:   512-byte footer, "conectix", current size at offset 48
+      #   qcow2: "QFI\xFB" magic, size at offset 24 (8 bytes, big endian)
+      #   raw:   file size
+      module DiskImage
+        QCOW2_MAGIC = "QFI\xFB".b
+        VHD_COOKIE = "conectix".b
+        QCOW2_OFFSET_MASK = 0x00ff_ffff_ffff_fe00
+        QCOW2_COMPRESSED = 1 << 62
+        QCOW2_MAX_L1 = 1 << 20
+        # The header of qcow2 version 2; version 3 adds to it.
+        QCOW2_HEADER = 72
+
+        Info = Struct.new(:format, :virtual_size, :file_size, keyword_init: true)
+
+        module_function
+
+        # @return [Info]
+        def inspect_file(path)
+          raise Error.new(:disk_image_missing, path: path) unless File.file?(path)
+
+          size = File.size(path)
+          File.open(path, "rb") do |f|
+            head = f.read(512) || "".b
+            if head.byteslice(0, 4) == QCOW2_MAGIC
+              raise Error.new(:disk_image_truncated, path: path.to_s) if head.bytesize < QCOW2_HEADER
+
+              check_qcow2!(path, head)
+              return Info.new(format: "qcow2", virtual_size: head.byteslice(24, 8).unpack1("Q>"), file_size: size)
+            end
+
+            footer = read_vhd_footer(f, size, head)
+            if footer
+              return Info.new(format: "vhd", virtual_size: footer.byteslice(48, 8).unpack1("Q>"), file_size: size)
+            end
+          end
+          Info.new(format: "raw", virtual_size: size, file_size: size)
+        end
+
+        # A qcow2 may refer to other files: a backing file (header offset 8,
+        # 8 bytes) or, in version 3, an external data file (incompatible
+        # feature bit 2 at offset 72). qemu-img would read those files on the
+        # local machine and copy them into the image, so such images are refused.
+        def check_qcow2!(path, head)
+          raise Error.new(:disk_image_backing_file, path: path.to_s) unless head.byteslice(8, 8).unpack1("Q>").zero?
+
+          version = head.byteslice(4, 4).unpack1("N")
+          return unless version >= 3 && head.bytesize >= 80
+
+          incompatible = head.byteslice(72, 8).unpack1("Q>")
+          raise Error.new(:disk_image_data_file, path: path.to_s) unless (incompatible & 0b100).zero?
+        end
+
+        # Why XAPI's qcow2 import (qcow-stream-tool) cannot take this image as
+        # it is, or nil. It refuses compressed clusters ("Compressed_unsupported"),
+        # which cloud images and many libvirt boxes use, after the whole file
+        # was uploaded. Incompatible features other than the dirty bit (zstd
+        # compression, extended L2 entries) are not taken either.
+        # @return [Symbol, nil] :compressed or :features
+        def qcow2_import_obstacle(path)
+          File.open(path, "rb") do |f|
+            head = f.read(104) || "".b
+            return nil unless head.byteslice(0, 4) == QCOW2_MAGIC && head.bytesize >= 48
+
+            if head.byteslice(4, 4).unpack1("N") >= 3 && head.bytesize >= 80
+              return :features unless (head.byteslice(72, 8).unpack1("Q>") & ~0b1).zero?
+            end
+            cluster_bits = head.byteslice(20, 4).unpack1("N")
+            l1_size = head.byteslice(36, 4).unpack1("N")
+            return nil if l1_size.zero?
+            return :features unless cluster_bits.between?(9, 21) && l1_size <= QCOW2_MAX_L1
+
+            compressed_clusters?(f, head.byteslice(40, 8).unpack1("Q>"), l1_size, 1 << cluster_bits) ? :compressed : nil
+          end
+        end
+
+        # Reads the L2 tables (a few MB even for big images) and stops at the
+        # first compressed cluster.
+        def compressed_clusters?(file, l1_offset, l1_size, cluster_size)
+          file.seek(l1_offset)
+          (file.read(8 * l1_size) || "").unpack("Q>*").any? do |l1_entry|
+            l2_offset = l1_entry & QCOW2_OFFSET_MASK
+            next false if l2_offset.zero?
+
+            file.seek(l2_offset)
+            (file.read(cluster_size) || "").unpack("Q>*").any? { |entry| (entry & QCOW2_COMPRESSED).nonzero? }
+          end
+        end
+
+        # The format XAPI's /import_raw_vdi expects for a file.
+        def format_of(path)
+          inspect_file(path).format
+        end
+
+        def read_vhd_footer(file, size, head)
+          if size >= 512
+            file.seek(size - 512)
+            tail = file.read(512)
+            return tail if tail&.byteslice(0, 8) == VHD_COOKIE
+
+            # Some tools write a 511-byte footer.
+            if size >= 511
+              file.seek(size - 511)
+              tail = file.read(511)
+              return tail if tail&.byteslice(0, 8) == VHD_COOKIE
+            end
+          end
+          # Dynamic VHDs carry a copy of the footer at the start.
+          head.byteslice(0, 8) == VHD_COOKIE ? head : nil
+        end
+      end
+    end
+  end
+end