r10k 3.12.0 → 3.14.1

data/lib/r10k/tarball.rb

@@ -0,0 +1,183 @@
+require 'fileutils'
+require 'find'
+require 'minitar'
+require 'tempfile'
+require 'uri'
+require 'zlib'
+require 'r10k/settings'
+require 'r10k/settings/mixin'
+require 'r10k/util/platform'
+require 'r10k/util/cacheable'
+require 'r10k/util/downloader'
+
+module R10K
+  class Tarball
+
+    include R10K::Settings::Mixin
+    include R10K::Util::Cacheable
+    include R10K::Util::Downloader
+
+    def_setting_attr :proxy      # Defaults to global proxy setting
+    def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir
+
+    # @!attribute [rw] name
+    #   @return [String] The tarball's name
+    attr_accessor :name
+
+    # @!attribute [rw] source
+    #   @return [String] The tarball's source
+    attr_accessor :source
+
+    # @!attribute [rw] checksum
+    #   @return [String] The tarball's expected sha256 digest
+    attr_accessor :checksum
+
+    # @param name [String] The name of the tarball content
+    # @param source [String] The source for the tarball content
+    # @param checksum [String] The sha256 digest of the tarball content
+    def initialize(name, source, checksum: nil)
+      @name = name
+      @source = source
+      @checksum = checksum
+
+      # At this time, the only checksum type supported is sha256. In the future,
+      # we may decide to support other algorithms if a use case arises. TBD.
+      checksum_algorithm = :SHA256
+    end
+
+    # @return [String] Directory. Where the cache_basename file will be created.
+    def cache_dirname
+      File.join(settings[:cache_root], 'tarball')
+    end
+
+    # The final cache_path should match one of the templates:
+    #
+    #   - {cachedir}/{checksum}.tar.gz
+    #   - {cachedir}/{source}.tar.gz
+    #
+    # @return [String] File. The full file path the tarball will be cached to.
+    def cache_path
+      File.join(cache_dirname, cache_basename)
+    end
+
+    # @return [String] The basename of the tarball cache file.
+    def cache_basename
+      if checksum.nil?
+        sanitized_dirname(source) + '.tar.gz'
+      else
+        checksum + '.tar.gz'
+      end
+    end
+
+    # Extract the cached tarball to the target directory.
+    #
+    # @param target_dir [String] Where to unpack the tarball
+    def unpack(target_dir)
+      file = File.open(cache_path, 'rb')
+      reader = Zlib::GzipReader.new(file)
+      begin
+        Minitar.unpack(reader, target_dir)
+      ensure
+        reader.close
+      end
+    end
+
+    # @param target_dir [String] The directory to check if is in sync with the
+    #        tarball content
+    # @param ignore_untracked_files [Boolean] If true, consider the target
+    #        dir to be in sync as long as all tracked content matches.
+    #
+    # @return [Boolean]
+    def insync?(target_dir, ignore_untracked_files: false)
+      target_tree_entries = Find.find(target_dir).map(&:to_s) - [target_dir]
+      each_tarball_entry do |entry|
+        found = target_tree_entries.delete(File.join(target_dir, entry.full_name.chomp('/')))
+        return false if found.nil?
+        next if entry.directory?
+        return false unless file_digest(found) == reader_digest(entry)
+      end
+
+      if ignore_untracked_files
+        # We wouldn't have gotten this far if there were discrepancies in
+        # tracked content
+        true
+      else
+        # If there are still files in target_tree_entries, then there is
+        # untracked content present in the target tree. If not, we're in sync.
+        target_tree_entries.empty?
+      end
+    end
+
+    # Download the tarball from @source to @cache_path
+    def get
+      Tempfile.open(cache_basename) do |tempfile|
+        tempfile.binmode
+        src_uri = URI.parse(source)
+
+        temp_digest = case src_uri.scheme
+                      when 'file', nil
+                        copy(src_uri.path, tempfile)
+                      when %r{^[a-z]$} # Windows drive letter
+                        copy(src_uri.to_s, tempfile)
+                      when %r{^https?$}
+                        download(src_uri, tempfile)
+                      else
+                        raise "Unexpected source scheme #{src_uri.scheme}"
+                      end
+
+        # Verify the download
+        unless (checksum == temp_digest) || checksum.nil?
+          raise 'Downloaded file does not match checksum'
+        end
+
+        # Move the download to cache_path
+        FileUtils::mkdir_p(cache_dirname)
+        begin
+          FileUtils.mv(tempfile.path, cache_path)
+        rescue Errno::EACCES
+          # It may be the case that permissions don't permit moving the file
+          # into place, but do permit overwriting an existing in-place file.
+          FileUtils.cp(tempfile.path, cache_path)
+        end
+      end
+    end
+
+    # Checks the cached tarball's digest against the expected checksum. Returns
+    # false if no cached file is present. If the tarball has no expected
+    # checksum, any cached file is assumed to be valid.
+    #
+    # @return [Boolean]
+    def cache_valid?
+      return false unless File.exist?(cache_path)
+      return true if checksum.nil?
+      checksum == file_digest(cache_path)
+    end
+
+    # List all of the files contained in the tarball and their paths. This is
+    # useful for implementing R10K::Purgable
+    #
+    # @return [Array] A normalized list of file paths contained in the archive
+    def paths
+      names = Array.new
+      each_tarball_entry { |entry| names << Pathname.new(entry).cleanpath.to_s }
+      names - ['.']
+    end
+
+    def cache_checksum
+      raise R10K::Error, _("Cache not present at %{path}") % {path: cache_path} unless File.exist?(cache_path)
+      file_digest(cache_path)
+    end
+
+    private
+
+    def each_tarball_entry(&block)
+      File.open(cache_path, 'rb') do |file|
+        Zlib::GzipReader.wrap(file) do |reader|
+          Archive::Tar::Minitar::Input.each_entry(reader) do |entry|
+            yield entry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/r10k/util/cacheable.rb

@@ -0,0 +1,31 @@
+module R10K
+  module Util
+
+    # Utility mixin for classes that need to implement caches
+    #
+    # @abstract Classes using this mixin need to implement {#managed_directory} and
+    #   {#desired_contents}
+    module Cacheable
+
+      # Provide a default cachedir location. This is consumed by R10K::Settings
+      # for appropriate global default values.
+      #
+      # @return [String] Path to the default cache directory
+      def self.default_cachedir(basename = 'cache')
+        if R10K::Util::Platform.windows?
+          File.join(ENV['LOCALAPPDATA'], 'r10k', basename)
+        else
+          File.join(ENV['HOME'] || '/root', '.r10k', basename)
+        end
+      end
+
+      # Reformat a string into something that can be used as a directory
+      #
+      # @param string [String] An identifier to create a sanitized dirname for
+      # @return [String] A sanitized dirname for the given string
+      def sanitized_dirname(string)
+        string.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
+      end
+    end
+  end
+end

data/lib/r10k/util/downloader.rb

@@ -0,0 +1,134 @@
+require 'digest'
+require 'net/http'
+
+module R10K
+  module Util
+
+    # Utility mixin for classes that need to download files
+    module Downloader
+
+      # Downloader objects need to checksum downloaded or saved content. The
+      # algorithm used to perform this checksumming (and therefore the kinds of
+      # checksums returned by various methods) is reported by this method.
+      #
+      # @return [Symbol] The checksum algorithm the downloader uses
+      def checksum_algorithm
+        @checksum_algorithm ||= :SHA256
+      end
+
+      private
+
+      # Set the checksum algorithm the downloader should use. It should be a
+      # symbol, and a valid Ruby 'digest' library algorithm. The default is
+      # :SHA256.
+      #
+      # @param algorithm [Symbol] The checksum algorithm the downloader should use
+      def checksum_algorithm=(algorithm)
+        @checksum_algorithm = algorithm
+      end
+
+      CHUNK_SIZE = 64 * 1024 # 64 kb
+
+      # @param src_uri [URI] The URI to download from
+      # @param dst_file [String] The file or path to save to
+      # @return [String] The downloaded file's hex digest
+      def download(src_uri, dst_file)
+        digest = Digest(checksum_algorithm).new
+        http_get(src_uri) do |resp|
+          File.open(dst_file, 'wb') do |output_stream|
+            resp.read_body do |chunk|
+              output_stream.write(chunk)
+              digest.update(chunk)
+            end
+          end
+        end
+
+        digest.hexdigest
+      end
+
+      # @param src_file The file or path to copy from
+      # @param dst_file The file or path to copy to
+      # @return [String] The copied file's sha256 hex digest
+      def copy(src_file, dst_file)
+        digest = Digest(checksum_algorithm).new
+        File.open(src_file, 'rb') do |input_stream|
+          File.open(dst_file, 'wb') do |output_stream|
+            until input_stream.eof?
+              chunk = input_stream.read(CHUNK_SIZE)
+              output_stream.write(chunk)
+              digest.update(chunk)
+            end
+          end
+        end
+
+        digest.hexdigest
+      end
+
+      # Start a Net::HTTP::Get connection, then yield the Net::HTTPSuccess object
+      # to the caller's block. Follow redirects if Net::HTTPRedirection responses
+      # are encountered, and use a proxy if directed.
+      #
+      # @param uri [URI] The URI to download the file from
+      # @param redirect_limit [Integer] How many redirects to permit before failing
+      # @param proxy [URI, String] The URI to use as a proxy
+      def http_get(uri, redirect_limit: 10, proxy: nil, &block)
+        raise "HTTP redirect too deep" if redirect_limit.zero?
+
+        session = Net::HTTP.new(uri.host, uri.port, *proxy_to_array(proxy))
+        session.use_ssl = true if uri.scheme == 'https'
+        session.start
+
+        begin
+          session.request_get(uri) do |response|
+            case response
+            when Net::HTTPRedirection
+              redirect = response['location']
+              session.finish
+              return http_get(URI.parse(redirect), redirect_limit: redirect_limit - 1, proxy: proxy, &block)
+            when Net::HTTPSuccess
+              yield response
+            else
+              raise "Unexpected response code #{response.code}: #{response}"
+            end
+          end
+        ensure
+          session.finish if session.active?
+        end
+      end
+
+      # Helper method to translate a proxy URI to array arguments for
+      # Net::HTTP#new. A nil argument returns nil array elements.
+      def proxy_to_array(proxy_uri)
+        if proxy_uri
+          px = proxy_uri.is_a?(URI) ? proxy_uri : URI.parse(proxy_uri)
+          [px.host, px.port, px.user, px.password]
+        else
+          [nil, nil, nil, nil]
+        end
+      end
+
+      # Return the sha256 digest of the file at the given path
+      #
+      # @param path [String] The path to the file
+      # @return [String] The file's sha256 hex digest
+      def file_digest(path)
+        File.open(path) do |file|
+          reader_digest(file)
+        end
+      end
+
+      # Return the sha256 digest of the readable data
+      #
+      # @param reader [String] An object that responds to #read
+      # @return [String] The read data's sha256 hex digest
+      def reader_digest(reader)
+        digest = Digest(checksum_algorithm).new
+        while chunk = reader.read(CHUNK_SIZE)
+          digest.update(chunk)
+        end
+
+        digest.hexdigest
+      end
+    end
+  end
+end

data/lib/r10k/util/purgeable.rb

@@ -1,3 +1,5 @@
+require 'r10k/logging'
+
 require 'fileutils'
 
 module R10K
@@ -9,6 +11,8 @@ module R10K
     #   {#desired_contents}
     module Purgeable
 
+      include R10K::Logging
+
       HIDDEN_FILE = /\.[^.]+/
 
       FN_MATCH_OPTS = File::FNM_PATHNAME | File::FNM_DOTMATCH
@@ -95,8 +99,8 @@ module R10K
         end
 
         children.flat_map do |child|
-          if File.directory?(child) && recurse
-            potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse)
+          if File.directory?(child) && !File.symlink?(child) && recurse
+            potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse) << child.to_s
           else
             child.to_s
           end

data/lib/r10k/util/setopts.rb

@@ -1,3 +1,5 @@
+require 'r10k/logging'
+
 module R10K
   module Util
 

data/lib/r10k/util/subprocess.rb

@@ -1,3 +1,4 @@
+require 'r10k/logging'
 require 'r10k/util/platform'
 
 module R10K

data/lib/r10k/version.rb

@@ -2,5 +2,5 @@ module R10K
   # When updating to a new major (X) or minor (Y) version, include `#major` or
   # `#minor` (respectively) in your commit message to trigger the appropriate
   # release. Otherwise, a new patch (Z) version will be released.
-  VERSION = '3.12.0'
+  VERSION = '3.14.1'
 end