r10k 3.13.0 → 3.14.0

data/lib/r10k/tarball.rb

@@ -0,0 +1,183 @@
+require 'fileutils'
+require 'find'
+require 'minitar'
+require 'tempfile'
+require 'uri'
+require 'zlib'
+require 'r10k/settings'
+require 'r10k/settings/mixin'
+require 'r10k/util/platform'
+require 'r10k/util/cacheable'
+require 'r10k/util/downloader'
+
+module R10K
+  class Tarball
+
+    include R10K::Settings::Mixin
+    include R10K::Util::Cacheable
+    include R10K::Util::Downloader
+
+    def_setting_attr :proxy      # Defaults to global proxy setting
+    def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir
+
+    # @!attribute [rw] name
+    #   @return [String] The tarball's name
+    attr_accessor :name
+
+    # @!attribute [rw] source
+    #   @return [String] The tarball's source
+    attr_accessor :source
+
+    # @!attribute [rw] checksum
+    #   @return [String] The tarball's expected sha256 digest
+    attr_accessor :checksum
+
+    # @param name [String] The name of the tarball content
+    # @param source [String] The source for the tarball content
+    # @param checksum [String] The sha256 digest of the tarball content
+    def initialize(name, source, checksum: nil)
+      @name = name
+      @source = source
+      @checksum = checksum
+
+      # At this time, the only checksum type supported is sha256. In the future,
+      # we may decide to support other algorithms if a use case arises. TBD.
+      checksum_algorithm = :SHA256
+    end
+
+    # @return [String] Directory. Where the cache_basename file will be created.
+    def cache_dirname
+      File.join(settings[:cache_root], 'tarball')
+    end
+
+    # The final cache_path should match one of the templates:
+    #
+    #   - {cachedir}/{checksum}.tar.gz
+    #   - {cachedir}/{source}.tar.gz
+    #
+    # @return [String] File. The full file path the tarball will be cached to.
+    def cache_path
+      File.join(cache_dirname, cache_basename)
+    end
+
+    # @return [String] The basename of the tarball cache file.
+    def cache_basename
+      if checksum.nil?
+        sanitized_dirname(source) + '.tar.gz'
+      else
+        checksum + '.tar.gz'
+      end
+    end
+
+    # Extract the cached tarball to the target directory.
+    #
+    # @param target_dir [String] Where to unpack the tarball
+    def unpack(target_dir)
+      file = File.open(cache_path, 'rb')
+      reader = Zlib::GzipReader.new(file)
+      begin
+        Minitar.unpack(reader, target_dir)
+      ensure
+        reader.close
+      end
+    end
+
+    # @param target_dir [String] The directory to check if is in sync with the
+    #        tarball content
+    # @param ignore_untracked_files [Boolean] If true, consider the target
+    #        dir to be in sync as long as all tracked content matches.
+    #
+    # @return [Boolean]
+    def insync?(target_dir, ignore_untracked_files: false)
+      target_tree_entries = Find.find(target_dir).map(&:to_s) - [target_dir]
+      each_tarball_entry do |entry|
+        found = target_tree_entries.delete(File.join(target_dir, entry.full_name.chomp('/')))
+        return false if found.nil?
+        next if entry.directory?
+        return false unless file_digest(found) == reader_digest(entry)
+      end
+
+      if ignore_untracked_files
+        # We wouldn't have gotten this far if there were discrepancies in
+        # tracked content
+        true
+      else
+        # If there are still files in target_tree_entries, then there is
+        # untracked content present in the target tree. If not, we're in sync.
+        target_tree_entries.empty?
+      end
+    end
+
+    # Download the tarball from @source to @cache_path
+    def get
+      Tempfile.open(cache_basename) do |tempfile|
+        tempfile.binmode
+        src_uri = URI.parse(source)
+
+        temp_digest = case src_uri.scheme
+                      when 'file', nil
+                        copy(src_uri.path, tempfile)
+                      when %r{^[a-z]$} # Windows drive letter
+                        copy(src_uri.to_s, tempfile)
+                      when %r{^https?$}
+                        download(src_uri, tempfile)
+                      else
+                        raise "Unexpected source scheme #{src_uri.scheme}"
+                      end
+
+        # Verify the download
+        unless (checksum == temp_digest) || checksum.nil?
+          raise 'Downloaded file does not match checksum'
+        end
+
+        # Move the download to cache_path
+        FileUtils::mkdir_p(cache_dirname)
+        begin
+          FileUtils.mv(tempfile.path, cache_path)
+        rescue Errno::EACCES
+          # It may be the case that permissions don't permit moving the file
+          # into place, but do permit overwriting an existing in-place file.
+          FileUtils.cp(tempfile.path, cache_path)
+        end
+      end
+    end
+
+    # Checks the cached tarball's digest against the expected checksum. Returns
+    # false if no cached file is present. If the tarball has no expected
+    # checksum, any cached file is assumed to be valid.
+    #
+    # @return [Boolean]
+    def cache_valid?
+      return false unless File.exist?(cache_path)
+      return true if checksum.nil?
+      checksum == file_digest(cache_path)
+    end
+
+    # List all of the files contained in the tarball and their paths. This is
+    # useful for implementing R10K::Purgable
+    #
+    # @return [Array] A normalized list of file paths contained in the archive
+    def paths
+      names = Array.new
+      each_tarball_entry { |entry| names << Pathname.new(entry).cleanpath.to_s }
+      names - ['.']
+    end
+
+    def cache_checksum
+      raise R10K::Error, _("Cache not present at %{path}") % {path: cache_path} unless File.exist?(cache_path)
+      file_digest(cache_path)
+    end
+
+    private
+
+    def each_tarball_entry(&block)
+      File.open(cache_path, 'rb') do |file|
+        Zlib::GzipReader.wrap(file) do |reader|
+          Archive::Tar::Minitar::Input.each_entry(reader) do |entry|
+            yield entry
+          end
+        end
+      end
+    end
+  end
+end

data/lib/r10k/util/cacheable.rb

@@ -0,0 +1,31 @@
+module R10K
+  module Util
+
+    # Utility mixin for classes that need to implement caches
+    #
+    # @abstract Classes using this mixin need to implement {#managed_directory} and
+    #   {#desired_contents}
+    module Cacheable
+
+      # Provide a default cachedir location. This is consumed by R10K::Settings
+      # for appropriate global default values.
+      #
+      # @return [String] Path to the default cache directory
+      def self.default_cachedir(basename = 'cache')
+        if R10K::Util::Platform.windows?
+          File.join(ENV['LOCALAPPDATA'], 'r10k', basename)
+        else
+          File.join(ENV['HOME'] || '/root', '.r10k', basename)
+        end
+      end
+
+      # Reformat a string into something that can be used as a directory
+      #
+      # @param string [String] An identifier to create a sanitized dirname for
+      # @return [String] A sanitized dirname for the given string
+      def sanitized_dirname(string)
+        string.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
+      end
+    end
+  end
+end

data/lib/r10k/util/downloader.rb

@@ -0,0 +1,134 @@
+require 'digest'
+require 'net/http'
+
+module R10K
+  module Util
+
+    # Utility mixin for classes that need to download files
+    module Downloader
+
+      # Downloader objects need to checksum downloaded or saved content. The
+      # algorithm used to perform this checksumming (and therefore the kinds of
+      # checksums returned by various methods) is reported by this method.
+      #
+      # @return [Symbol] The checksum algorithm the downloader uses
+      def checksum_algorithm
+        @checksum_algorithm ||= :SHA256
+      end
+
+      private
+
+      # Set the checksum algorithm the downloader should use. It should be a
+      # symbol, and a valid Ruby 'digest' library algorithm. The default is
+      # :SHA256.
+      #
+      # @param algorithm [Symbol] The checksum algorithm the downloader should use
+      def checksum_algorithm=(algorithm)
+        @checksum_algorithm = algorithm
+      end
+
+      CHUNK_SIZE = 64 * 1024 # 64 kb
+
+      # @param src_uri [URI] The URI to download from
+      # @param dst_file [String] The file or path to save to
+      # @return [String] The downloaded file's hex digest
+      def download(src_uri, dst_file)
+        digest = Digest(checksum_algorithm).new
+        http_get(src_uri) do |resp|
+          File.open(dst_file, 'wb') do |output_stream|
+            resp.read_body do |chunk|
+              output_stream.write(chunk)
+              digest.update(chunk)
+            end
+          end
+        end
+
+        digest.hexdigest
+      end
+
+      # @param src_file The file or path to copy from
+      # @param dst_file The file or path to copy to
+      # @return [String] The copied file's sha256 hex digest
+      def copy(src_file, dst_file)
+        digest = Digest(checksum_algorithm).new
+        File.open(src_file, 'rb') do |input_stream|
+          File.open(dst_file, 'wb') do |output_stream|
+            until input_stream.eof?
+              chunk = input_stream.read(CHUNK_SIZE)
+              output_stream.write(chunk)
+              digest.update(chunk)
+            end
+          end
+        end
+
+        digest.hexdigest
+      end
+
+      # Start a Net::HTTP::Get connection, then yield the Net::HTTPSuccess object
+      # to the caller's block. Follow redirects if Net::HTTPRedirection responses
+      # are encountered, and use a proxy if directed.
+      #
+      # @param uri [URI] The URI to download the file from
+      # @param redirect_limit [Integer] How many redirects to permit before failing
+      # @param proxy [URI, String] The URI to use as a proxy
+      def http_get(uri, redirect_limit: 10, proxy: nil, &block)
+        raise "HTTP redirect too deep" if redirect_limit.zero?
+
+        session = Net::HTTP.new(uri.host, uri.port, *proxy_to_array(proxy))
+        session.use_ssl = true if uri.scheme == 'https'
+        session.start
+
+        begin
+          session.request_get(uri) do |response|
+            case response
+            when Net::HTTPRedirection
+              redirect = response['location']
+              session.finish
+              return http_get(URI.parse(redirect), redirect_limit: redirect_limit - 1, proxy: proxy, &block)
+            when Net::HTTPSuccess
+              yield response
+            else
+              raise "Unexpected response code #{response.code}: #{response}"
+            end
+          end
+        ensure
+          session.finish if session.active?
+        end
+      end
+
+      # Helper method to translate a proxy URI to array arguments for
+      # Net::HTTP#new. A nil argument returns nil array elements.
+      def proxy_to_array(proxy_uri)
+        if proxy_uri
+          px = proxy_uri.is_a?(URI) ? proxy_uri : URI.parse(proxy_uri)
+          [px.host, px.port, px.user, px.password]
+        else
+          [nil, nil, nil, nil]
+        end
+      end
+
+      # Return the sha256 digest of the file at the given path
+      #
+      # @param path [String] The path to the file
+      # @return [String] The file's sha256 hex digest
+      def file_digest(path)
+        File.open(path) do |file|
+          reader_digest(file)
+        end
+      end
+
+      # Return the sha256 digest of the readable data
+      #
+      # @param reader [String] An object that responds to #read
+      # @return [String] The read data's sha256 hex digest
+      def reader_digest(reader)
+        digest = Digest(checksum_algorithm).new
+        while chunk = reader.read(CHUNK_SIZE)
+          digest.update(chunk)
+        end
+
+        digest.hexdigest
+      end
+    end
+  end
+end

data/lib/r10k/version.rb

@@ -2,5 +2,5 @@ module R10K
   # When updating to a new major (X) or minor (Y) version, include `#major` or
   # `#minor` (respectively) in your commit message to trigger the appropriate
   # release. Otherwise, a new patch (Z) version will be released.
-  VERSION = '3.13.0'
+  VERSION = '3.14.0'
 end

data/locales/r10k.pot

@@ -6,11 +6,11 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: r10k 3.9.3-134-g9f93336f\n"
+"Project-Id-Version: r10k 3.9.3-170-g8cb36435\n"
 "\n"
 "Report-Msgid-Bugs-To: docs@puppetlabs.com\n"
-"POT-Creation-Date: 2021-10-25 18:18+0000\n"
-"PO-Revision-Date: 2021-10-25 18:18+0000\n"
+"POT-Creation-Date: 2021-11-16 21:43+0000\n"
+"PO-Revision-Date: 2021-11-16 21:43+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -75,15 +75,15 @@ msgstr ""
 msgid "Syntax OK"
 msgstr ""
 
-#: ../lib/r10k/action/runner.rb:57 ../lib/r10k/deployment/config.rb:42
+#: ../lib/r10k/action/runner.rb:63 ../lib/r10k/deployment/config.rb:42
 msgid "Overriding config file setting '%{key}': '%{old_val}' -> '%{new_val}'"
 msgstr ""
 
-#: ../lib/r10k/action/runner.rb:99
+#: ../lib/r10k/action/runner.rb:105
 msgid "Reading configuration from %{config_path}"
 msgstr ""
 
-#: ../lib/r10k/action/runner.rb:102
+#: ../lib/r10k/action/runner.rb:108
 msgid "No config file explicitly given and no default config file could be found, default settings will be used."
 msgstr ""
 
@@ -155,19 +155,19 @@ msgstr ""
 msgid "Proc %{block} for feature %{name} returned %{output}"
 msgstr ""
 
-#: ../lib/r10k/forge/module_release.rb:196
+#: ../lib/r10k/forge/module_release.rb:197
 msgid "Unpacking %{tarball_cache_path} to %{target_dir} (with tmpdir %{tmp_path})"
 msgstr ""
 
-#: ../lib/r10k/forge/module_release.rb:198
+#: ../lib/r10k/forge/module_release.rb:199
 msgid "Valid files unpacked: %{valid_files}"
 msgstr ""
 
-#: ../lib/r10k/forge/module_release.rb:200
+#: ../lib/r10k/forge/module_release.rb:201
 msgid "These files existed in the module's tar file, but are invalid filetypes and were not unpacked: %{invalid_files}"
 msgstr ""
 
-#: ../lib/r10k/forge/module_release.rb:203
+#: ../lib/r10k/forge/module_release.rb:204
 msgid "Symlinks are unsupported and were not unpacked from the module tarball. %{release_slug} contained these ignored symlinks: %{symlinks}"
 msgstr ""
 
@@ -203,11 +203,11 @@ msgstr ""
 msgid "Cannot write %{file}; parent directory does not exist"
 msgstr ""
 
-#: ../lib/r10k/git/cache.rb:65
+#: ../lib/r10k/git/cache.rb:57
 msgid "%{class}#path is deprecated; use #git_dir"
 msgstr ""
 
-#: ../lib/r10k/git/cache.rb:94
+#: ../lib/r10k/git/cache.rb:86
 msgid "Creating new git cache for %{remote}"
 msgstr ""
 
@@ -223,6 +223,14 @@ msgstr ""
 msgid "Rugged versions prior to 0.24.0 do not support pruning stale branches during fetch, please upgrade your \\'rugged\\' gem. (Current version is: %{version})"
 msgstr ""
 
+#: ../lib/r10k/git/rugged/base_repository.rb:24
+msgid "Unable to resolve %{pattern}: %{e} "
+msgstr ""
+
+#: ../lib/r10k/git/rugged/base_repository.rb:69
+msgid "Remote URL is different from cache, updating %{orig} to %{update}"
+msgstr ""
+
 #: ../lib/r10k/git/rugged/credentials.rb:28
 msgid "Authentication failed for Git remote %{url}."
 msgstr ""
@@ -367,7 +375,7 @@ msgstr ""
 msgid "%{repo_path} is already at Git ref %{ref}"
 msgstr ""
 
-#: ../lib/r10k/initializers.rb:30
+#: ../lib/r10k/initializers.rb:31
 msgid "the purgedirs key in r10k.yaml is deprecated. it is currently ignored."
 msgstr ""
 
@@ -379,7 +387,7 @@ msgstr ""
 msgid "No class registered for %{key}"
 msgstr ""
 
-#: ../lib/r10k/logging.rb:60
+#: ../lib/r10k/logging.rb:73 ../lib/r10k/logging.rb:100 ../lib/r10k/logging.rb:109
 msgid "Invalid log level '%{val}'. Valid levels are %{log_levels}"
 msgstr ""
 
@@ -439,39 +447,43 @@ msgstr ""
 msgid "Could not read metadata.json"
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:62
+#: ../lib/r10k/module_loader/puppetfile.rb:64
 msgid "Using Puppetfile '%{puppetfile}'"
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:63
+#: ../lib/r10k/module_loader/puppetfile.rb:65
 msgid "Using moduledir '%{moduledir}'"
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:84
+#: ../lib/r10k/module_loader/puppetfile.rb:86
 msgid "Failed to evaluate %{path}"
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:101
+#: ../lib/r10k/module_loader/puppetfile.rb:103
 msgid "Unable to preload Puppetfile because of %{msg}"
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:119
+#: ../lib/r10k/module_loader/puppetfile.rb:121
 msgid "Using Forge from Puppetfile: %{forge}"
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:122
+#: ../lib/r10k/module_loader/puppetfile.rb:124
 msgid "Ignoring Forge declaration in Puppetfile, using value from settings: %{forge}."
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:183 ../lib/r10k/puppetfile.rb:104
+#: ../lib/r10k/module_loader/puppetfile.rb:173
+msgid "\"basedir\" is deprecated. Please use \"environment_name\" instead. \"basedir\" will be removed in a future version."
+msgstr ""
+
+#: ../lib/r10k/module_loader/puppetfile.rb:192 ../lib/r10k/puppetfile.rb:104
 msgid "Puppetfile %{path} missing or unreadable"
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:219
+#: ../lib/r10k/module_loader/puppetfile.rb:228
 msgid "Puppetfiles cannot contain duplicate module names."
 msgstr ""
 
-#: ../lib/r10k/module_loader/puppetfile.rb:221
+#: ../lib/r10k/module_loader/puppetfile.rb:230
 msgid "Remove the duplicates of the following modules: %{dupes}"
 msgstr ""
 
@@ -602,6 +614,10 @@ msgstr ""
 msgid "Both username and password must be specified"
 msgstr ""
 
+#: ../lib/r10k/tarball.rb:167
+msgid "Cache not present at %{path}"
+msgstr ""
+
 #: ../lib/r10k/util/basedir.rb:34
 msgid "Expected Array<#desired_contents>, got R10K::Deployment"
 msgstr ""

data/r10k.gemspec

@@ -32,18 +32,18 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'gettext-setup', '~>0.24'
   # These two pins narrow what is allowed by gettext-setup,
-  # to preserver compatability with Ruby 2.4
+  # to preserve compatability with Ruby 2.4
   s.add_dependency 'fast_gettext', '~> 1.1.0'
   s.add_dependency 'gettext', ['>= 3.0.2', '< 3.3.0']
 
   s.add_dependency 'jwt', '~> 2.2.3'
+  s.add_dependency 'minitar', '~> 0.9.0'
 
   s.add_development_dependency 'rspec', '~> 3.1'
 
   s.add_development_dependency 'rake'
 
   s.add_development_dependency 'yard', '~> 0.9.11'
-  s.add_development_dependency 'minitar', '~> 0.9.0'
 
   s.files        = %x[git ls-files].split($/)
   s.require_path = 'lib'

data/spec/integration/git/rugged/cache_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+require 'r10k/git/rugged/cache'
+
+describe R10K::Git::Rugged::Cache, :if => R10K::Features.available?(:rugged) do
+  include_context 'Git integration'
+
+  let(:dirname) { 'working-repo' }
+  let(:remote_name) { 'origin' }
+
+  subject { described_class.new(remote) }
+
+  context "syncing with the remote" do
+    before(:each) do
+      subject.reset!
+    end
+
+    describe "with the correct configuration" do
+      it "is able to sync with the remote" do
+        subject.sync
+        expect(subject.synced?).to eq(true)
+      end
+    end
+
+    describe "with a out of date cached remote" do
+      it "updates the cached remote configuration" do
+        subject.repo.update_remote('foo', remote_name)
+        expect(subject.repo.remotes[remote_name]).to eq('foo')
+        subject.sync
+        expect(subject.repo.remotes[remote_name]).to eq(remote)
+      end
+    end
+  end
+end

data/spec/shared-contexts/tarball.rb

@@ -0,0 +1,32 @@
+require 'tmpdir'
+require 'fileutils'
+
+shared_context "Tarball" do
+  # Suggested subject:
+  #
+  #   subject { described_class.new('fixture-tarball', fixture_tarball, checksum: fixture_checksum) }
+  #
+  let(:fixture_tarball) do
+    File.expand_path('spec/fixtures/tarball/tarball.tar.gz', PROJECT_ROOT)
+  end
+
+  let(:fixture_checksum) { '292e692ad18faabd4f9b21037d51f0185e04b69f82c522a54af91fb5b88c2d3b' }
+
+  # Use tmpdir for cached tarballs
+  let(:tmpdir) { Dir.mktmpdir }
+
+  # `moduledir` and `cache_root` are available for examples to use in creating
+  # their subjects
+  let(:moduledir) { File.join(tmpdir, 'modules').tap { |path| Dir.mkdir(path) } }
+  let(:cache_root) { File.join(tmpdir, 'cache').tap { |path| Dir.mkdir(path) } }
+
+  around(:each) do |example|
+    if subject.is_a?(R10K::Tarball)
+      subject.settings[:cache_root] = cache_root
+    elsif subject.respond_to?(:tarball) && subject.tarball.is_a?(R10K::Tarball)
+      subject.tarball.settings[:cache_root] = cache_root
+    end
+    example.run
+    FileUtils.remove_entry_secure(tmpdir)
+  end
+end

data/spec/spec_helper.rb

@@ -19,6 +19,7 @@ require 'r10k'
 Dir.glob(File.expand_path('spec/shared-examples/**/*.rb', PROJECT_ROOT)).each { |file| require file }
 
 require 'shared-contexts/git-fixtures'
+require 'shared-contexts/tarball'
 require 'matchers/exit_with'
 require 'matchers/match_realpath'
 require 'r10k-mocks'

data/spec/unit/action/deploy/module_spec.rb

@@ -241,7 +241,7 @@ describe R10K::Action::Deploy::Module do
 
       allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo)
       allow(R10K::Git).to receive_message_chain(:cache, :generate).and_return(cache)
-      allow_any_instance_of(R10K::Source::Git).to receive(:branch_names).and_return([R10K::Environment::Name.new('first', {})])
+      allow_any_instance_of(R10K::Source::Git).to receive(:environment_names).and_return([R10K::Environment::Name.new('first', {})])
 
       expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
         # For this test we want to have realistic Modules and access to
@@ -300,7 +300,7 @@ describe R10K::Action::Deploy::Module do
 
       allow(R10K::Git::StatefulRepository).to receive(:new).and_return(repo)
       allow(R10K::Git).to receive_message_chain(:cache, :generate).and_return(cache)
-      allow_any_instance_of(R10K::Source::Git).to receive(:branch_names).and_return([R10K::Environment::Name.new('first', {}),
+      allow_any_instance_of(R10K::Source::Git).to receive(:environment_names).and_return([R10K::Environment::Name.new('first', {}),
                                                                                      R10K::Environment::Name.new('second', {})])
 
       expect(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|