r10k 3.5.2

data/lib/r10k/errors/formatting.rb

@@ -0,0 +1,28 @@
+require 'r10k/errors'
+
+module R10K
+  module Errors
+    module Formatting
+      module_function
+
+      # Format this exception for displaying to the user
+      #
+      # @param exc [Exception] The exception to format
+      # @param with_backtrace [true, false] Whether the backtrace should be
+      #   included with this exception
+      # @return [String]
+      def format_exception(exc, with_backtrace = false)
+        lines = []
+        lines << exc.message
+        if with_backtrace
+          lines.concat(exc.backtrace)
+        end
+        if exc.respond_to?(:original) && exc.original
+          lines << "Original exception:"
+          lines<< format_exception(exc.original, with_backtrace)
+        end
+        lines.join("\n")
+      end
+    end
+  end
+end

data/lib/r10k/feature.rb

@@ -0,0 +1,56 @@
+require 'r10k/logging'
+
+module R10K
+  # Detect whether a given feature is present or absent
+  class Feature
+
+    include R10K::Logging
+
+    # @attribute [r] name
+    #   @return [Symbol] The name of this feature
+    attr_reader :name
+
+    # @param name [Symbol] The name of this feature
+    # @param opts [Hash]
+    # @param block [Proc] An optional block to detect if this feature is available
+    #
+    # @option opts [String, Array<String>] :libraries One or more libraries to
+    #   require to make sure this feature is present.
+    def initialize(name, opts = {}, &block)
+      @name      = name
+      @libraries = Array(opts.delete(:libraries))
+      @block     = block
+    end
+
+    # @return [true, false] Is this feature available?
+    def available?
+      logger.debug1 { _("Testing to see if feature %{name} is available.") % {name: @name} }
+      rv = @libraries.all? { |lib| library_available?(lib) } && proc_available?
+      msg = rv ? "is" : "is not"
+      logger.debug1 { _("Feature %{name} %{message} available.") % {name: @name, message: msg} }
+      rv
+    end
+
+    private
+
+    def library_available?(lib)
+      logger.debug2 { _("Attempting to load library '%{lib}' for feature %{name}") % {lib: lib, name: @name} }
+      require lib
+      true
+    rescue ScriptError => e
+      logger.debug2 { _("Error while loading library %{lib} for feature %{name}: %{error_msg}") % {lib: lib, name: @name, error_msg: e.message} }
+      false
+    end
+
+    def proc_available?
+      if @block
+        logger.debug2 { _("Evaluating proc %{block} to test for feature %{name}") % {block: @block.inspect, name: @name} }
+        output = @block.call
+        logger.debug2 { _("Proc %{block} for feature %{name} returned %{output}") % {block: @block.inspect, name: @name, output: output.inspect } }
+        !!output
+      else
+        true
+      end
+    end
+  end
+end

data/lib/r10k/feature/collection.rb

@@ -0,0 +1,23 @@
+require 'r10k/feature'
+
+# Store all features and indicate if they're available.
+class R10K::Feature::Collection
+  def initialize
+    @features = {}
+  end
+
+  # @param name [Symbol] The feature to add
+  # @param opts [Hash] Additional options for the feature, see {R10K::Feature}
+  # @param block [Proc] An optional block to detect if this feature is present
+  # @return [void]
+  def add(name, opts = {}, &block)
+    @features[name] = R10K::Feature.new(name, opts, &block)
+  end
+
+  # @return [true, false] Does a feature by this name exist and is it available?
+  def available?(name)
+    if @features.key?(name)
+      @features[name].available?
+    end
+  end
+end

data/lib/r10k/features.rb

@@ -0,0 +1,20 @@
+require 'r10k/feature/collection'
+require 'forwardable'
+require 'r10k/util/commands'
+
+module R10K
+  module Features
+    @features = R10K::Feature::Collection.new
+
+    class << self
+      extend Forwardable
+      def_delegators :@features, :add, :available?
+    end
+  end
+end
+
+R10K::Features.add(:shellgit) { R10K::Util::Commands.which('git') }
+
+R10K::Features.add(:rugged, :libraries => 'rugged')
+
+R10K::Features.add(:pe_license, :libraries => 'pe_license')

data/lib/r10k/forge/module_release.rb

@@ -0,0 +1,228 @@
+require 'r10k/logging'
+require 'r10k/settings/mixin'
+require 'fileutils'
+require 'tmpdir'
+require 'puppet_forge'
+
+module R10K
+  module Forge
+    # Download, unpack, and install modules from the Puppet Forge
+    class ModuleRelease
+
+      include R10K::Settings::Mixin
+
+      def_setting_attr :proxy
+      def_setting_attr :baseurl
+      def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/cache': '/root/.r10k/cache')
+
+      include R10K::Logging
+
+      # @!attribute [r] forge_release
+      #   @api private
+      #   @return [PuppetForge::V3::ModuleRelease] The Forge V3 API module
+      #     release object used for downloading and verifying the module
+      #     release.
+      attr_reader :forge_release
+
+      # @!attribute [rw] download_path
+      #   @return [Pathname] Where the module tarball will be downloaded to.
+      attr_accessor :download_path
+
+      # @!attribute [rw] tarball_cache_path
+      #   @return [Pathname] Where the module tarball will be cached to.
+      attr_accessor :tarball_cache_path
+
+      # @!attribute [rw] tarball_cache_root
+      #   @return [Pathname] Directory where the module tarball will be cached to.
+      attr_accessor :tarball_cache_root
+
+      # @!attribute [rw] md5_file_path
+      #   @return [Pathname] Where the md5 of the cached tarball is stored.
+      attr_accessor :md5_file_path
+
+      # @!attribute [rw] sha256_file_path
+      #   @return [Pathname] Where the SHA256 of the cached tarball is stored.
+      attr_accessor :sha256_file_path
+
+      # @!attribute [rw] unpack_path
+      #   @return [Pathname] Where the module will be unpacked to.
+      attr_accessor :unpack_path
+
+      # @param full_name [String] The hyphen separated name of the module
+      # @param version [String] The version of the module
+      def initialize(full_name, version)
+        @full_name = PuppetForge::V3.normalize_name(full_name)
+        @version   = version
+
+        # Copy the PuppetForge base connection to the release class; the connection
+        # objects are created in the class instances and thus are not shared with
+        # subclasses.
+        PuppetForge::V3::Release.conn = PuppetForge::V3::Base.conn
+
+        @forge_release = PuppetForge::V3::Release.new({ :name => @full_name, :version => @version, :slug => "#{@full_name}-#{@version}" })
+
+        tarball_name = @forge_release.slug + '.tar.gz'
+        @download_path = Pathname.new(Dir.mktmpdir) + (tarball_name)
+        @tarball_cache_root = Pathname.new(settings[:cache_root]) + (@forge_release.slug + "/tarball/")
+        @tarball_cache_path = @tarball_cache_root + tarball_name
+
+        md5_filename = @forge_release.slug + '.md5'
+        @md5_file_path = @tarball_cache_root + md5_filename
+
+        sha256_filename = @forge_release.slug + '.sha256'
+        @sha256_file_path = @tarball_cache_root + sha256_filename
+
+        @unpack_path   = Pathname.new(Dir.mktmpdir) + @forge_release.slug
+      end
+
+      # Download, unpack, and install this module release to the target directory.
+      #
+      # @example
+      #   environment_path = Pathname.new('/etc/puppetlabs/puppet/environments/production')
+      #   target_dir = environment_path + 'eight_hundred'
+      #   mod = R10K::Forge::ModuleRelease.new('branan-eight_hundred', '8.0.0')
+      #   mod.install(target_dir)
+      #
+      # @param target_dir [Pathname] The full path to where the module should be installed.
+      # @return [void]
+      def install(target_dir)
+        download
+        verify
+        unpack(target_dir)
+      ensure
+        cleanup
+      end
+
+      # Download the module release to {#download_path} and cache to {#tarball_cache_path}
+      #
+      # @return [void]
+      def download
+        if @tarball_cache_path.exist?
+          logger.debug1 "Using cached copy of #{@forge_release.slug} tarball"
+        else
+          logger.debug1 "Downloading #{@forge_release.slug} from #{PuppetForge::Release.conn.url_prefix} to #{@download_path}"
+          @forge_release.download(download_path)
+          FileUtils::mkdir_p(@tarball_cache_root)
+          FileUtils::mv(@download_path, @tarball_cache_path)
+        end
+      end
+
+      # Verify the module release cached in {#tarball_cache_path} against the
+      # module release checksum given by the Puppet Forge. On mismatch, remove
+      # the cached copy.
+      #
+      # @raise [R10K::Error] when no SHA256 is available and FIPS mode is on
+      # @return [void]
+      def verify
+        logger.debug1 "Verifying that #{@tarball_cache_path} matches checksum"
+
+        sha256_of_tarball = Digest(:SHA256).file(@tarball_cache_path).hexdigest
+
+        if @sha256_file_path.exist?
+          verify_from_file(sha256_of_tarball, @sha256_file_path)
+        else
+          if @forge_release.respond_to?(:file_sha256) && !@forge_release.file_sha256.nil? && !@forge_release.file_sha256.size.zero?
+            forge_256_checksum = @forge_release.file_sha256
+            verify_from_forge(sha256_of_tarball, forge_256_checksum, @sha256_file_path)
+          else
+            if R10K::Util::Platform.fips?
+              raise R10K::Error, "Could not verify module, no SHA256 checksum available, and MD5 checksums not allowed in FIPS mode"
+            end
+
+            logger.debug1 "No SHA256 checksum available, falling back to MD5"
+            md5_of_tarball = Digest(:MD5).file(@tarball_cache_path).hexdigest
+            if @md5_file_path.exist?
+              verify_from_file(md5_of_tarball, @md5_file_path)
+            else
+              verify_from_forge(md5_of_tarball, @forge_release.file_md5, @md5_file_path)
+            end
+          end
+        end
+      end
+
+      # Verify the checksum of the cached tarball against the
+      # module release checksum stored in the cache as well.
+      # On mismatch, remove the cached copy of both files.
+      # @param tarball_checksum [String] the checksum (either md5 or SHA256)
+      #                         of the downloaded module tarball
+      # @param file [Pathname] the file containing the checksum as downloaded
+      #             previously from the forge
+      # @param digest_class [Digest::SHA256, Digest::MD5] which checksum type
+      #                     to verify with
+      #
+      # @raise [PuppetForge::V3::Release::ChecksumMismatch] The
+      #   cached module release checksum doesn't match the cached checksum.
+      #
+      # @return [void]
+      def verify_from_file(tarball_checksum, checksum_file_path)
+        checksum_from_file = File.read(checksum_file_path).strip
+        if tarball_checksum != checksum_from_file
+          logger.error "Checksum of #{@tarball_cache_path} (#{tarball_checksum}) does not match checksum #{checksum_from_file} in #{checksum_file_path}. Removing both files."
+          @tarball_cache_path.delete
+          checksum_file_path.delete
+          raise PuppetForge::V3::Release::ChecksumMismatch.new
+        end
+      end
+
+      # Verify the checksum of the cached tarball against the
+      # module release checksum from the forge.
+      # On mismatch, remove the cached copy of the tarball.
+      # @param tarball_checksum [String] the checksum (either md5 or SHA256)
+      #                         of the downloaded module tarball
+      # @param forge_checksum [String] the checksum downloaded from the Forge
+      # @param checksum_file_path [Pathname] the path to write the verified
+      #                            checksum to
+      #
+      # @raise [PuppetForge::V3::Release::ChecksumMismatch] The
+      #   cached module release checksum doesn't match the forge checksum.
+      #
+      # @return [void]
+      def verify_from_forge(tarball_checksum, forge_checksum, checksum_file_path)
+        if tarball_checksum != forge_checksum
+          logger.debug1 "Checksum of #{@tarball_cache_path} (#{tarball_checksum}) does not match checksum #{forge_checksum} found on the forge. Removing tarball."
+          @tarball_cache_path.delete
+          raise PuppetForge::V3::Release::ChecksumMismatch.new
+        else
+          File.write(checksum_file_path, forge_checksum)
+        end
+      end
+
+      # Unpack the module release at {#tarball_cache_path}  into the given target_dir
+      #
+      # @param target_dir [Pathname] The final path where the module release
+      #   should be unpacked/installed into.
+      # @return [void]
+      def unpack(target_dir)
+        logger.debug1 _("Unpacking %{tarball_cache_path} to %{target_dir} (with tmpdir %{tmp_path})") % {tarball_cache_path: tarball_cache_path, target_dir: target_dir, tmp_path: unpack_path}
+        file_lists = PuppetForge::Unpacker.unpack(tarball_cache_path.to_s, target_dir.to_s, unpack_path.to_s)
+        logger.debug2 _("Valid files unpacked: %{valid_files}") % {valid_files: file_lists[:valid]}
+        if !file_lists[:invalid].empty?
+          logger.debug1 _("These files existed in the module's tar file, but are invalid filetypes and were not unpacked: %{invalid_files}") % {invalid_files: file_lists[:invalid]}
+        end
+        if !file_lists[:symlinks].empty?
+          logger.warn _("Symlinks are unsupported and were not unpacked from the module tarball. %{release_slug} contained these ignored symlinks: %{symlinks}") % {release_slug: @forge_release.slug, symlinks: file_lists[:symlinks]}
+        end
+      end
+
+      # Remove all files created while downloading and unpacking the module.
+      def cleanup
+        cleanup_unpack_path
+        cleanup_download_path
+      end
+
+      # Remove the temporary directory used for unpacking the module.
+      def cleanup_unpack_path
+        if unpack_path.parent.exist?
+          unpack_path.parent.rmtree
+        end
+      end
+
+      # Remove the downloaded module release.
+      def cleanup_download_path
+        if download_path.parent.exist?
+          download_path.parent.rmtree
+        end
+      end
+    end
+  end
+end

data/lib/r10k/git.rb

@@ -0,0 +1,196 @@
+require 'uri'
+require 'r10k/features'
+require 'r10k/errors'
+require 'r10k/settings'
+require 'r10k/logging'
+require 'r10k/util/platform'
+
+module R10K
+  module Git
+    require 'r10k/git/shellgit'
+    require 'r10k/git/rugged'
+
+    extend R10K::Logging
+
+    # A list of Git providers, sorted by priority. Providers have features that
+    # must be available for them to be used, and a module which is the namespace
+    # containing the implementation.
+    @providers = [
+      [ :shellgit,
+        {
+          :feature => :shellgit,
+          :module  => R10K::Git::ShellGit,
+        }
+      ],
+      [ :rugged,
+        {
+          :feature => :rugged,
+          :module  => R10K::Git::Rugged,
+          :on_set  => proc do
+            [:ssh, :https].each do |transport|
+              if !::Rugged.features.include?(transport)
+                logger.warn _("Rugged has been compiled without support for %{transport}; Git repositories will not be reachable via %{transport}.") % {transport: transport}
+              end
+            end
+          end
+        }
+      ],
+    ]
+
+    # Mark the current provider as invalid.
+    #
+    # If a provider is set to an invalid provider, we need to make sure that
+    # the provider doesn't fall back to the default value, thereby ignoring the
+    # explicit value and silently continuing. If the current provider is
+    # assigned to this value, no provider will be used until the provider is
+    # either reset or assigned a valid provider.
+    #
+    # @api private
+    NULL_PROVIDER = Object.new
+
+    # Mark the current provider as unset.
+    #
+    # If the provider has never been set we need to indicate that there is no
+    # current value but the default value can be used. If the current provider
+    # is assigned to this value and the provider is looked up, the default
+    # provider will be looked up and used.
+    #
+    # @api private
+    UNSET_PROVIDER = Object.new
+
+    # Return the first available Git provider.
+    #
+    # @raise [R10K::Error] if no Git providers are functional.
+    # @return [String] The name of the first available Git implementation.
+    def self.default_name
+      name, _ = @providers.find { |(_, hash)| R10K::Features.available?(hash[:feature]) }
+      if name.nil?
+        raise R10K::Error, _("No Git providers are functional.")
+      end
+      name
+    end
+
+    extend R10K::Logging
+
+    # Manually set the Git provider by name.
+    #
+    # @param name [Symbol] The name of the Git provider to use.
+    # @raise [R10K::Error] if the requested Git provider doesn't exist.
+    # @raise [R10K::Error] if the requested Git provider isn't functional.
+    # @return [void]
+    def self.provider=(name)
+      _, attrs = @providers.find { |(providername, _)| name == providername }
+      if attrs.nil?
+        @provider = NULL_PROVIDER
+        raise R10K::Error, _("No Git provider named '%{name}'.") % {name: name}
+      end
+      if !R10K::Features.available?(attrs[:feature])
+        @provider = NULL_PROVIDER
+        raise R10K::Error, _("Git provider '%{name}' is not functional.") % {name: name}
+      end
+      if attrs[:on_set]
+        attrs[:on_set].call
+      end
+
+      @provider = attrs[:module]
+      logger.debug1 { _("Setting Git provider to %{provider}") % {provider: @provider.name} }
+    end
+
+    # @return [Module] The namespace of the first available Git implementation.
+    #   Implementation classes should be looked up against this returned Module.
+    def self.provider
+      case @provider
+      when NULL_PROVIDER
+        raise R10K::Error, _("No Git provider set.")
+      when UNSET_PROVIDER
+        self.provider = default_name
+        logger.debug1 { _("Setting Git provider to default provider %{name}") % {name: default_name} }
+      end
+
+      @provider
+    end
+
+    def self.cache
+      provider::Cache
+    end
+
+    def self.bare_repository
+      provider::BareRepository
+    end
+
+    def self.thin_repository
+      provider::ThinRepository
+    end
+
+    # Clear the currently set provider.
+    #
+    # @api private
+    def self.reset!
+      @provider = UNSET_PROVIDER
+    end
+
+    @provider = UNSET_PROVIDER
+
+    extend R10K::Settings::Mixin::ClassMethods
+
+    def_setting_attr :private_key
+    def_setting_attr :proxy
+    def_setting_attr :username
+    def_setting_attr :repositories, {}
+
+    def self.get_repo_settings(remote)
+      self.settings[:repositories].find {|r| r[:remote] == remote }
+    end
+
+    def self.get_proxy_for_remote(remote)
+      # We only support proxy for HTTP(S) transport
+      return nil unless remote =~ /^http(s)?/i
+
+      repo_settings = self.get_repo_settings(remote)
+
+      if repo_settings && repo_settings.has_key?(:proxy)
+        proxy = repo_settings[:proxy] unless repo_settings[:proxy].nil? || repo_settings[:proxy].empty?
+      else
+        proxy = self.settings[:proxy]
+      end
+
+      R10K::Git.log_proxy_for_remote(proxy, remote) if proxy
+
+      proxy
+    end
+
+    def self.log_proxy_for_remote(proxy, remote)
+      # Sanitize passwords out of the proxy URI for loggging.
+      proxy_uri = URI.parse(proxy)
+      proxy_str = "#{proxy_uri.scheme}://"
+      proxy_str << "#{proxy_uri.userinfo.gsub(/:(.*)$/, ':<FILTERED>')}@" if proxy_uri.userinfo
+      proxy_str << "#{proxy_uri.host}:#{proxy_uri.port}"
+
+      logger.debug { "Using HTTP proxy '#{proxy_str}' for '#{remote}'." }
+
+      nil
+    end
+
+    # Execute block with given proxy configured in ENV
+    def self.with_proxy(new_proxy)
+      unless new_proxy.nil?
+        old_proxy = Hash[
+          ['HTTPS_PROXY', 'HTTP_PROXY', 'https_proxy', 'http_proxy'].collect do |var|
+            old_value = ENV[var]
+            ENV[var] = new_proxy
+
+            [var, old_value]
+          end
+        ]
+      end
+
+      begin
+        yield
+      ensure
+        ENV.update(old_proxy) if old_proxy
+      end
+
+      nil
+    end
+  end
+end