r10k 3.3.0 → 3.3.1

data/docker/Makefile

@@ -4,18 +4,18 @@ git_describe = $(shell git describe)
 vcs_ref := $(shell git rev-parse HEAD)
 build_date := $(shell date -u +%FT%T)
 hadolint_available := $(shell hadolint --help > /dev/null 2>&1; echo $$?)
-hadolint_command := hadolint --ignore DL3008 --ignore DL3018 --ignore DL4000 --ignore DL4001
+hadolint_command := hadolint --ignore DL3008 --ignore DL3018 --ignore DL3028 --ignore DL4000 --ignore DL4001
 hadolint_container := hadolint/hadolint:latest
 pwd := $(shell pwd)
 export BUNDLE_PATH = $(pwd)/.bundle/gems
 export BUNDLE_BIN = $(pwd)/.bundle/bin
 export GEMFILE = $(pwd)/Gemfile
 
-version = $(shell echo $(git_describe) | sed 's/-.*//')
+version := $(shell command grep VERSION ../lib/r10k/version.rb | awk '{print $$3}' | sed "s/'//g")
 dockerfile := Dockerfile
 
 prep:
-	@git fetch --unshallow ||:
+	@git fetch --unshallow 2> /dev/null ||:
 	@git fetch origin 'refs/tags/*:refs/tags/*'
 
 lint:
@@ -34,8 +34,7 @@ build: prep
 		--build-arg version=$(version) \
 		--build-arg pupperware_analytics_stream=$(PUPPERWARE_ANALYTICS_STREAM) \
 		--file r10k/$(dockerfile) \
-		--tag $(NAMESPACE)/r10k:$(version) \
-		r10k
+		--tag $(NAMESPACE)/r10k:$(version) $(pwd)/..
 ifeq ($(IS_LATEST),true)
 	@docker tag $(NAMESPACE)/r10k:$(version) puppet/r10k:latest
 endif
@@ -44,7 +43,7 @@ test: prep
 	@bundle install --path $$BUNDLE_PATH --gemfile $$GEMFILE
 	@PUPPET_TEST_DOCKER_IMAGE=$(NAMESPACE)/r10k:$(version) \
 		bundle exec --gemfile $$GEMFILE \
-		rspec r10k/spec
+		rspec spec
 
 push-image: prep
 	@docker push $(NAMESPACE)/r10k:$(version)
@@ -64,4 +63,4 @@ push-readme:
 
 publish: push-image push-readme
 
-.PHONY: prep lint build test publish push-image push-readme
+.PHONY: lint build test prep publish push-image push-readme

data/docker/r10k/Dockerfile

@@ -1,3 +1,12 @@
+FROM alpine:3.9 as build
+
+RUN apk add --no-cache ruby git
+RUN mkdir /workspace
+WORKDIR /workspace
+COPY . /workspace
+RUN gem build r10k.gemspec && \
+    mv r10k*.gem r10k.gem
+
 FROM alpine:3.9
 
 ARG vcs_ref
@@ -23,14 +32,15 @@ LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
       org.label-schema.dockerfile="/Dockerfile"
 
 RUN apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-dev make gcc musl-dev
+COPY --from=build /workspace/r10k.gem /
+RUN gem install --no-doc r10k.gem json etc && \
+    rm -f r10k.gem
 
-RUN gem install --no-doc r10k:"$R10K_VERSION" json etc
-
-COPY docker-entrypoint.sh /
+COPY docker/r10k/docker-entrypoint.sh /
 RUN chmod +x /docker-entrypoint.sh
-COPY docker-entrypoint.d /docker-entrypoint.d
+COPY docker/r10k/docker-entrypoint.d /docker-entrypoint.d
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["help"]
 
-COPY Dockerfile /
+COPY docker/r10k/Dockerfile /

data/docker/{r10k/spec → spec}/dockerfile_spec.rb

@@ -14,16 +14,7 @@ describe 'r10k container' do
   end
 
   before(:all) do
-    @image = ENV['PUPPET_TEST_DOCKER_IMAGE']
-    if @image.nil?
-      error_message = <<-MSG
-* * * * *
-  PUPPET_TEST_DOCKER_IMAGE environment variable must be set so we
-  know which image to test against!
-* * * * *
-      MSG
-      fail error_message
-    end
+    @image = require_test_image
   end
 
   after(:all) do

data/lib/r10k/action/deploy/environment.rb

@@ -129,7 +129,7 @@ module R10K
         end
 
         def visit_puppetfile(puppetfile)
-          puppetfile.load
+          puppetfile.load(@opts[:'default-branch-override'])
 
           yield
 
@@ -145,11 +145,24 @@ module R10K
         end
 
         def write_environment_info!(environment, started_at, success)
+          module_deploys = []
+          begin
+            environment.puppetfile.modules.each do |mod|
+              name = mod.name
+              version = mod.version
+              sha = mod.repo.head rescue nil
+              module_deploys.push({:name => name, :version => version, :sha => sha})
+            end
+          rescue
+            logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}")
+          end
+
           File.open("#{environment.path}/.r10k-deploy.json", 'w') do |f|
             deploy_info = environment.info.merge({
               :started_at => started_at,
               :finished_at => Time.new,
               :deploy_success => success,
+              :module_deploys => module_deploys,
             })
 
             f.puts(JSON.pretty_generate(deploy_info))
@@ -166,7 +179,12 @@ module R10K
         end
 
         def allowed_initialize_opts
-          super.merge(puppetfile: :self, cachedir: :self, 'no-force': :self, 'generate-types': :self, 'puppet-path': :self)
+          super.merge(puppetfile: :self,
+                      cachedir: :self,
+                      'no-force': :self,
+                      'generate-types': :self,
+                      'puppet-path': :self,
+                      'default-branch-override': :self)
         end
       end
     end

data/lib/r10k/cli/deploy.rb

@@ -61,6 +61,7 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off.
           DESCRIPTION
 
           flag :p, :puppetfile, 'Deploy modules from a puppetfile'
+          required nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile'
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Environment)
         end

data/lib/r10k/forge/module_release.rb

@@ -40,6 +40,10 @@ module R10K
       #   @return [Pathname] Where the md5 of the cached tarball is stored.
       attr_accessor :md5_file_path
 
+      # @!attribute [rw] sha256_file_path
+      #   @return [Pathname] Where the SHA256 of the cached tarball is stored.
+      attr_accessor :sha256_file_path
+
       # @!attribute [rw] unpack_path
       #   @return [Pathname] Where the module will be unpacked to.
       attr_accessor :unpack_path
@@ -65,6 +69,9 @@ module R10K
         md5_filename = @forge_release.slug + '.md5'
         @md5_file_path = @tarball_cache_root + md5_filename
 
+        sha256_filename = @forge_release.slug + '.sha256'
+        @sha256_file_path = @tarball_cache_root + sha256_filename
+
         @unpack_path   = Pathname.new(Dir.mktmpdir) + @forge_release.slug
       end
 
@@ -104,54 +111,79 @@ module R10K
       # module release checksum given by the Puppet Forge. On mismatch, remove
       # the cached copy.
       #
+      # @raise [R10K::Error] when no SHA256 is available and FIPS mode is on
       # @return [void]
       def verify
         logger.debug1 "Verifying that #{@tarball_cache_path} matches checksum"
 
-        md5_of_tarball = Digest::MD5.hexdigest(File.read(@tarball_cache_path, mode: 'rb'))
+        sha256_of_tarball = Digest::SHA256.file(@tarball_cache_path).hexdigest
 
-        if @md5_file_path.exist?
-          verify_from_md5_file(md5_of_tarball)
+        if @sha256_file_path.exist?
+          verify_from_file(sha256_of_tarball, @sha256_file_path)
         else
-          verify_from_forge(md5_of_tarball)
+          if @forge_release.respond_to?(:file_sha256) && !@forge_release.file_sha256.nil? && !@forge_release.file_sha256.size.zero?
+            forge_256_checksum = @forge_release.file_sha256
+            verify_from_forge(sha256_of_tarball, forge_256_checksum, @sha256_file_path)
+          else
+            if R10K::Util::Platform.fips?
+              raise R10K::Error, "Could not verify module, no SHA256 checksum available, and MD5 checksums not allowed in FIPS mode"
+            end
+
+            logger.debug1 "No SHA256 checksum available, falling back to MD5"
+            md5_of_tarball = Digest::MD5.file(@tarball_cache_path).hexdigest
+            if @md5_file_path.exist?
+              verify_from_file(md5_of_tarball, @md5_file_path)
+            else
+              verify_from_forge(md5_of_tarball, @forge_release.file_md5, @md5_file_path)
+            end
+          end
         end
       end
 
-      # Verify the md5 of the cached tarball against the
+      # Verify the checksum of the cached tarball against the
       # module release checksum stored in the cache as well.
       # On mismatch, remove the cached copy of both files.
+      # @param tarball_checksum [String] the checksum (either md5 or SHA256)
+      #                         of the downloaded module tarball
+      # @param file [Pathname] the file containing the checksum as downloaded
+      #             previously from the forge
+      # @param digest_class [Digest::SHA256, Digest::MD5] which checksum type
+      #                     to verify with
       #
       # @raise [PuppetForge::V3::Release::ChecksumMismatch] The
       #   cached module release checksum doesn't match the cached checksum.
       #
       # @return [void]
-      def verify_from_md5_file(md5_of_tarball)
-        md5_from_file = File.read(@md5_file_path).strip
-        if md5_of_tarball != md5_from_file
-          logger.error "MD5 of #{@tarball_cache_path} (#{md5_of_tarball}) does not match checksum #{md5_from_file} in #{@md5_file_path}. Removing both files."
-          cleanup_cached_tarball_path
-          cleanup_md5_file_path
+      def verify_from_file(tarball_checksum, checksum_file_path)
+        checksum_from_file = File.read(checksum_file_path).strip
+        if tarball_checksum != checksum_from_file
+          logger.error "Checksum of #{@tarball_cache_path} (#{tarball_checksum}) does not match checksum #{checksum_from_file} in #{checksum_file_path}. Removing both files."
+          @tarball_cache_path.delete
+          checksum_file_path.delete
           raise PuppetForge::V3::Release::ChecksumMismatch.new
         end
       end
 
-      # Verify the md5 of the cached tarball against the
+      # Verify the checksum of the cached tarball against the
       # module release checksum from the forge.
       # On mismatch, remove the cached copy of the tarball.
+      # @param tarball_checksum [String] the checksum (either md5 or SHA256)
+      #                         of the downloaded module tarball
+      # @param forge_checksum [String] the checksum downloaded from the Forge
+      # @param checksum_file_path [Pathname] the path to write the verified
+      #                            checksum to
       #
       # @raise [PuppetForge::V3::Release::ChecksumMismatch] The
       #   cached module release checksum doesn't match the forge checksum.
       #
       # @return [void]
-      def verify_from_forge(md5_of_tarball)
-        md5_from_forge = @forge_release.file_md5
-        #compare file_md5 to md5_of_tarball
-        if md5_of_tarball != md5_from_forge
-          logger.debug1 "MD5 of #{@tarball_cache_path} (#{md5_of_tarball}) does not match checksum #{md5_from_forge} found on the forge. Removing tarball."
-          cleanup_cached_tarball_path
+      def verify_from_forge(tarball_checksum, forge_checksum, checksum_file_path)
+        if tarball_checksum != forge_checksum
+          logger.debug1 "Checksum of #{@tarball_cache_path} (#{tarball_checksum}) does not match checksum #{forge_checksum} found on the forge. Removing tarball."
+          @tarball_cache_path.delete
           raise PuppetForge::V3::Release::ChecksumMismatch.new
         else
-          File.write(@md5_file_path, md5_from_forge)
+          File.write(checksum_file_path, forge_checksum)
         end
       end
 
@@ -191,20 +223,6 @@ module R10K
           download_path.parent.rmtree
         end
       end
-
-      # Remove the cached module release.
-      def cleanup_cached_tarball_path
-        if tarball_cache_path.exist?
-          tarball_cache_path.delete
-        end
-      end
-
-      # Remove the module release md5.
-      def cleanup_md5_file_path
-        if md5_file_path.exist?
-          md5_file_path.delete
-        end
-      end
     end
   end
 end

data/lib/r10k/module/git.rb

@@ -23,6 +23,11 @@ class R10K::Module::Git < R10K::Module::Base
   #   @return [String]
   attr_reader :desired_ref
 
+  # @!attribute [r] default_ref
+  #   @api private
+  #   @return [String]
+  attr_reader :default_ref
+
   def initialize(title, dirname, args, environment=nil)
     super
 

data/lib/r10k/puppetfile.rb

@@ -63,17 +63,20 @@ class Puppetfile
     @loaded = false
   end
 
-  def load
+  def load(default_branch_override = nil)
     if File.readable? @puppetfile_path
-      self.load!
+      self.load!(default_branch_override)
     else
       logger.debug _("Puppetfile %{path} missing or unreadable") % {path: @puppetfile_path.inspect}
     end
   end
 
-  def load!
+  def load!(default_branch_override = nil)
+    @default_branch_override = default_branch_override
+
     dsl = R10K::Puppetfile::DSL.new(self)
     dsl.instance_eval(puppetfile_contents, @puppetfile_path)
+    
     validate_no_duplicate_names(@modules)
     @loaded = true
   rescue SyntaxError, LoadError, ArgumentError, NameError => e
@@ -118,6 +121,10 @@ class Puppetfile
       install_path = @moduledir
     end
 
+    if args.is_a?(Hash) && @default_branch_override != nil
+      args[:default_branch] = @default_branch_override
+    end
+
     # Keep track of all the content this Puppetfile is managing to enable purging.
     @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
 
@@ -179,7 +186,21 @@ class Puppetfile
     logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
     mods_queue = modules_queue(visitor)
     thread_pool = pool_size.times.map { visitor_thread(visitor, mods_queue) }
-    thread_pool.each(&:join)
+    thread_exception = nil
+
+    # If any threads raise an exception the deployment is considered a failure.
+    # In that event clear the queue, wait for other threads to finish their
+    # current work, then re-raise the first exception caught.
+    begin
+      thread_pool.each(&:join)
+    rescue => e
+      logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
+      mods_queue.clear
+      thread_exception ||= e
+      retry
+    ensure
+      raise thread_exception unless thread_exception.nil?
+    end
   end
 
   def modules_queue(visitor)
@@ -195,8 +216,10 @@ class Puppetfile
       begin
         while mod = mods_queue.pop(true) do mod.accept(visitor) end
       rescue ThreadError => e
-        logger.error _("Thread error during concurrent module deploy: %{message}") % {message: e.message}
+        logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
         Thread.exit
+      rescue => e
+        Thread.main.raise(e)
       end
     end
   end

data/lib/r10k/util/platform.rb

@@ -3,6 +3,8 @@ require 'rbconfig'
 module R10K
   module Util
     module Platform
+      FIPS_FILE = "/proc/sys/crypto/fips_enabled"
+
       def self.platform
         # Test JRuby first to handle JRuby on Windows as well.
         if self.jruby?
@@ -14,6 +16,16 @@ module R10K
         end
       end
 
+      # We currently only suport FIPS mode on redhat 7, where it is
+      # toggled via a file.
+      def self.fips?
+        if File.exist?(FIPS_FILE)
+          File.read(FIPS_FILE).chomp == "1"
+        else
+          false
+        end
+      end
+
       def self.windows?
         RbConfig::CONFIG['host_os'] =~ /mswin|win32|dos|mingw|cygwin/i
       end

data/lib/r10k/version.rb

@@ -1,3 +1,3 @@
 module R10K
-  VERSION = '3.3.0'
+  VERSION = '3.3.1'
 end

data/locales/r10k.pot

@@ -6,11 +6,11 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: r10k 3.2.0-7-g213f14b\n"
+"Project-Id-Version: r10k 3.3.0-7-g6a2159a\n"
 "\n"
 "Report-Msgid-Bugs-To: docs@puppetlabs.com\n"
-"POT-Creation-Date: 2019-04-18 21:10+0000\n"
-"PO-Revision-Date: 2019-04-18 21:10+0000\n"
+"POT-Creation-Date: 2019-06-14 22:40+0000\n"
+"PO-Revision-Date: 2019-06-14 22:40+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -343,11 +343,15 @@ msgstr ""
 msgid "Updating modules with %{pool_size} threads"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:198
-msgid "Thread error during concurrent module deploy: %{message}"
+#: ../lib/r10k/puppetfile.rb:190
+msgid "Error during concurrent deploy of a module: %{message}"
 msgstr ""
 
-#: ../lib/r10k/puppetfile.rb:253
+#: ../lib/r10k/puppetfile.rb:212
+msgid "Module thread %{id} exiting: %{message}"
+msgstr ""
+
+#: ../lib/r10k/puppetfile.rb:269
 msgid "unrecognized declaration '%{method}'"
 msgstr ""