r10k 2.5.5 → 2.6.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 1751a994285442b34b30d2274ed0049c242ad7b4
-  data.tar.gz: bea09a73ff5954f2f434ab018c307f3efc59e420
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NTZjY2UzMDMyMTJlNjIzZmQzNzk4MGViMGMzMjM0Yzc2MzdkZjg2MQ==
+  data.tar.gz: !binary |-
+    YTE1ZjhkYzBiZjY1YTFiOGUyMTliM2Q2MDAzNzg3YzA4NjUyN2Q0Ng==
 SHA512:
-  metadata.gz: 727021b3144cdf93ccadb11c9718ceccc2f054bc310614fdec5881e1ae704b65b50a385ceed17f68b446cc31ee5ed5bdd194c7c3ae66bf1183c0fd17de55b10d
-  data.tar.gz: 9975f54ffa36bdbd0a5c27d0478fca297f95a151a957b931fc12749f5d0fe62c5655afa3979f03bf6c4ea09cc0cd31dc40c4b7ed85caa1ecd63e3a0a990c7737
+  metadata.gz: !binary |-
+    NDRiNTI5NDA3NGQzNmVjODZjN2NlNmMyYjVjMjVjM2ZkYjNiMmY0ZWUyMjNj
+    NWI1NjExNTk2NDJhM2M0NjFlN2ExZTVmNGU5MzRmNjJjYWY1NzliMGY2Mzgy
+    NmUyN2FlNzZjMDAwMWFhMTdmZDA4ZTY4NThiZTBlMmM5MWZjOGE=
+  data.tar.gz: !binary |-
+    MmM4YzFlNTBmNTYyMzMxYjA0Zjc2YWU0OTJlMzQ1YzFmM2E2NDEzZGM2MWZi
+    NTczNzVhODdhNTc0Yjg0OWVhYjIxOTUwY2NkOTUyYmY0MzMyY2FhNDc1Mzhj
+    YWNlYTVjMGMwYjJmNjUwMDIxMTFiOTg4NTE4Yjk3ODVlNjg3Mzg=

data/.travis.yml

@@ -5,6 +5,7 @@ script: "bundle exec rspec --color --format documentation spec/unit"
 notifications:
   email: false
 sudo: false
+before_install: gem update bundler
 rvm:
   - "2.3.0"
   - "2.2.0"

data/CHANGELOG.mkd

@@ -1,6 +1,26 @@
 CHANGELOG
 =========
 
+2.6.0
+-----
+
+### New Features
+
+(RK-307) Branches can now be ignored by prefixes during deployment.
+
+(RK-305) Add --no-force to deploy action to avoid overwriting local module changes.
+
+(RK-264) Add --force action to puppetfile install to force overwriting local
+module changes.
+
+(RK-291) (RK-304) Add caching of forge modules.
+
+### Changes
+
+(RK-306) Remove the dependency on semantic_puppet.
+
+(RK-161) Deprecate the usage of PUPPETFILE and PUPPETFILE_DIR environment variables.
+
 2.5.5
 -----
 

data/doc/dynamic-environments/configuration.mkd

@@ -356,6 +356,32 @@ sources:
 * if `false` (default) environment folder will not be prefixed
 * if `String` environment folder will be prefixed with the `prefix` value.
 
+### ignore_branch_prefixes
+
+The 'ignore_branch_prefixes' setting causes environments to be ignored which match in part or whole
+to any of the prefixes listed in the setting.
+The setting is a list of strings. Each branch in
+the 'git' repo will have its name tested against all prefixes and, if the prefix
+is found, then an environment will not be deployed for this branch.
+If no 'ignore_branch_prefixes' is specified, then all branches in the 'git' repo will
+be deployed (default behavior).
+
+#### ignore_branch_prefixes behaviour
+* if empty, deploy environments for all branches
+* for each branch in git repo
+** if `branch.name` has a prefix found in `ignore_branch_prefixes`, then do not deploy an environment for branch
+
+Example: do not deploy branches with names starting with (or completely named) 'test' or 'dev'.
+```yaml
+---
+sources:
+  mysource:
+    basedir: '/etc/puppet/environments'
+    ignore_branch_prefixes:
+      - 'test'
+      - 'dev'
+```
+
 Examples
 --------
 

data/doc/puppetfile.mkd

@@ -306,7 +306,7 @@ The given 'install\_path' can be an absolute path or a path relative to the base
 the environment. Note that r10k will exit with an error if you attempt to set the
 'path' option to a directory outside of the environment.
 
-## Environment variables
+## Environment variables (**DEPRECATED** as of 2.5.6)
 
 It is possible to set an alternate name/location for your `Puppetfile` and
 `modules` directory. This is useful if you want to control multiple environments

data/integration/lib/r10k_utils.rb

@@ -76,6 +76,10 @@ def r10k_revert_environment(host, commit_sha, git_repo_path)
   #Force push changes to remote.
   git_on(host, 'push origin --mirror --force', git_repo_path)
   git_on(host, 'push origin --mirror --force', git_repo_path)
+
+  #Remove r10k cache
+  cachedir = '/var/cache/r10k'
+  on(master, "rm -rf #{cachedir}")
 end
 
 # Clean-up the r10k environment on the master to bring it back to a known good state.
@@ -102,6 +106,13 @@ def clean_up_r10k(master, commit_sha, git_repo_path)
   step 'Reset Git Repo to Known Good State'
   r10k_revert_environment(master, commit_sha, git_repo_path)
 
+  # RK-297 workaround. Without this, tests will fail with an error like the following:
+  # [2017-06-02 11:11:46 - ERROR] Object not found - no match for id (60e4ea82c9fdf86974a13f78b839a497325de04b)
+  # This cleanup should not be necessary when RK-297 has been resolved.
+  #
+  step 'Remove git directories from codedir to prevent cache errors'
+  on(master, "find #{environment_path } -name .git -type d -print0 | xargs -r0 -- rm -r")
+
   step 'Restore Original "production" Environment'
   on(master, "#{r10k_fqp} deploy environment -v")
 

data/integration/tests/git_source/HTTP_proxy_and_git_source.rb

@@ -66,7 +66,5 @@ git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_pat
 on(master, "#{r10k_fqp} deploy environment -p", :accept_all_exit_codes => true) do |r|
   regex = /proxy.*ilovecatvideos\.com/
   assert(r.exit_code == 1, 'expected error code was not observed')
-  expect_failure('Failure due to RK-262') do
-    assert_match(regex, r.stderr, 'The expected error message was not observed' )
-  end
+  assert_match(regex, r.stderr, 'The expected error message was not observed' )
 end

data/lib/r10k/action/deploy/environment.rb

@@ -13,6 +13,8 @@ module R10K
 
         include R10K::Action::Deploy::DeployHelpers
 
+        attr_reader :force
+
         def initialize(opts, argv, settings = nil)
           settings ||= {}
           @purge_levels = settings.fetch(:deploy, {}).fetch(:purge_levels, [])
@@ -20,6 +22,8 @@ module R10K
 
           super
 
+          # @force here is used to make it easier to reason about
+          @force = !@no_force
           @argv = @argv.map { |arg| arg.gsub(/\W/,'_') }
         end
 
@@ -117,7 +121,7 @@ module R10K
 
         def visit_module(mod)
           logger.info _("Deploying Puppetfile content %{mod_path}") % {mod_path: mod.path}
-          mod.sync
+          mod.sync(force: @force)
         end
 
         def write_environment_info!(environment, started_at, success)
@@ -142,7 +146,7 @@ module R10K
         end
 
         def allowed_initialize_opts
-          super.merge(puppetfile: :self, cachedir: :self)
+          super.merge(puppetfile: :self, cachedir: :self, :'no-force' => :self)
         end
       end
     end

data/lib/r10k/action/deploy/module.rb

@@ -10,6 +10,17 @@ module R10K
 
         include R10K::Action::Deploy::DeployHelpers
 
+        attr_reader :force
+
+        def initialize(opts, argv, settings = nil)
+          settings ||= {}
+
+          super
+
+          # @force here is used to make it easier to reason about
+          @force = !@no_force
+        end
+
         def call
           @visit_ok = true
 
@@ -50,14 +61,14 @@ module R10K
         def visit_module(mod)
           if @argv.include?(mod.name)
             logger.info _("Deploying module %{mod_path}") % {mod_path: mod.path}
-            mod.sync
+            mod.sync(force: @force)
           else
             logger.debug1(_("Only updating modules %{modules}, skipping module %{mod_name}") % {modules: @argv.inspect, mod_name: mod.name})
           end
         end
 
         def allowed_initialize_opts
-          super.merge(environment: true)
+          super.merge(environment: true, :'no-force' => :self)
         end
       end
     end

data/lib/r10k/action/puppetfile/cri_runner.rb

@@ -8,10 +8,16 @@ module R10K
       #
       # @api private
       # @deprecated The use of these environment variables is deprecated and
-      #   will be removed in 2.0.0.
+      #   will be removed in 3.0.0.
       class CriRunner < R10K::Action::CriRunner
+
+        include R10K::Logging
+
         def handle_opts(opts)
           opts[:root]       ||= wd
+          if env['PUPPETFILE_DIR'] || env['PUPPETFILE']
+            logger.warn _("The use of the PUPPETFILE and PUPPETFILE_DIR environment variables is deprecated.")
+          end
           opts[:moduledir]  ||= env['PUPPETFILE_DIR']
           opts[:puppetfile] ||= env['PUPPETFILE']
           super(opts)

data/lib/r10k/action/puppetfile/install.rb

@@ -10,7 +10,7 @@ module R10K
 
         def call
           @visit_ok = true
-          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
+          pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile, nil , @force)
           pf.accept(self)
           @visit_ok
         end
@@ -26,17 +26,18 @@ module R10K
         end
 
         def visit_module(mod)
+          @force ||= false
           logger.info _("Updating module %{mod_path}") % {mod_path: mod.path}
 
           if mod.respond_to?(:desired_ref) && mod.desired_ref == :control_branch
             logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available." % {name: mod.name})
           end
 
-          mod.sync(force: false) # Don't force sync for 'puppetfile install' RK-265
+          mod.sync(force: @force)
         end
 
         def allowed_initialize_opts
-          super.merge(root: :self, puppetfile: :self, moduledir: :self)
+          super.merge(root: :self, puppetfile: :self, moduledir: :self, force: :self )
         end
       end
     end

data/lib/r10k/cli/deploy.rb

@@ -22,6 +22,7 @@ module R10K::CLI
         DESCRIPTION
 
         required nil, :cachedir, 'Specify a cachedir, overriding the value in config'
+        flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
 
         run do |opts, args, cmd|
           puts cmd.help(:verbose => opts[:verbose])

data/lib/r10k/cli/puppetfile.rb

@@ -30,10 +30,9 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           name    'install'
           usage   'install'
           summary 'Install all modules from a Puppetfile'
-
           required nil, :moduledir, 'Path to install modules to'
           required nil, :puppetfile, 'Path to puppetfile'
-          # @todo add --no-purge option
+          flag     nil, :force, 'Force locally changed files to be overwritten'
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Install)
         end
       end

data/lib/r10k/forge/module_release.rb

@@ -13,6 +13,7 @@ module R10K
 
       def_setting_attr :proxy
       def_setting_attr :baseurl
+      def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/cache': '/root/.r10k/cache')
 
       include R10K::Logging
 
@@ -27,6 +28,18 @@ module R10K
       #   @return [Pathname] Where the module tarball will be downloaded to.
       attr_accessor :download_path
 
+      # @!attribute [rw] tarball_cache_path
+      #   @return [Pathname] Where the module tarball will be cached to.
+      attr_accessor :tarball_cache_path
+
+      # @!attribute [rw] tarball_cache_root
+      #   @return [Pathname] Directory where the module tarball will be cached to.
+      attr_accessor :tarball_cache_root
+
+      # @!attribute [rw] md5_file_path
+      #   @return [Pathname] Where the md5 of the cached tarball is stored.
+      attr_accessor :md5_file_path
+
       # @!attribute [rw] unpack_path
       #   @return [Pathname] Where the module will be unpacked to.
       attr_accessor :unpack_path
@@ -41,9 +54,17 @@ module R10K
         # objects are created in the class instances and thus are not shared with
         # subclasses.
         PuppetForge::V3::Release.conn = PuppetForge::V3::Base.conn
+
         @forge_release = PuppetForge::V3::Release.new({ :name => @full_name, :version => @version, :slug => "#{@full_name}-#{@version}" })
 
-        @download_path = Pathname.new(Dir.mktmpdir) + (@forge_release.slug + '.tar.gz')
+        tarball_name = @forge_release.slug + '.tar.gz'
+        @download_path = Pathname.new(Dir.mktmpdir) + (tarball_name)
+        @tarball_cache_root = Pathname.new(settings[:cache_root]) + (@forge_release.slug + "/tarball/")
+        @tarball_cache_path = @tarball_cache_root + tarball_name
+
+        md5_filename = @forge_release.slug + '.md5'
+        @md5_file_path = @tarball_cache_root + md5_filename
+
         @unpack_path   = Pathname.new(Dir.mktmpdir) + @forge_release.slug
       end
 
@@ -65,34 +86,83 @@ module R10K
         cleanup
       end
 
-      # Download the module release to {#download_path}
+      # Download the module release to {#download_path} and cache to {#tarball_cache_path}
       #
       # @return [void]
       def download
-        logger.debug1 "Downloading #{@forge_release.slug} from #{PuppetForge::Release.conn.url_prefix} to #{@download_path}"
-        @forge_release.download(download_path)
+        if @tarball_cache_path.exist?
+          logger.debug1 "Using cached copy of #{@forge_release.slug} tarball"
+        else
+          logger.debug1 "Downloading #{@forge_release.slug} from #{PuppetForge::Release.conn.url_prefix} to #{@download_path}"
+          @forge_release.download(download_path)
+          FileUtils::mkdir_p(@tarball_cache_root)
+          FileUtils::mv(@download_path, @tarball_cache_path)
+        end
       end
 
-      # Verify the module release downloaded to {#download_path} against the
-      # module release checksum given by the Puppet Forge
+      # Verify the module release cached in {#tarball_cache_path} against the
+      # module release checksum given by the Puppet Forge. On mismatch, remove
+      # the cached copy.
       #
-      # @raise [PuppetForge::V3::Release::ChecksumMismatch] The
-      #   downloaded module release checksum doesn't match the expected Forge
-      #   module release checksum.
       # @return [void]
       def verify
-        logger.debug1 "Verifying that #{download_path} matches checksum #{@forge_release.file_md5}"
-        @forge_release.verify(download_path)
+        logger.debug1 "Verifying that #{@tarball_cache_path} matches checksum"
+
+        md5_of_tarball = Digest::MD5.hexdigest(File.read(@tarball_cache_path))
+
+        if @md5_file_path.exist?
+          verify_from_md5_file(md5_of_tarball)
+        else
+          verify_from_forge(md5_of_tarball)
+        end
       end
 
-      # Unpack the module release at {#download_path} into the given target_dir
+      # Verify the md5 of the cached tarball against the
+      # module release checksum stored in the cache as well.
+      # On mismatch, remove the cached copy of both files.
+      #
+      # @raise [PuppetForge::V3::Release::ChecksumMismatch] The
+      #   cached module release checksum doesn't match the cached checksum.
+      #
+      # @return [void]
+      def verify_from_md5_file(md5_of_tarball)
+        md5_from_file = File.read(@md5_file_path).strip
+        if md5_of_tarball != md5_from_file
+          logger.error "MD5 of #{@tarball_cache_path} (#{md5_of_tarball}) does not match checksum #{md5_from_file} in #{@md5_file_path}. Removing both files."
+          cleanup_cached_tarball_path
+          cleanup_md5_file_path
+          raise PuppetForge::V3::Release::ChecksumMismatch.new
+        end
+      end
+
+      # Verify the md5 of the cached tarball against the
+      # module release checksum from the forge.
+      # On mismatch, remove the cached copy of the tarball.
+      #
+      # @raise [PuppetForge::V3::Release::ChecksumMismatch] The
+      #   cached module release checksum doesn't match the forge checksum.
+      #
+      # @return [void]
+      def verify_from_forge(md5_of_tarball)
+        md5_from_forge = @forge_release.file_md5
+        #compare file_md5 to md5_of_tarball
+        if md5_of_tarball != md5_from_forge
+          logger.debug1 "MD5 of #{@tarball_cache_path} (#{md5_of_tarball}) does not match checksum #{md5_from_forge} found on the forge. Removing tarball."
+          cleanup_cached_tarball_path
+          raise PuppetForge::V3::Release::ChecksumMismatch.new
+        else
+          File.write(@md5_file_path, md5_from_forge)
+        end
+      end
+
+      # Unpack the module release at {#tarball_cache_path}  into the given target_dir
       #
       # @param target_dir [Pathname] The final path where the module release
       #   should be unpacked/installed into.
       # @return [void]
       def unpack(target_dir)
-        logger.debug1 _("Unpacking %{download_path} to %{target_dir} (with tmpdir %{tmp_path})") % {download_path: download_path, target_dir: target_dir, tmp_path: unpack_path}
-        file_lists = PuppetForge::Unpacker.unpack(download_path.to_s, target_dir.to_s, unpack_path.to_s)
+        logger.debug1 _("Unpacking %{tarball_cache_path} to %{target_dir} (with tmpdir %{tmp_path})") % {tarball_cache_path: tarball_cache_path, target_dir: target_dir, tmp_path: unpack_path}
+        file_lists = PuppetForge::Unpacker.unpack(tarball_cache_path.to_s, target_dir.to_s, unpack_path.to_s)
         logger.debug2 _("Valid files unpacked: %{valid_files}") % {valid_files: file_lists[:valid]}
         if !file_lists[:invalid].empty?
           logger.debug1 _("These files existed in the module's tar file, but are invalid filetypes and were not unpacked: %{invalid_files}") % {invalid_files: file_lists[:invalid]}
@@ -121,6 +191,20 @@ module R10K
           download_path.delete
         end
       end
+
+      # Remove the cached module release.
+      def cleanup_cached_tarball_path
+        if tarball_cache_path.exist?
+          tarball_cache_path.delete
+        end
+      end
+
+      # Remove the module release md5.
+      def cleanup_md5_file_path
+        if md5_file_path.exist?
+          md5_file_path.delete
+        end
+      end
     end
   end
 end

data/lib/r10k/initializers.rb

@@ -31,6 +31,7 @@ module R10K
         end
 
         with_setting(:cachedir) { |value| R10K::Git::Cache.settings[:cache_root] = value }
+        with_setting(:cachedir) { |value| R10K::Forge::ModuleRelease.settings[:cache_root] = value }
 
         with_setting(:git) { |value| GitInitializer.new(value).call }
         with_setting(:forge) { |value| ForgeInitializer.new(value).call }

data/lib/r10k/module/forge.rb

@@ -6,7 +6,7 @@ require 'r10k/forge/module_release'
 
 require 'pathname'
 require 'fileutils'
-require 'puppet_forge'
+require 'puppet_forge/util'
 
 class R10K::Module::Forge < R10K::Module::Base
 
@@ -17,7 +17,7 @@ class R10K::Module::Forge < R10K::Module::Base
   end
 
   def self.valid_version?(expected_version)
-    expected_version == :latest || expected_version.nil? || SemanticPuppet::Version.valid?(expected_version)
+    expected_version == :latest || expected_version.nil? || PuppetForge::Util.version_valid?(expected_version)
   end
 
   # @!attribute [r] metadata

data/lib/r10k/puppetfile.rb

@@ -33,11 +33,18 @@ class Puppetfile
   #   @return [R10K::Environment] Optional R10K::Environment that this Puppetfile belongs to.
   attr_accessor :environment
 
+  # @!attribute [rw] force
+  #   @return [Boolean] Overwrite any locally made changes
+  attr_accessor :force
+
   # @param [String] basedir
   # @param [String] moduledir The directory to install the modules, default to #{basedir}/modules
   # @param [String] puppetfile_path The path to the Puppetfile, default to #{basedir}/Puppetfile
-  def initialize(basedir, moduledir = nil, puppetfile_path = nil)
+  # @param [String] puppetfile_name The name of the Puppetfile, default to 'Puppetfile'
+  # @param [Boolean] force Shall we overwrite locally made changes?
+  def initialize(basedir, moduledir = nil, puppetfile_path = nil, puppetfile_name = nil, force = nil )
     @basedir         = basedir
+    @force           = force || false
     @moduledir       = moduledir  || File.join(basedir, 'modules')
     @puppetfile_path = puppetfile_path || File.join(basedir, 'Puppetfile')
 

data/lib/r10k/settings.rb

@@ -51,6 +51,10 @@ module R10K
               :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
               :default => :inherit,
             }),
+            
+            Definition.new(:ignore_branch_prefixes, {
+              :desc => "Array of strings used to prefix branch names that will not be deployed as environments.",
+            }),
           ])
         },
         {

data/lib/r10k/source/git.rb

@@ -36,6 +36,11 @@ class R10K::Source::Git < R10K::Source::Base
   #     Puppet environments will be handled.
   attr_reader :invalid_branches
 
+  # @!attribute [r] ignore_branch_prefixes
+  #   @return [Array<String>] Array of strings used to remove repository branches
+  #     that will be deployed as environments.
+  attr_reader :ignore_branch_prefixes
+
   # Initialize the given source.
   #
   # @param name [String] The identifier for this source.
@@ -53,8 +58,9 @@ class R10K::Source::Git < R10K::Source::Base
 
     @environments = []
 
-    @remote           = options[:remote]
+    @remote = options[:remote]
     @invalid_branches = (options[:invalid_branches] || 'correct_and_warn')
+    @ignore_branch_prefixes = options[:ignore_branch_prefixes]
 
     @cache  = R10K::Git.cache.generate(@remote)
   end
@@ -109,11 +115,27 @@ class R10K::Source::Git < R10K::Source::Base
     environments.map {|env| env.dirname }
   end
 
+  def filter_branches(branches, ignore_prefixes)
+    filter = Regexp.new("^#{Regexp.union(ignore_prefixes)}")
+    branches = branches.reject do |branch|
+      result = filter.match(branch)
+      if result
+        logger.warn _("Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}") % {branch: branch, ibp: @ignore_branch_prefixes}
+      end
+      result
+    end
+    branches
+  end
+
   private
 
   def branch_names
     opts = {:prefix => @prefix, :invalid => @invalid_branches, :source => @name}
-    @cache.branches.map do |branch|
+    branches = @cache.branches
+    if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty?
+      branches = filter_branches(branches, @ignore_branch_prefixes)
+    end
+    branches.map do |branch|
       R10K::Environment::Name.new(branch, opts)
     end
   end

data/lib/r10k/source/svn.rb

@@ -37,6 +37,11 @@ class R10K::Source::SVN < R10K::Source::Base
   #   @api private
   attr_reader :password
 
+  # @!attribute [r] ignore_branch_prefixes
+  #   @return [Array<String>] Array of strings used to remove repository branches
+  #     that will be deployed as environments.
+  attr_reader :ignore_branch_prefixes
+
   include R10K::Util::Setopts
 
   # Initialize the given source.
@@ -56,6 +61,7 @@ class R10K::Source::SVN < R10K::Source::Base
     setopts(options, {:remote => :self, :username => :self, :password => :self})
     @environments = []
     @svn_remote = R10K::SVN::Remote.new(@remote, :username => @username, :password => @password)
+    @ignore_branch_prefixes = options[:ignore_branch_prefixes]
   end
 
   # Enumerate the environments associated with this SVN source.
@@ -97,18 +103,32 @@ class R10K::Source::SVN < R10K::Source::Base
 
   include R10K::Logging
 
+  def filter_branches(branches, ignore_prefixes)
+    filter = Regexp.new("^(#{ignore_prefixes.join('|')})")
+    branches = branches.reject do |branch|
+      result = filter.match(branch)
+      if result
+        logger.warn _("Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}") % {branch: branch, ibp: @ignore_branch_prefixes}
+      end
+      result
+    end
+    branches
+  end
+
   private
 
   def names_and_paths
     branches = []
-
     opts = {:prefix => @prefix, :correct => false, :validate => false, :source => @name}
-
     branches << [R10K::Environment::Name.new('production', opts), "#{@remote}/trunk"]
-    @svn_remote.branches.each do |branch|
-      branches << [R10K::Environment::Name.new(branch, opts), "#{@remote}/branches/#{branch}"]
+    additional_branch_names = @svn_remote.branches
+    if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty?
+      additional_branch_names = filter_branches(additional_branch_names, @ignore_branch_prefixes)
     end
 
+    additional_branch_names.each do |branch|
+      branches << [R10K::Environment::Name.new(branch, opts), "#{@remote}/branches/#{branch}"]
+    end
     branches
   end
 end

data/lib/r10k/util/setopts.rb

@@ -39,9 +39,11 @@ module R10K
             when NilClass, FalseClass
               # Ignore nil options
             when :self, TrueClass
-              instance_variable_set("@#{key}".to_sym, value)
+              # tr here is because instance variables cannot have hyphens in their names.
+              instance_variable_set("@#{key}".tr('-','_').to_sym, value)
             else
-              instance_variable_set("@#{rhs}".to_sym, value)
+              # tr here same as previous
+              instance_variable_set("@#{rhs}".tr('-','_').to_sym, value)
             end
           else
             raise ArgumentError, _("%{class_name} cannot handle option '%{key}'") % {class_name: self.class.name, key: key}

data/lib/r10k/version.rb

@@ -1,3 +1,3 @@
 module R10K
-  VERSION = '2.5.5'
+  VERSION = '2.6.0'
 end

data/r10k.gemspec

@@ -28,8 +28,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'log4r',     '1.1.10'
   s.add_dependency 'multi_json', '~> 1.10'
 
-  s.add_dependency 'puppet_forge', '~> 2.2'
-  s.add_dependency 'semantic_puppet', '~> 0.1.0'
+  s.add_dependency 'puppet_forge', '~> 2.2.8'
 
   s.add_dependency 'gettext-setup', '~> 0.5'
 

data/spec/shared-examples/puppetfile-action.rb

@@ -13,5 +13,27 @@ shared_examples_for "a puppetfile action" do
     it "accepts the :moduledir option" do
       described_class.new({moduledir: "/some/nonexistent/path/modules"}, [])
     end
+
+  end
+end
+
+shared_examples_for "a puppetfile install action" do
+  describe "initializing" do
+    it "accepts the :root option" do
+      described_class.new({root: "/some/nonexistent/path"}, [])
+    end
+
+    it "accepts the :puppetfile option" do
+      described_class.new({puppetfile: "/some/nonexistent/path/Puppetfile"}, [])
+    end
+
+    it "accepts the :moduledir option" do
+      described_class.new({moduledir: "/some/nonexistent/path/modules"}, [])
+    end
+
+    it "accepts the :force option" do
+      described_class.new({force: true}, [])
+    end
+
   end
 end

data/spec/unit/action/deploy/environment_spec.rb

@@ -19,6 +19,10 @@ describe R10K::Action::Deploy::Environment do
       described_class.new({puppetfile: true}, [])
     end
 
+    it "can accept a no-force option" do
+      described_class.new({:'no-force' => true}, [])
+    end
+
     it "normalizes environment names in the arg vector"
   end
 
@@ -53,6 +57,14 @@ describe R10K::Action::Deploy::Environment do
       end
     end
 
+    describe "with no-force" do
+      subject { described_class.new({ config: "/some/nonexistent/path", puppetfile: true, :'no-force' => true}, %w[first]) }
+
+      it "tries to preserve local modifications" do
+        expect(subject.force).to equal(false)
+      end
+    end
+
     describe "purge_levels" do
       let(:settings) { { deploy: { purge_levels: purge_levels } } }
 

data/spec/unit/action/deploy/module_spec.rb

@@ -13,5 +13,18 @@ describe R10K::Action::Deploy::Module do
     it "accepts an environment option" do
       described_class.new({environment: "production"}, [])
     end
+
+    it "can accept a no-force option" do
+      described_class.new({:'no-force' => true}, [])
+    end
+  end
+
+  describe "with no-force" do
+
+    subject { described_class.new({ config: "/some/nonexistent/path", :'no-force' => true}, [] )}
+
+    it "tries to preserve local modifications" do
+      expect(subject.force).to equal(false)
+    end
   end
 end

data/spec/unit/action/puppetfile/install_spec.rb

@@ -7,11 +7,12 @@ describe R10K::Action::Puppetfile::Install do
 
   let(:puppetfile) { R10K::Puppetfile.new('/some/nonexistent/path', nil, nil) }
 
-  before do
-    allow(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil).and_return(puppetfile)
+  before(:each) do
+    allow(puppetfile).to receive(:load!).and_return(nil)
+    allow(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil, nil, nil).and_return(puppetfile)
   end
 
-  it_behaves_like "a puppetfile action"
+  it_behaves_like "a puppetfile install action"
 
   describe "installing modules" do
     let(:modules) do
@@ -55,14 +56,27 @@ describe R10K::Action::Puppetfile::Install do
 
     it "can use a custom puppetfile path" do
       subject = described_class.new({root: "/some/nonexistent/path", puppetfile: "/some/other/path/Puppetfile"}, [])
-      expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, "/some/other/path/Puppetfile").and_return(puppetfile)
+      expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, "/some/other/path/Puppetfile", nil, nil).and_return(puppetfile)
       subject.call
     end
 
     it "can use a custom moduledir path" do
       subject = described_class.new({root: "/some/nonexistent/path", moduledir: "/some/other/path/site-modules"}, [])
-      expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", "/some/other/path/site-modules", nil).and_return(puppetfile)
+      expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", "/some/other/path/site-modules", nil, nil, nil).and_return(puppetfile)
       subject.call
     end
   end
+
+  describe "forcing to overwrite local changes" do
+    before do
+      allow(puppetfile).to receive(:modules).and_return([])
+    end
+
+    it "can use the force overwrite option" do
+      subject = described_class.new({root: "/some/nonexistent/path", force: true}, [])
+      expect(R10K::Puppetfile).to receive(:new).with("/some/nonexistent/path", nil, nil, nil, true).and_return(puppetfile)
+      subject.call
+    end
+
+  end
 end

data/spec/unit/forge/module_release_spec.rb

@@ -7,35 +7,105 @@ describe R10K::Forge::ModuleRelease do
   subject { described_class.new('branan-eight_hundred', '8.0.0') }
 
   let(:forge_release_class) { PuppetForge::V3::Release }
+  let(:md5_digest_class) { Digest::MD5 }
 
   let(:download_path) { instance_double('Pathname') }
+  let(:tarball_cache_path) { instance_double('Pathname') }
+  let(:tarball_cache_root) { instance_double('Pathname') }
   let(:unpack_path) { instance_double('Pathname') }
   let(:target_dir) { instance_double('Pathname') }
+  let(:md5_file_path) { instance_double('Pathname') }
+
   let(:file_lists) { {:valid=>['valid_ex'], :invalid=>[], :symlinks=>['symlink_ex']} }
 
+  let(:file_contents) { "skeletor's closet" }
+  let(:md5_of_tarball) { "something_hexy" }
+  let(:good_md5) { md5_of_tarball }
+  let(:bad_md5) { "different_hexy_thing" }
+
   before do
     subject.download_path = download_path
+    subject.tarball_cache_path = tarball_cache_path
+    subject.tarball_cache_root = tarball_cache_root
     subject.unpack_path = unpack_path
+    subject.md5_file_path = md5_file_path
+  end
+
+  context "no cached tarball" do
+    describe '#download' do
+      it "downloads the module from the forge into `download_path`" do
+        expect(tarball_cache_path).to receive(:exist?).and_return(false)
+        expect(subject.forge_release).to receive(:download).with(download_path)
+        allow(FileUtils).to receive(:mkdir_p).with(tarball_cache_root)
+        expect(FileUtils).to receive(:mv).with(download_path, tarball_cache_path)
+        subject.download
+      end
+    end
   end
 
-  describe '#download' do
-    it "downloads the module from the forge into `download_path`" do
-      expect(subject.forge_release).to receive(:download).with(download_path)
-      subject.download
+  context "with cached tarball" do
+    describe '#download' do
+      it "does not download a new tarball" do
+        expect(tarball_cache_path).to receive(:exist?).and_return(true)
+        expect(subject.forge_release).not_to receive(:download).with(download_path)
+        subject.download
+      end
     end
   end
 
   describe '#verify' do
-    it "verifies the module checksum based on the Forge file checksum" do
-      allow(subject.forge_release).to receive(:file_md5).and_return('something')
-      expect(subject.forge_release).to receive(:verify).with(download_path)
+
+    it "verifies using the file md5, if that exists" do
+      allow(File).to receive(:read).and_return(file_contents)
+      allow(md5_digest_class).to receive(:hexdigest).and_return(md5_of_tarball)
+      allow(md5_file_path).to receive(:exist?).and_return(true)
+      expect(subject).to receive(:verify_from_md5_file).with(md5_of_tarball)
       subject.verify
     end
+
+    it "verifies using the forge file_md5, if no md5 file exists" do
+      allow(File).to receive(:read).and_return(file_contents)
+      allow(md5_digest_class).to receive(:hexdigest).and_return(md5_of_tarball)
+      allow(md5_file_path).to receive(:exist?).and_return(false)
+      expect(subject).to receive(:verify_from_forge).with(md5_of_tarball)
+      subject.verify
+    end
+  end
+
+  describe '#verify_from_md5_file' do
+
+    it "does nothing when the checksums match" do
+      expect(File).to receive(:read).with(md5_file_path).and_return(good_md5)
+      expect(subject).not_to receive(:cleanup_cached_tarball_path)
+      subject.verify_from_md5_file(md5_of_tarball)
+    end
+
+    it "raises an error and cleans up when the checksums do not match" do
+      expect(File).to receive(:read).with(md5_file_path).and_return(bad_md5)
+      expect(subject).to receive(:cleanup_cached_tarball_path)
+      expect(subject).to receive(:cleanup_md5_file_path)
+      expect { subject.verify_from_md5_file(md5_of_tarball) }.to raise_error(PuppetForge::V3::Release::ChecksumMismatch)
+    end
+  end
+
+  describe '#verify_from_forge' do
+    it "write the md5 to file when the checksums match" do
+      expect(subject.forge_release).to receive(:file_md5).and_return(good_md5)
+      expect(subject).not_to receive(:cleanup_cached_tarball_path)
+      expect(File).to receive(:write).with(md5_file_path, good_md5)
+      subject.verify_from_forge(md5_of_tarball)
+    end
+
+    it "raises an error and cleans up when the checksums do not match" do
+      expect(subject.forge_release).to receive(:file_md5).and_return(bad_md5)
+      expect(subject).to receive(:cleanup_cached_tarball_path)
+      expect { subject.verify_from_forge(md5_of_tarball) }.to raise_error(PuppetForge::V3::Release::ChecksumMismatch)
+    end
   end
 
   describe '#unpack' do
-    it "unpacks the module tarball in `download_path` into the provided target path" do
-      expect(PuppetForge::Unpacker).to receive(:unpack).with(download_path.to_s, target_dir.to_s, unpack_path.to_s).\
+    it "unpacks the module tarball in `tarball_cache_path` into the provided target path" do
+      expect(PuppetForge::Unpacker).to receive(:unpack).with(tarball_cache_path.to_s, target_dir.to_s, unpack_path.to_s).\
           and_return({:valid=>["extractedmodule/metadata.json"], :invalid=>[], :symlinks=>[]})
       subject.unpack(target_dir)
     end
@@ -52,7 +122,7 @@ describe R10K::Forge::ModuleRelease do
   end
 
   describe "#cleanup" do
-    it "cleans up the download and unpack paths" do
+    it "cleans up the unpack paths" do
       expect(subject).to receive(:cleanup_unpack_path)
       expect(subject).to receive(:cleanup_download_path)
       subject.cleanup

data/spec/unit/source/git_spec.rb

@@ -62,6 +62,40 @@ describe R10K::Source::Git do
       expect(master_env.dirname).to eq 'master'
     end
   end
+
+  describe "generate_environments respects ignore_branch_prefixes setting" do
+    before do
+      allow(subject.cache).to receive(:branches).and_return ['master', 'development', 'production', 'not_dev_test_me', 'dev_test', 'dev', 'test_2']
+      subject.instance_variable_set(:@ignore_branch_prefixes, ['dev', 'test'])
+    end
+
+    let(:environments) { subject.generate_environments }
+
+    it "creates an environment for each branch not in ignore_branch_prefixes" do
+      expect(subject.generate_environments.size).to eq(3)
+    end
+
+    it "copies the source remote to the environment" do
+      expect(environments[0].remote).to eq subject.remote
+      expect(environments[1].remote).to eq subject.remote
+      expect(environments[2].remote).to eq subject.remote
+    end
+
+    it "uses the branch name as the directory by default" do
+      expect(environments[0].dirname).to eq 'master'
+      expect(environments[1].dirname).to eq 'production'
+      expect(environments[2].dirname).to eq 'not_dev_test_me'
+    end
+  end
+
+  describe "filtering branches with ignore prefixes" do
+    let(:branches) { ['master', 'development', 'production', 'not_dev_test_me', 'dev_test', 'dev', 'test_2'] }
+    let(:ignore_prefixes) { ['dev', 'test'] }
+
+    it "filters branches" do
+      expect(subject.filter_branches(branches, ignore_prefixes)).to eq(['master', 'production', 'not_dev_test_me'])
+    end
+  end
 end
 
 describe R10K::Source::Git, "handling invalid branch names" do

data/spec/unit/source/svn_spec.rb

@@ -61,6 +61,34 @@ describe R10K::Source::SVN do
       expect(environments[3].dirname).to eq 'robobutler'
     end
   end
+
+  describe "generate_environments respects ignore_branch_prefixes setting" do
+    before do
+      allow(subject.svn_remote).to receive(:branches).and_return ['master', 'development', 'not_dev_test_me', 'dev_test', 'dev', 'test_2']
+      subject.instance_variable_set(:@ignore_branch_prefixes, ['dev', 'test'])
+    end
+
+    let(:environments) { subject.generate_environments }
+
+    it "creates an environment for each branch not in ignore_branch_prefixes" do
+      expect(subject.generate_environments.size).to eq(3)
+    end
+
+    it "uses the branch name as the directory by default" do
+      expect(environments[0].name).to eq 'production'
+      expect(environments[1].name).to eq 'master'
+      expect(environments[2].name).to eq 'not_dev_test_me'
+    end
+  end
+
+  describe "filtering branches with ignore prefixes" do
+    let(:branches) { ['master', 'development', 'production', 'not_dev_test_me', 'dev_test', 'dev', 'test_2'] }
+    let(:ignore_prefixes) { ['dev', 'test'] }
+
+    it "filters branches" do
+      expect(subject.filter_branches(branches, ignore_prefixes)).to eq(['master', 'production', 'not_dev_test_me'])
+    end
+  end
 end
 
 describe R10K::Source::SVN, 'when prefixing is enabled' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: r10k
 version: !ruby/object:Gem::Version
-  version: 2.5.5
+  version: 2.6.0
 platform: ruby
 authors:
 - Adrien Thebo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-02 00:00:00.000000000 Z
+date: 2017-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colored
@@ -28,14 +28,14 @@ dependencies:
   name: cri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 2.6.1
 - !ruby/object:Gem::Dependency
@@ -56,126 +56,111 @@ dependencies:
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.10'
 - !ruby/object:Gem::Dependency
   name: puppet_forge
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: 2.2.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '2.2'
-- !ruby/object:Gem::Dependency
-  name: semantic_puppet
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.1.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 2.2.8
 - !ruby/object:Gem::Dependency
   name: gettext-setup
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '0.5'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '3.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.8.7.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.8.7.3
 - !ruby/object:Gem::Dependency
   name: minitar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: 0.6.1
-description: |2
-      R10K provides a general purpose toolset for deploying Puppet environments and modules.
-      It implements the Puppetfile format and provides a native implementation of Puppet
-      dynamic environments.
+description: ! "    R10K provides a general purpose toolset for deploying Puppet environments
+  and modules.\n    It implements the Puppetfile format and provides a native implementation
+  of Puppet\n    dynamic environments.\n"
 email: adrien@somethingsinistral.net
 executables:
 - r10k
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".travis.yml"
+- .gitignore
+- .travis.yml
 - CHANGELOG.mkd
 - CONTRIBUTING.mkd
 - Gemfile
@@ -515,18 +500,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Puppet environment and module deployment
 test_files: []
+has_rdoc: