r10k 3.5.2 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2fbe395273c7e4fa08a3cb6fa552c05d4745c6c3f8add19b7fe7bb63c03b2d4
-  data.tar.gz: 7e59bb47f25fcac554075de51a4b37d55de612cdb43079a1212adf4d3c84707a
+  metadata.gz: 4e54158e0a651c799246229971ba62ad36bd40a670ca84e61479aa5280306c8a
+  data.tar.gz: 153cc105ce9276d79e2501dc99accb398bba8fec423fbf0726a1b183993a2748
 SHA512:
-  metadata.gz: 7925d5b8d45bcb502ce11a4d190048f3bcf03310ee42a9128a5ff66e72425a48269a35fba1177905a895175a096b45441fb2f0090e174ecd2bbd86217d5ed641
-  data.tar.gz: 796889d0936244967388973e0903e9011cc11b11a8fcb6b91bc083f13e8fb4e1400b96f79337f778576d04c917e3cc170b1094e124f3a05a8174b1de82d28824
+  metadata.gz: 264ea2591c062d460edab02b8edf7a59c9ebc08ebea27ee2b103e43bdf71c936b797094014d66b776573196d4a9e86dfc9f0fd345626f27d7e25c8170ebaa83a
+  data.tar.gz: 224e40f3f13466ab2ea3296bc3741b10d591ea65c97ba60b3dd81078f70149770c07c46ac1464a44fd59af0d20dbc8bbf1a5e3990e5d7921cc05403772d99640

data/.github/pull_request_template.md

@@ -1 +1,4 @@
-Please add all notable changes to the "Unreleased" section of the CHANGELOG.
+Please add all notable changes to the "Unreleased" section of the CHANGELOG in the format:
+```
+- (JIRA ticket) Summary of changes. [Issue or PR #](link to issue or PR)
+```

data/CHANGELOG.mkd

@@ -2,7 +2,15 @@ CHANGELOG
 =========
 
 Unreleased
-----
+----------
+
+3.6.0
+-----
+
+- Add filter_command configuration option for git repositories. (Thanks to [mhumpula](https://github.com/mhumpula) for the feature.) [#823](https://github.com/puppetlabs/r10k/pull/823)
+- Increase default pool_size to 4, allowing modules to be downloaded on 4 threads concurrently. [#1038](https://github.com/puppetlabs/r10k/issues/1038)
+- Ensure that modules that share a cachedir download serially, to avoid cache corruption. [#1058](https://github.com/puppetlabs/r10k/issues/1058)
+- Don't purge root when using `puppetfile install`. [#1084](https://github.com/puppetlabs/r10k/issues/1084)
 
 3.5.2
 -----
@@ -175,6 +183,11 @@ Ruby >= 2.3 official and documented.
     - Unable to parse Puppetfile
   - Various perfomance improvements
 
+2.6.9
+----
+
+- Don't purge root when using `puppetfile install`. [#1084](https://github.com/puppetlabs/r10k/issues/1084)
+
 2.6.8
 ----
 

data/README.mkd

@@ -98,6 +98,14 @@ When localizing the strings found in R10k, follow the prescribed
 The workflow describes the rake tasks provided to generate the .po files for
 each locale.
 
+Releasing
+---------
+To release a new version of the r10k gem, ensure the [changelog](CHANGELOG.mkd) is up to date and open a pull request updating the [version file](lib/r10k/version.rb). When the PR is merged, a new release of the gem will be triggered.
+
+By default, a patch (Z) release will be triggered. To release a new major (X) or minor (Y) version, include `#major` or `#minor` (respectively) in your commit message to trigger the appropriate release.
+
+NOTE: This currently only works for the default branch. If you would like to release from a different branch, please contact the [CODEOWNERS](CODEOWNERS).
+
 Getting help
 ------------
 
@@ -113,6 +121,7 @@ Please see the CHANGELOG for a listing of the (very awesome) contributors.
 
 ## Maintenance
 
-See [MAINTAINERS](MAINTAINERS) (json file)
+See [CODEOWNERS](CODEOWNERS) for active repo maintainers.
+
+Open [issues](https://github.com/puppetlabs/r10k/issues) directly in the r10k repo.
 
-Tickets: File at https://tickets.puppet.com/browse/RK

data/azure-pipelines.yml

@@ -19,6 +19,7 @@ variables:
   CONTAINER_BUILD_PATH: .
   LINT_IGNORES:
   DOCKER_BUILDKIT: 1
+  BUILD_OPTIONS: --build-arg alpine_version=3.9
 
 workspace:
   clean: resources
@@ -53,7 +54,7 @@ steps:
 
 - powershell: |
     . "$(bundle show pupperware)/ci/build.ps1"
-    Build-Container -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE -PathOrUri $ENV:CONTAINER_BUILD_PATH
+    Build-Container -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE -PathOrUri $ENV:CONTAINER_BUILD_PATH -AdditionalOptions ($ENV:BUILD_OPTIONS -split ' ')
   displayName: Build $(CONTAINER_NAME) Container
   timeoutInMinutes: 5
   name: build_container

data/doc/dynamic-environments/configuration.mkd

@@ -76,9 +76,9 @@ The proxy server being used will be logged at the "debug" level when r10k runs.
 ### pool_size
 
 The pool_size setting is a number to determine how many threads should be spawn
-while updating modules. The default value is 1, which means the default behaviour
-is to update modules in a serial manner. Increasing this number should bring
-some performance gains.
+while updating modules. The default value is 4, which means modules will be updated
+in parallel. If this causes issues, change this setting to 1 to cause modules to be
+updated serially.
 
 ### git
 
@@ -411,6 +411,29 @@ sources:
       - 'dev'
 ```
 
+### filter_command
+
+You can filter out any branch based on the result of the command specified as
+'filter_command'. Currently it only works with git repository. Non zero return
+status of the command results in a branch beeing removed. The command is passed
+additional environment variables
+
+* GIT_DIR – path to the cached git repository
+* R10K_BRANCH – branch which is being filtered
+* R10K_NAME – source name from r10k configuration
+
+This can be used for example for filtering out the branches with invalid gpg signature of their latest commit
+
+```yaml
+---
+sources:
+  mysource:
+    basedir: '/etc/puppet/environments'
+    filter_command: 'git verify-commit $R10K_BRANCH 2> /dev/null'
+```
+
+Beware that if the production branch of manifests is filtered out, you will end up with empty environment.
+
 Examples
 --------
 

data/docker/Makefile

@@ -6,6 +6,7 @@ build_date := $(shell date -u +%FT%T)
 hadolint_available := $(shell hadolint --help > /dev/null 2>&1; echo $$?)
 hadolint_command := hadolint
 hadolint_container := hadolint/hadolint:latest
+alpine_version := 3.9
 export BUNDLE_PATH = $(PWD)/.bundle/gems
 export BUNDLE_BIN = $(PWD)/.bundle/bin
 export GEMFILE = $(PWD)/Gemfile
@@ -48,9 +49,10 @@ else
 endif
 
 build: prep
+	docker pull alpine:$(alpine_version)
 	docker build \
 		${DOCKER_BUILD_FLAGS} \
-		--pull \
+		--build-arg alpine_version=$(alpine_version) \
 		--build-arg vcs_ref=$(vcs_ref) \
 		--build-arg build_date=$(build_date) \
 		--build-arg version=$(VERSION) \

data/docker/r10k/Dockerfile

@@ -1,4 +1,5 @@
-FROM alpine:3.9 as build
+ARG alpine_version=3.9
+FROM alpine:${alpine_version} as build
 
 # hadolint ignore=DL3018
 RUN apk add --no-cache ruby git && \
@@ -8,7 +9,7 @@ COPY . /workspace
 RUN gem build r10k.gemspec && \
     mv r10k*.gem r10k.gem
 
-FROM alpine:3.9
+FROM alpine:${alpine_version}
 
 ARG vcs_ref
 ARG build_date

data/docker/r10k/release.Dockerfile

@@ -1,4 +1,5 @@
-FROM alpine:3.9
+ARG alpine_version=3.9
+FROM alpine:${alpine_version}
 
 ARG vcs_ref
 ARG build_date

data/integration/tests/git_source/git_source_repeated_remote.rb

@@ -0,0 +1,68 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+test_name 'Verify the same remote can be used in more than one object'
+
+env_path = on(master, puppet('config print environmentpath')).stdout.rstrip
+r10k_fqp = get_r10k_fqp(master)
+git_environments_path = '/root/environments'
+git_repo_path = '/git_repos'
+git_repo_name = 'environments'
+git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git")
+code_dir = "#{env_path}/production"
+
+last_commit = git_last_commit(master, git_environments_path)
+git_provider = ENV['GIT_PROVIDER']
+r10k_config_path = get_r10k_config_file_path(master)
+r10k_config_bak_path = "#{r10k_config_path}.bak"
+#In-line files
+r10k_conf = <<-CONF
+cachedir: '/var/cache/r10k'
+git:
+  provider: '#{git_provider}'
+sources:
+  control:
+    basedir: "#{env_path}"
+    remote: "#{git_control_remote}"
+CONF
+
+# Install the same module in two different places
+puppetfile = <<-EOS
+mod 'prod_apache',
+  :git => 'git://github.com/puppetlabs/puppetlabs-apache.git',
+  :branch => 'master'
+
+mod 'test_apache',
+  :git => 'git://github.com/puppetlabs/puppetlabs-apache.git',
+  :branch => 'master'
+EOS
+
+teardown do
+  step 'Restore Original "r10k" Config'
+  on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}")
+
+  clean_up_r10k(master, last_commit, git_environments_path)
+end
+
+step 'Stub the forge'
+stub_forge_on(master)
+
+step 'Backup Current "r10k" Config'
+on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}")
+
+step 'Update the "r10k" Config'
+create_remote_file(master, r10k_config_path, r10k_conf)
+
+step 'Ask r10k to deploy'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Add puppetfile with repeated remote'
+create_remote_file(master, "#{git_environments_path}/Puppetfile", puppetfile)
+git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path)
+
+step 'Deploy r10k'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Verify module was installed in both places'
+on(master, "test -d #{code_dir}/modules/prod_apache")
+on(master, "test -d #{code_dir}/modules/test_apache")

data/lib/r10k/git/cache.rb

@@ -99,10 +99,8 @@ class R10K::Git::Cache
 
   alias cached? exist?
 
-  private
-
   # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @remote.gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-')
   end
 end

data/lib/r10k/git/stateful_repository.rb

@@ -12,6 +12,10 @@ class R10K::Git::StatefulRepository
   #   @api private
   attr_reader :repo
 
+  # @!attribute [r] cache
+  #   @api private
+  attr_reader :cache
+
   extend Forwardable
   def_delegators :@repo, :head, :tracked_paths
 

data/lib/r10k/module/base.rb

@@ -99,6 +99,14 @@ class R10K::Module::Base
     raise NotImplementedError
   end
 
+  # Return the module's cachedir. Subclasses that implement a cache
+  # will override this to return a real directory location.
+  #
+  # @return [String, :none]
+  def cachedir
+    :none
+  end
+
   private
 
   def parse_title(title)

data/lib/r10k/module/git.rb

@@ -57,6 +57,10 @@ class R10K::Module::Git < R10K::Module::Base
     @repo.status(version)
   end
 
+  def cachedir
+    @repo.cache.sanitized_dirname
+  end
+
   private
 
   def validate_ref(desired, default)

data/lib/r10k/puppetfile.rb

@@ -10,7 +10,7 @@ class Puppetfile
 
   include R10K::Settings::Mixin
 
-  def_setting_attr :pool_size, 1
+  def_setting_attr :pool_size, 4
 
   include R10K::Logging
 
@@ -42,6 +42,11 @@ class Puppetfile
   #   @return [Boolean] Overwrite any locally made changes
   attr_accessor :force
 
+  # @!attribute [r] modules_by_vcs_cachedir
+  #   @api private Only exposed for testing purposes
+  #   @return [Hash{:none, String => Array<R10K::Module>}]
+  attr_reader :modules_by_vcs_cachedir
+
   # @param [String] basedir
   # @param [String] moduledir The directory to install the modules, default to #{basedir}/modules
   # @param [String] puppetfile_path The path to the Puppetfile, default to #{basedir}/Puppetfile
@@ -58,6 +63,7 @@ class Puppetfile
 
     @modules = []
     @managed_content = {}
+    @modules_by_vcs_cachedir = {}
     @forge   = 'forgeapi.puppetlabs.com'
 
     @loaded = false
@@ -137,6 +143,9 @@ class Puppetfile
     mod.origin = 'Puppetfile'
 
     @managed_content[install_path] << mod.name
+    cachedir = mod.cachedir
+    @modules_by_vcs_cachedir[cachedir] ||= []
+    @modules_by_vcs_cachedir[cachedir] << mod
     @modules << mod
   end
 
@@ -145,7 +154,9 @@ class Puppetfile
   def managed_directories
     self.load unless @loaded
 
-    @managed_content.keys
+    dirs = @managed_content.keys
+    dirs.delete(real_basedir)
+    dirs
   end
 
   # Returns an array of the full paths to all the content being managed.
@@ -212,15 +223,22 @@ class Puppetfile
   def modules_queue(visitor)
     Queue.new.tap do |queue|
       visitor.visit(:puppetfile, self) do
-        modules.each { |mod| queue << mod }
+        modules_by_cachedir = modules_by_vcs_cachedir.clone
+        modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
+
+        modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
+        modules_by_cachedir.values.each {|mods| queue << mods }
       end
     end
   end
+  public :modules_queue
 
   def visitor_thread(visitor, mods_queue)
     Thread.new do
       begin
-        while mod = mods_queue.pop(true) do mod.accept(visitor) end
+        while mods = mods_queue.pop(true) do
+          mods.each {|mod| mod.accept(visitor) }
+        end
       rescue ThreadError => e
         logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
         Thread.exit
@@ -248,8 +266,6 @@ class Puppetfile
   end
 
   def validate_install_path(path, modname)
-    real_basedir = Pathname.new(basedir).cleanpath.to_s
-
     unless /^#{Regexp.escape(real_basedir)}.*/ =~ path
       raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}")
     end
@@ -257,6 +273,10 @@ class Puppetfile
     true
   end
 
+  def real_basedir
+    Pathname.new(basedir).cleanpath.to_s
+  end
+
   class DSL
     # A barebones implementation of the Puppetfile DSL
     #

data/lib/r10k/settings.rb

@@ -160,7 +160,7 @@ module R10K
 
         Definition.new(:pool_size, {
           :desc => "The amount of threads used to concurrently install modules. The default value is 1: install one module at a time.",
-          :default => 1,
+          :default => 4,
           :validate => lambda do |value|
             if !value.is_a?(Integer)
               raise ArgumentError, "The pool_size setting should be an integer, not a #{value.class}"

data/lib/r10k/source/git.rb

@@ -41,6 +41,10 @@ class R10K::Source::Git < R10K::Source::Base
   #     that will be deployed as environments.
   attr_reader :ignore_branch_prefixes
 
+  # @!attribute [r] filter_command
+  #   @return [String] Command to run to filter branches
+  attr_reader :filter_command
+
   # Initialize the given source.
   #
   # @param name [String] The identifier for this source.
@@ -61,6 +65,7 @@ class R10K::Source::Git < R10K::Source::Base
     @remote = options[:remote]
     @invalid_branches = (options[:invalid_branches] || 'correct_and_warn')
     @ignore_branch_prefixes = options[:ignore_branch_prefixes]
+    @filter_command = options[:filter_command]
 
     @cache  = R10K::Git.cache.generate(@remote)
   end
@@ -115,7 +120,7 @@ class R10K::Source::Git < R10K::Source::Base
     environments.map {|env| env.dirname }
   end
 
-  def filter_branches(branches, ignore_prefixes)
+  def filter_branches_by_regexp(branches, ignore_prefixes)
     filter = Regexp.new("^#{Regexp.union(ignore_prefixes)}")
     branches = branches.reject do |branch|
       result = filter.match(branch)
@@ -127,14 +132,29 @@ class R10K::Source::Git < R10K::Source::Base
     branches
   end
 
+  def filter_branches_by_command(branches, command)
+    branches.select do |branch|
+      result = system({'GIT_DIR' => @cache.git_dir.to_s, 'R10K_BRANCH' => branch, 'R10K_NAME' => @name.to_s}, command)
+      unless result
+        logger.warn _("Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}") % {name: @name, branch: branch, cmd: command}
+      end
+      result
+    end
+  end
+
   private
 
   def branch_names
     opts = {:prefix => @prefix, :invalid => @invalid_branches, :source => @name}
     branches = @cache.branches
     if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty?
-      branches = filter_branches(branches, @ignore_branch_prefixes)
+      branches = filter_branches_by_regexp(branches, @ignore_branch_prefixes)
+    end
+
+    if @filter_command && !@filter_command.empty?
+      branches = filter_branches_by_command(branches, @filter_command)
     end
+
     branches.map do |branch|
       R10K::Environment::Name.new(branch, opts)
     end

data/lib/r10k/version.rb

@@ -2,5 +2,5 @@ module R10K
   # When updating to a new major (X) or minor (Y) version, include `#major` or
   # `#minor` (respectively) in your commit message to trigger the appropriate
   # release. Otherwise, a new patch (Z) version will be released.
-  VERSION = '3.5.2'
+  VERSION = '3.6.0'
 end

data/locales/r10k.pot

@@ -6,11 +6,11 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: r10k 3.4.1-20-g8fe925b\n"
+"Project-Id-Version: r10k 3.4.1-57-g2eb088a\n"
 "\n"
 "Report-Msgid-Bugs-To: docs@puppetlabs.com\n"
-"POT-Creation-Date: 2020-04-28 18:41+0000\n"
-"PO-Revision-Date: 2020-04-28 18:41+0000\n"
+"POT-Creation-Date: 2020-07-22 16:41+0000\n"
+"PO-Revision-Date: 2020-07-22 16:41+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -442,26 +442,30 @@ msgid ""
 "Returned: %{data}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:72
+#: ../lib/r10k/source/git.rb:77
 msgid "Fetching '%{remote}' to determine current branches."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:75
+#: ../lib/r10k/source/git.rb:80
 msgid "Unable to determine current branches for Git source '%{name}' (%{basedir})"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:100
+#: ../lib/r10k/source/git.rb:105
 msgid "Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:104
+#: ../lib/r10k/source/git.rb:109
 msgid "Environment %{env_name} contained non-word characters, ignoring it."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:123 ../lib/r10k/source/svn.rb:113
+#: ../lib/r10k/source/git.rb:128 ../lib/r10k/source/svn.rb:113
 msgid "Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}"
 msgstr ""
 
+#: ../lib/r10k/source/git.rb:139
+msgid "Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}"
+msgstr ""
+
 #: ../lib/r10k/source/yaml.rb:10
 msgid "Couldn't open environments file %{file}: %{err}"
 msgstr ""

data/spec/unit/action/deploy/module_spec.rb

@@ -70,8 +70,8 @@ describe R10K::Action::Deploy::Module do
 
       before do
         allow(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
-          expect(environment.puppetfile).to receive(:modules).and_return(
-            [R10K::Module::Local.new(environment.name, '/fakedir', [], environment)]
+          expect(environment.puppetfile).to receive(:modules_by_vcs_cachedir).and_return(
+            {none: [R10K::Module::Local.new(environment.name, '/fakedir', [], environment)]}
           )
           original.call(environment, &block)
         end

data/spec/unit/action/puppetfile/install_spec.rb

@@ -27,6 +27,7 @@ describe R10K::Action::Puppetfile::Install do
     before do
       allow(puppetfile).to receive(:purge!)
       allow(puppetfile).to receive(:modules).and_return(modules)
+      allow(puppetfile).to receive(:modules_by_vcs_cachedir).and_return({none: modules})
     end
 
     it "syncs each module in the Puppetfile" do

data/spec/unit/puppetfile_spec.rb

@@ -127,6 +127,25 @@ describe R10K::Puppetfile do
 
       expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules }
     end
+
+    it "groups modules by vcs cache location" do
+      module_opts = { install_path: File.join(subject.basedir, 'vendor') }
+      opts1 = module_opts.merge(git: 'git@example.com:puppet/test_module.git')
+      opts2 = module_opts.merge(git: 'git@example.com:puppet/test_module_c.git')
+      sanitized_name1 = "git@example.com-puppet-test_module.git"
+      sanitized_name2 = "git@example.com-puppet-test_module_c.git"
+
+      subject.add_module('puppet/test_module_a', opts1)
+      subject.add_module('puppet/test_module_b', opts1)
+      subject.add_module('puppet/test_module_c', opts2)
+      subject.add_module('puppet/test_module_d', '1.2.3')
+
+      mods_by_cachedir = subject.modules_by_vcs_cachedir
+
+      expect(mods_by_cachedir[:none].length).to be 1
+      expect(mods_by_cachedir[sanitized_name1].length).to be 2
+      expect(mods_by_cachedir[sanitized_name2].length).to be 1
+    end
   end
 
   describe "#purge_exclusions" do
@@ -154,6 +173,26 @@ describe R10K::Puppetfile do
     end
   end
 
+  describe '#managed_directories' do
+    it 'returns an array of paths that can be purged' do
+      allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, '1.2.3', anything).and_call_original
+
+      subject.add_module('puppet/test_module', '1.2.3')
+      expect(subject.managed_directories).to match_array(["/some/nonexistent/basedir/modules"])
+    end
+
+    context 'with a module with install_path == \'\'' do
+      it 'basedir isn\'t in the list of paths to purge' do
+        module_opts = { install_path: '', git: 'git@example.com:puppet/test_module.git' }
+
+        allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.basedir, module_opts, anything).and_call_original
+
+        subject.add_module('puppet/test_module', module_opts)
+        expect(subject.managed_directories).to be_empty
+      end
+    end
+  end
+
   describe "evaluating a Puppetfile" do
     def expect_wrapped_error(orig, pf_path, wrapped_error)
       expect(orig).to be_a_kind_of(R10K::Error)
@@ -268,7 +307,7 @@ describe R10K::Puppetfile do
       mod2 = spy('module')
       expect(mod2).to receive(:accept).with(visitor)
 
-      expect(subject).to receive(:modules).and_return([mod1, mod2])
+      expect(subject).to receive(:modules_by_vcs_cachedir).and_return({none: [mod1, mod2]})
       subject.accept(visitor)
     end
 
@@ -289,12 +328,38 @@ describe R10K::Puppetfile do
       mod2 = spy('module')
       expect(mod2).to receive(:accept).with(visitor)
 
-      expect(subject).to receive(:modules).and_return([mod1, mod2])
+      expect(subject).to receive(:modules_by_vcs_cachedir).and_return({none: [mod1, mod2]})
 
       expect(Thread).to receive(:new).exactly(pool_size).and_call_original
       expect(Queue).to receive(:new).and_call_original
 
       subject.accept(visitor)
     end
+
+    it "Creates queues of modules grouped by cachedir" do
+      visitor = spy('visitor')
+      expect(visitor).to receive(:visit) do |type, other, &block|
+        expect(type).to eq :puppetfile
+        expect(other).to eq subject
+        block.call
+      end
+
+      mod1 = spy('module1')
+      mod2 = spy('module2')
+      mod3 = spy('module3')
+      mod4 = spy('module4')
+      mod5 = spy('module5')
+      mod6 = spy('module6')
+
+      expect(subject).to receive(:modules_by_vcs_cachedir)
+        .and_return({:none => [mod1, mod2],
+                     "foo-cachedir" => [mod3, mod4],
+                     "bar-cachedir" => [mod5, mod6]})
+
+      queue = subject.modules_queue(visitor)
+      expect(queue.length).to be 4
+      queue_array = 4.times.map { queue.pop }
+      expect(queue_array).to match_array([[mod1], [mod2], [mod3, mod4], [mod5, mod6]])
+    end
   end
 end

data/spec/unit/source/git_spec.rb

@@ -93,9 +93,45 @@ describe R10K::Source::Git do
     let(:ignore_prefixes) { ['dev', 'test'] }
 
     it "filters branches" do
-      expect(subject.filter_branches(branches, ignore_prefixes)).to eq(['master', 'production', 'not_dev_test_me'])
+      expect(subject.filter_branches_by_regexp(branches, ignore_prefixes)).to eq(['master', 'production', 'not_dev_test_me'])
     end
   end
+
+  describe "filtering branches with command" do
+    let(:branches) { ['master', 'development', 'production'] }
+    let(:filter_command) { 'sh -c "[ $R10K_BRANCH != development ]"' }
+
+    it "filters branches" do
+      expect(subject.filter_branches_by_command(branches, filter_command)).to eq(['master', 'production'])
+    end
+  end
+
+  describe "generate_environments respects filter_command setting" do
+    before do
+      allow(subject.cache).to receive(:branches).and_return ['master', 'development', 'production']
+      subject.instance_variable_set(:@filter_command, '[ $R10K_BRANCH != master ]')
+    end
+
+    let(:environments) { subject.generate_environments }
+
+    it "creates an environment for each branch not filtered by filter_command" do
+      expect(subject.generate_environments.size).to eq(2)
+    end
+  end
+
+  describe "generate_environments respects filter_command setting and name" do
+    before do
+      allow(subject.cache).to receive(:branches).and_return ['master', 'development', 'production']
+      subject.instance_variable_set(:@filter_command, '[ $R10K_NAME = mysource ]')
+    end
+
+    let(:environments) { subject.generate_environments }
+
+    it "creates an environment for each branch not filtered by filter_command" do
+      expect(subject.generate_environments.size).to eq(3)
+    end
+  end
+
 end
 
 describe R10K::Source::Git, "handling invalid branch names" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: r10k
 version: !ruby/object:Gem::Version
-  version: 3.5.2
+  version: 3.6.0
 platform: ruby
 authors:
 - Adrien Thebo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-21 00:00:00.000000000 Z
+date: 2020-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colored2
@@ -282,6 +282,7 @@ files:
 - integration/tests/command_line/negative/neg_invalid_cmd_line_arg.rb
 - integration/tests/git_source/HTTP_proxy_and_git_source.rb
 - integration/tests/git_source/git_source_git.rb
+- integration/tests/git_source/git_source_repeated_remote.rb
 - integration/tests/git_source/git_source_ssh.rb
 - integration/tests/git_source/git_source_submodule.rb
 - integration/tests/git_source/negative/neg_git_broken_remote.rb