r10k 3.4.0 → 3.6.0

data/lib/r10k/settings.rb

@@ -160,7 +160,7 @@ module R10K
 
         Definition.new(:pool_size, {
           :desc => "The amount of threads used to concurrently install modules. The default value is 1: install one module at a time.",
-          :default => 1,
+          :default => 4,
           :validate => lambda do |value|
             if !value.is_a?(Integer)
               raise ArgumentError, "The pool_size setting should be an integer, not a #{value.class}"

data/lib/r10k/source.rb

@@ -37,5 +37,6 @@ module R10K
     require 'r10k/source/svn'
     require 'r10k/source/yaml'
     require 'r10k/source/yamldir'
+    require 'r10k/source/exec'
   end
 end

data/lib/r10k/source/exec.rb

@@ -0,0 +1,51 @@
+require 'r10k/util/subprocess'
+require 'json'
+require 'yaml'
+
+class R10K::Source::Exec < R10K::Source::Hash
+  R10K::Source.register(:exec, self)
+
+  def initialize(name, basedir, options = {})
+    unless @command = options[:command]
+      raise ConfigError, _('Environment source %{name} missing required parameter: command') % {name: name}
+    end
+
+    # We haven't set the environments option yet. We will do that by
+    # overloading the #environments method.
+    super(name, basedir, options)
+  end
+
+  def environments_hash
+    @environments_hash ||= set_environments_hash(run_environments_command)
+  end
+
+  private
+
+  def run_environments_command
+    subproc = R10K::Util::Subprocess.new([@command])
+    subproc.raise_on_fail = true
+    subproc.logger = self.logger
+    procresult = subproc.execute
+
+    begin
+      environments = JSON.parse(procresult.stdout)
+    rescue JSON::ParserError => json_err
+      begin
+        environments = YAML.safe_load(procresult.stdout)
+      rescue Psych::SyntaxError => yaml_err
+        raise R10K::Error, _("Error parsing command output for exec source %{name}:\n" \
+                             "Not valid JSON: %{j_msg}\n" \
+                             "Not valid YAML: %{y_msg}\n" \
+                             "Stdout:\n%{out}") % {name: name, j_msg: json_err.message, y_msg: yaml_err.message, out: procresult.stdout}
+      end
+    end
+
+    unless R10K::Source::Hash.valid_environments_hash?(environments)
+      raise R10K::Error, _("Environment source %{name} command %{cmd} did not return valid environment data.\n" \
+                           'Returned: %{data}') % {name: name, cmd: @command, data: environments}
+    end
+
+    # Return the resulting environments hash
+    environments
+  end
+end

data/lib/r10k/source/git.rb

@@ -41,6 +41,10 @@ class R10K::Source::Git < R10K::Source::Base
   #     that will be deployed as environments.
   attr_reader :ignore_branch_prefixes
 
+  # @!attribute [r] filter_command
+  #   @return [String] Command to run to filter branches
+  attr_reader :filter_command
+
   # Initialize the given source.
   #
   # @param name [String] The identifier for this source.
@@ -61,6 +65,7 @@ class R10K::Source::Git < R10K::Source::Base
     @remote = options[:remote]
     @invalid_branches = (options[:invalid_branches] || 'correct_and_warn')
     @ignore_branch_prefixes = options[:ignore_branch_prefixes]
+    @filter_command = options[:filter_command]
 
     @cache  = R10K::Git.cache.generate(@remote)
   end
@@ -115,7 +120,7 @@ class R10K::Source::Git < R10K::Source::Base
     environments.map {|env| env.dirname }
   end
 
-  def filter_branches(branches, ignore_prefixes)
+  def filter_branches_by_regexp(branches, ignore_prefixes)
     filter = Regexp.new("^#{Regexp.union(ignore_prefixes)}")
     branches = branches.reject do |branch|
       result = filter.match(branch)
@@ -127,14 +132,29 @@ class R10K::Source::Git < R10K::Source::Base
     branches
   end
 
+  def filter_branches_by_command(branches, command)
+    branches.select do |branch|
+      result = system({'GIT_DIR' => @cache.git_dir.to_s, 'R10K_BRANCH' => branch, 'R10K_NAME' => @name.to_s}, command)
+      unless result
+        logger.warn _("Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}") % {name: @name, branch: branch, cmd: command}
+      end
+      result
+    end
+  end
+
   private
 
   def branch_names
     opts = {:prefix => @prefix, :invalid => @invalid_branches, :source => @name}
     branches = @cache.branches
     if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty?
-      branches = filter_branches(branches, @ignore_branch_prefixes)
+      branches = filter_branches_by_regexp(branches, @ignore_branch_prefixes)
+    end
+
+    if @filter_command && !@filter_command.empty?
+      branches = filter_branches_by_command(branches, @filter_command)
     end
+
     branches.map do |branch|
       R10K::Environment::Name.new(branch, opts)
     end

data/lib/r10k/source/hash.rb

@@ -120,6 +120,16 @@
 #
 class R10K::Source::Hash < R10K::Source::Base
 
+  include R10K::Logging
+
+  # @param hash [Hash] A hash to validate.
+  # @return [Boolean] False if the hash is obviously invalid. A true return
+  #   means _maybe_ it's valid.
+  def self.valid_environments_hash?(hash)
+    # TODO: more robust schema valiation
+    hash.is_a?(Hash)
+  end
+
   # @param name [String] The identifier for this source.
   # @param basedir [String] The base directory where the generated environments will be created.
   # @param options [Hash] An additional set of options for this source. The
@@ -131,19 +141,33 @@ class R10K::Source::Hash < R10K::Source::Base
   # @option options [Hash] :environments The hash definition of environments
   def initialize(name, basedir, options = {})
     super(name, basedir, options)
+  end
 
-    @environments_hash = options.delete(:environments) || {}
-
-    @environments_hash.keys.each do |name|
-      # TODO: deal with names that aren't valid
-      ::R10K::Util::SymbolizeKeys.symbolize_keys!(@environments_hash[name])
-      @environments_hash[name][:basedir] = basedir
-      @environments_hash[name][:dirname] = name
+  # Set the environment hash for the source. The environment hash is what the
+  # source uses to generate enviroments.
+  # @param hash [Hash] The hash to sanitize and use as the source's environments.
+  #   Should be formatted for use with R10K::Environment#from_hash.
+  def set_environments_hash(hash)
+    @environments_hash = hash.reduce({}) do |memo,(name,opts)|
+      R10K::Util::SymbolizeKeys.symbolize_keys!(opts)
+      memo.merge({ 
+        name => opts.merge({
+          :basedir => @basedir,
+          :dirname => R10K::Environment::Name.new(name, {prefix: @prefix, source: @name}).dirname
+        })
+      })
     end
   end
 
+  # Return the sanitized environments hash for this source. The environments
+  # hash should contain objects formatted for use with R10K::Environment#from_hash.
+  # If the hash does not exist it will be built based on @options.
+  def environments_hash
+    @environments_hash ||= set_environments_hash(@options.fetch(:environments, {}))
+  end
+
   def environments
-    @environments ||= @environments_hash.map do |name, hash|
+    @environments ||= environments_hash.map do |name, hash|
       R10K::Environment.from_hash(name, hash)
     end
   end

data/lib/r10k/version.rb

@@ -1,3 +1,6 @@
 module R10K
-  VERSION = '3.4.0'
+  # When updating to a new major (X) or minor (Y) version, include `#major` or
+  # `#minor` (respectively) in your commit message to trigger the appropriate
+  # release. Otherwise, a new patch (Z) version will be released.
+  VERSION = '3.6.0'
 end

data/locales/r10k.pot

@@ -1,16 +1,16 @@
 # SOME DESCRIPTIVE TITLE.
-# Copyright (C) 2019 Puppet, Inc.
+# Copyright (C) 2020 Puppet, Inc.
 # This file is distributed under the same license as the r10k package.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2019.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
 #
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: r10k 3.3.3-22-g3035320\n"
+"Project-Id-Version: r10k 3.4.1-57-g2eb088a\n"
 "\n"
 "Report-Msgid-Bugs-To: docs@puppetlabs.com\n"
-"POT-Creation-Date: 2019-12-17 14:55+0000\n"
-"PO-Revision-Date: 2019-12-17 14:55+0000\n"
+"POT-Creation-Date: 2020-07-22 16:41+0000\n"
+"PO-Revision-Date: 2020-07-22 16:41+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -423,26 +423,49 @@ msgstr ""
 msgid "Setting %{name} requires a URL but '%{value}' could not be parsed as a URL"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:72
+#: ../lib/r10k/source/exec.rb:10
+msgid "Environment source %{name} missing required parameter: command"
+msgstr ""
+
+#: ../lib/r10k/source/exec.rb:36
+msgid ""
+"Error parsing command output for exec source %{name}:\n"
+"Not valid JSON: %{j_msg}\n"
+"Not valid YAML: %{y_msg}\n"
+"Stdout:\n"
+"%{out}"
+msgstr ""
+
+#: ../lib/r10k/source/exec.rb:44
+msgid ""
+"Environment source %{name} command %{cmd} did not return valid environment data.\n"
+"Returned: %{data}"
+msgstr ""
+
+#: ../lib/r10k/source/git.rb:77
 msgid "Fetching '%{remote}' to determine current branches."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:75
+#: ../lib/r10k/source/git.rb:80
 msgid "Unable to determine current branches for Git source '%{name}' (%{basedir})"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:100
+#: ../lib/r10k/source/git.rb:105
 msgid "Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:104
+#: ../lib/r10k/source/git.rb:109
 msgid "Environment %{env_name} contained non-word characters, ignoring it."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:123 ../lib/r10k/source/svn.rb:113
+#: ../lib/r10k/source/git.rb:128 ../lib/r10k/source/svn.rb:113
 msgid "Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}"
 msgstr ""
 
+#: ../lib/r10k/source/git.rb:139
+msgid "Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}"
+msgstr ""
+
 #: ../lib/r10k/source/yaml.rb:10
 msgid "Couldn't open environments file %{file}: %{err}"
 msgstr ""

data/r10k.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
   s.license  = 'Apache-2.0'
 
   s.add_dependency 'colored2',   '3.1.2'
-  s.add_dependency 'cri',       '2.15.6'
+  s.add_dependency 'cri', ['>= 2.15.10', '< 3.0.0']
 
   s.add_dependency 'log4r',     '1.1.10'
   s.add_dependency 'multi_json', '~> 1.10'
@@ -31,6 +31,10 @@ Gem::Specification.new do |s|
   s.add_dependency 'puppet_forge', '~> 2.3.0'
 
   s.add_dependency 'gettext-setup', '~>0.24'
+  # These two pins narrow what is allowed by gettext-setup,
+  # to preserver compatability with Ruby 2.4
+  s.add_dependency 'fast_gettext', '~> 1.1.0'
+  s.add_dependency 'gettext', ['>= 3.0.2', '< 3.3.0']
 
   s.add_development_dependency 'rspec', '~> 3.1'
 

data/spec/unit/action/deploy/module_spec.rb

@@ -25,6 +25,10 @@ describe R10K::Action::Deploy::Module do
     it 'can accept a puppet-path option' do
       described_class.new({ 'puppet-path': '/nonexistent' }, [])
     end
+
+    it 'can accept a cachedir option' do
+      described_class.new({ cachedir: '/nonexistent' }, [])
+    end
   end
 
   describe "with no-force" do
@@ -66,8 +70,8 @@ describe R10K::Action::Deploy::Module do
 
       before do
         allow(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
-          expect(environment.puppetfile).to receive(:modules).and_return(
-            [R10K::Module::Local.new(environment.name, '/fakedir', [], environment)]
+          expect(environment.puppetfile).to receive(:modules_by_vcs_cachedir).and_return(
+            {none: [R10K::Module::Local.new(environment.name, '/fakedir', [], environment)]}
           )
           original.call(environment, &block)
         end
@@ -123,4 +127,13 @@ describe R10K::Action::Deploy::Module do
       expect(subject.instance_variable_get(:@puppet_path)).to eq('/nonexistent')
     end
   end
+
+  describe 'with cachedir' do
+
+    subject { described_class.new({ config: '/some/nonexistent/path', cachedir: '/nonexistent' }, []) }
+
+    it 'sets puppet_path' do
+      expect(subject.instance_variable_get(:@cachedir)).to eq('/nonexistent')
+    end
+  end
 end

data/spec/unit/action/puppetfile/install_spec.rb

@@ -19,12 +19,15 @@ describe R10K::Action::Puppetfile::Install do
 
   describe "installing modules" do
     let(:modules) do
-      Array.new(4, R10K::Module::Base.new('author/modname', "/some/nonexistent/path/modname", nil))
+      (1..4).map do |idx|
+        R10K::Module::Base.new("author/modname#{idx}", "/some/nonexistent/path/modname#{idx}", nil)
+      end
     end
 
     before do
       allow(puppetfile).to receive(:purge!)
       allow(puppetfile).to receive(:modules).and_return(modules)
+      allow(puppetfile).to receive(:modules_by_vcs_cachedir).and_return({none: modules})
     end
 
     it "syncs each module in the Puppetfile" do

data/spec/unit/action/runner_spec.rb

@@ -94,7 +94,7 @@ describe R10K::Action::Runner do
                         else
                           { "#{conf_path}": override }
                         end
-            expect(global_settings).to receive(:evaluate).with(overrides).and_call_original
+            expect(global_settings).to receive(:evaluate).with(hash_including(overrides)).and_call_original
             runner.call
           end
         end
@@ -109,7 +109,7 @@ describe R10K::Action::Runner do
                         else
                           { "#{conf_path}": override }
                         end
-            expect(global_settings).to receive(:evaluate).with(overrides).and_call_original
+            expect(global_settings).to receive(:evaluate).with(hash_including(overrides)).and_call_original
             runner.call
           end
         end

data/spec/unit/forge/module_release_spec.rb

@@ -166,33 +166,37 @@ describe R10K::Forge::ModuleRelease do
   end
 
   describe "#cleanup_unpack_path" do
-    it "ignores the unpack_path if it doesn't exist" do
-      expect(unpack_path).to receive(:exist?).and_return false
-      expect(unpack_path).to_not receive(:parent)
+    it "ignores the unpack_path if the parent doesn't exist" do
+      parent = instance_double('Pathname')
+      expect(parent).to receive(:exist?).and_return false
+      expect(parent).to_not receive(:rmtree)
+      expect(unpack_path).to receive(:parent).and_return(parent)
       subject.cleanup_unpack_path
     end
 
     it "removes the containing directory of unpack_path if it exists" do
       parent = instance_double('Pathname')
       expect(parent).to receive(:rmtree)
-      expect(unpack_path).to receive(:exist?).and_return true
-      expect(unpack_path).to receive(:parent).and_return(parent)
+      expect(parent).to receive(:exist?).and_return true
+      expect(unpack_path).to receive(:parent).and_return(parent).exactly(2).times
       subject.cleanup_unpack_path
     end
   end
 
   describe "#cleanup_download_path" do
-    it "ignores the download_path if it doesn't exist" do
-      expect(download_path).to receive(:exist?).and_return false
-      expect(download_path).to_not receive(:parent)
+    it "ignores the download_path if the parent doesn't exist" do
+      parent = instance_double('Pathname')
+      expect(parent).to receive(:exist?).and_return false
+      expect(parent).to_not receive(:rmtree)
+      expect(download_path).to receive(:parent).and_return(parent)
       subject.cleanup_download_path
     end
 
     it "removes the containing directory of download_path if it exists" do
       parent = instance_double('Pathname')
       expect(parent).to receive(:rmtree)
-      expect(download_path).to receive(:exist?).and_return true
-      expect(download_path).to receive(:parent).and_return(parent)
+      expect(parent).to receive(:exist?).and_return true
+      expect(download_path).to receive(:parent).and_return(parent).exactly(2).times
       subject.cleanup_download_path
     end
   end

data/spec/unit/puppetfile_spec.rb

@@ -127,6 +127,25 @@ describe R10K::Puppetfile do
 
       expect { subject.add_module('puppet/test_module', module_opts) }.to raise_error(R10K::Error, /cannot manage content.*is not within/i).and not_change { subject.modules }
     end
+
+    it "groups modules by vcs cache location" do
+      module_opts = { install_path: File.join(subject.basedir, 'vendor') }
+      opts1 = module_opts.merge(git: 'git@example.com:puppet/test_module.git')
+      opts2 = module_opts.merge(git: 'git@example.com:puppet/test_module_c.git')
+      sanitized_name1 = "git@example.com-puppet-test_module.git"
+      sanitized_name2 = "git@example.com-puppet-test_module_c.git"
+
+      subject.add_module('puppet/test_module_a', opts1)
+      subject.add_module('puppet/test_module_b', opts1)
+      subject.add_module('puppet/test_module_c', opts2)
+      subject.add_module('puppet/test_module_d', '1.2.3')
+
+      mods_by_cachedir = subject.modules_by_vcs_cachedir
+
+      expect(mods_by_cachedir[:none].length).to be 1
+      expect(mods_by_cachedir[sanitized_name1].length).to be 2
+      expect(mods_by_cachedir[sanitized_name2].length).to be 1
+    end
   end
 
   describe "#purge_exclusions" do
@@ -154,6 +173,26 @@ describe R10K::Puppetfile do
     end
   end
 
+  describe '#managed_directories' do
+    it 'returns an array of paths that can be purged' do
+      allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.moduledir, '1.2.3', anything).and_call_original
+
+      subject.add_module('puppet/test_module', '1.2.3')
+      expect(subject.managed_directories).to match_array(["/some/nonexistent/basedir/modules"])
+    end
+
+    context 'with a module with install_path == \'\'' do
+      it 'basedir isn\'t in the list of paths to purge' do
+        module_opts = { install_path: '', git: 'git@example.com:puppet/test_module.git' }
+
+        allow(R10K::Module).to receive(:new).with('puppet/test_module', subject.basedir, module_opts, anything).and_call_original
+
+        subject.add_module('puppet/test_module', module_opts)
+        expect(subject.managed_directories).to be_empty
+      end
+    end
+  end
+
   describe "evaluating a Puppetfile" do
     def expect_wrapped_error(orig, pf_path, wrapped_error)
       expect(orig).to be_a_kind_of(R10K::Error)
@@ -268,7 +307,7 @@ describe R10K::Puppetfile do
       mod2 = spy('module')
       expect(mod2).to receive(:accept).with(visitor)
 
-      expect(subject).to receive(:modules).and_return([mod1, mod2])
+      expect(subject).to receive(:modules_by_vcs_cachedir).and_return({none: [mod1, mod2]})
       subject.accept(visitor)
     end
 
@@ -289,12 +328,38 @@ describe R10K::Puppetfile do
       mod2 = spy('module')
       expect(mod2).to receive(:accept).with(visitor)
 
-      expect(subject).to receive(:modules).and_return([mod1, mod2])
+      expect(subject).to receive(:modules_by_vcs_cachedir).and_return({none: [mod1, mod2]})
 
       expect(Thread).to receive(:new).exactly(pool_size).and_call_original
       expect(Queue).to receive(:new).and_call_original
 
       subject.accept(visitor)
     end
+
+    it "Creates queues of modules grouped by cachedir" do
+      visitor = spy('visitor')
+      expect(visitor).to receive(:visit) do |type, other, &block|
+        expect(type).to eq :puppetfile
+        expect(other).to eq subject
+        block.call
+      end
+
+      mod1 = spy('module1')
+      mod2 = spy('module2')
+      mod3 = spy('module3')
+      mod4 = spy('module4')
+      mod5 = spy('module5')
+      mod6 = spy('module6')
+
+      expect(subject).to receive(:modules_by_vcs_cachedir)
+        .and_return({:none => [mod1, mod2],
+                     "foo-cachedir" => [mod3, mod4],
+                     "bar-cachedir" => [mod5, mod6]})
+
+      queue = subject.modules_queue(visitor)
+      expect(queue.length).to be 4
+      queue_array = 4.times.map { queue.pop }
+      expect(queue_array).to match_array([[mod1], [mod2], [mod3, mod4], [mod5, mod6]])
+    end
   end
 end