r10k 3.4.1 → 3.7.0

data/lib/r10k/settings.rb

@@ -12,6 +12,8 @@ module R10K
     class << self
       # Path to puppet executable
       attr_accessor :puppet_path
+      # Path to puppet.conf
+      attr_accessor :puppet_conf
     end
 
     def self.git_settings
@@ -131,6 +133,15 @@ module R10K
             end
           end
         }),
+        Definition.new(:puppet_conf, {
+          :desc => "Path to puppet.conf. Defaults to /etc/puppetlabs/puppet/puppet.conf.",
+          :default => '/etc/puppetlabs/puppet/puppet.conf',
+          :validate => lambda do |value|
+            unless File.readable? value
+              raise ArgumentError, "The specified puppet.conf #{value} is not readable"
+            end
+          end
+        }),
       ])
     end
 
@@ -160,7 +171,7 @@ module R10K
 
         Definition.new(:pool_size, {
           :desc => "The amount of threads used to concurrently install modules. The default value is 1: install one module at a time.",
-          :default => 1,
+          :default => 4,
           :validate => lambda do |value|
             if !value.is_a?(Integer)
               raise ArgumentError, "The pool_size setting should be an integer, not a #{value.class}"

data/lib/r10k/source.rb

@@ -37,5 +37,6 @@ module R10K
     require 'r10k/source/svn'
     require 'r10k/source/yaml'
     require 'r10k/source/yamldir'
+    require 'r10k/source/exec'
   end
 end

data/lib/r10k/source/exec.rb

@@ -0,0 +1,51 @@
+require 'r10k/util/subprocess'
+require 'json'
+require 'yaml'
+
+class R10K::Source::Exec < R10K::Source::Hash
+  R10K::Source.register(:exec, self)
+
+  def initialize(name, basedir, options = {})
+    unless @command = options[:command]
+      raise ConfigError, _('Environment source %{name} missing required parameter: command') % {name: name}
+    end
+
+    # We haven't set the environments option yet. We will do that by
+    # overloading the #environments method.
+    super(name, basedir, options)
+  end
+
+  def environments_hash
+    @environments_hash ||= set_environments_hash(run_environments_command)
+  end
+
+  private
+
+  def run_environments_command
+    subproc = R10K::Util::Subprocess.new([@command])
+    subproc.raise_on_fail = true
+    subproc.logger = self.logger
+    procresult = subproc.execute
+
+    begin
+      environments = JSON.parse(procresult.stdout)
+    rescue JSON::ParserError => json_err
+      begin
+        environments = YAML.safe_load(procresult.stdout)
+      rescue Psych::SyntaxError => yaml_err
+        raise R10K::Error, _("Error parsing command output for exec source %{name}:\n" \
+                             "Not valid JSON: %{j_msg}\n" \
+                             "Not valid YAML: %{y_msg}\n" \
+                             "Stdout:\n%{out}") % {name: name, j_msg: json_err.message, y_msg: yaml_err.message, out: procresult.stdout}
+      end
+    end
+
+    unless R10K::Source::Hash.valid_environments_hash?(environments)
+      raise R10K::Error, _("Environment source %{name} command %{cmd} did not return valid environment data.\n" \
+                           'Returned: %{data}') % {name: name, cmd: @command, data: environments}
+    end
+
+    # Return the resulting environments hash
+    environments
+  end
+end

data/lib/r10k/source/git.rb

@@ -41,6 +41,10 @@ class R10K::Source::Git < R10K::Source::Base
   #     that will be deployed as environments.
   attr_reader :ignore_branch_prefixes
 
+  # @!attribute [r] filter_command
+  #   @return [String] Command to run to filter branches
+  attr_reader :filter_command
+
   # Initialize the given source.
   #
   # @param name [String] The identifier for this source.
@@ -61,6 +65,7 @@ class R10K::Source::Git < R10K::Source::Base
     @remote = options[:remote]
     @invalid_branches = (options[:invalid_branches] || 'correct_and_warn')
     @ignore_branch_prefixes = options[:ignore_branch_prefixes]
+    @filter_command = options[:filter_command]
 
     @cache  = R10K::Git.cache.generate(@remote)
   end
@@ -115,7 +120,7 @@ class R10K::Source::Git < R10K::Source::Base
     environments.map {|env| env.dirname }
   end
 
-  def filter_branches(branches, ignore_prefixes)
+  def filter_branches_by_regexp(branches, ignore_prefixes)
     filter = Regexp.new("^#{Regexp.union(ignore_prefixes)}")
     branches = branches.reject do |branch|
       result = filter.match(branch)
@@ -127,14 +132,29 @@ class R10K::Source::Git < R10K::Source::Base
     branches
   end
 
+  def filter_branches_by_command(branches, command)
+    branches.select do |branch|
+      result = system({'GIT_DIR' => @cache.git_dir.to_s, 'R10K_BRANCH' => branch, 'R10K_NAME' => @name.to_s}, command)
+      unless result
+        logger.warn _("Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}") % {name: @name, branch: branch, cmd: command}
+      end
+      result
+    end
+  end
+
   private
 
   def branch_names
     opts = {:prefix => @prefix, :invalid => @invalid_branches, :source => @name}
     branches = @cache.branches
     if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty?
-      branches = filter_branches(branches, @ignore_branch_prefixes)
+      branches = filter_branches_by_regexp(branches, @ignore_branch_prefixes)
+    end
+
+    if @filter_command && !@filter_command.empty?
+      branches = filter_branches_by_command(branches, @filter_command)
     end
+
     branches.map do |branch|
       R10K::Environment::Name.new(branch, opts)
     end

data/lib/r10k/source/hash.rb

@@ -120,6 +120,16 @@
 #
 class R10K::Source::Hash < R10K::Source::Base
 
+  include R10K::Logging
+
+  # @param hash [Hash] A hash to validate.
+  # @return [Boolean] False if the hash is obviously invalid. A true return
+  #   means _maybe_ it's valid.
+  def self.valid_environments_hash?(hash)
+    # TODO: more robust schema valiation
+    hash.is_a?(Hash)
+  end
+
   # @param name [String] The identifier for this source.
   # @param basedir [String] The base directory where the generated environments will be created.
   # @param options [Hash] An additional set of options for this source. The
@@ -131,19 +141,33 @@ class R10K::Source::Hash < R10K::Source::Base
   # @option options [Hash] :environments The hash definition of environments
   def initialize(name, basedir, options = {})
     super(name, basedir, options)
+  end
 
-    @environments_hash = options.delete(:environments) || {}
-
-    @environments_hash.keys.each do |name|
-      # TODO: deal with names that aren't valid
-      ::R10K::Util::SymbolizeKeys.symbolize_keys!(@environments_hash[name])
-      @environments_hash[name][:basedir] = basedir
-      @environments_hash[name][:dirname] = name
+  # Set the environment hash for the source. The environment hash is what the
+  # source uses to generate enviroments.
+  # @param hash [Hash] The hash to sanitize and use as the source's environments.
+  #   Should be formatted for use with R10K::Environment#from_hash.
+  def set_environments_hash(hash)
+    @environments_hash = hash.reduce({}) do |memo,(name,opts)|
+      R10K::Util::SymbolizeKeys.symbolize_keys!(opts)
+      memo.merge({ 
+        name => opts.merge({
+          :basedir => @basedir,
+          :dirname => R10K::Environment::Name.new(name, {prefix: @prefix, source: @name}).dirname
+        })
+      })
     end
   end
 
+  # Return the sanitized environments hash for this source. The environments
+  # hash should contain objects formatted for use with R10K::Environment#from_hash.
+  # If the hash does not exist it will be built based on @options.
+  def environments_hash
+    @environments_hash ||= set_environments_hash(@options.fetch(:environments, {}))
+  end
+
   def environments
-    @environments ||= @environments_hash.map do |name, hash|
+    @environments ||= environments_hash.map do |name, hash|
       R10K::Environment.from_hash(name, hash)
     end
   end

data/lib/r10k/version.rb

@@ -2,5 +2,5 @@ module R10K
   # When updating to a new major (X) or minor (Y) version, include `#major` or
   # `#minor` (respectively) in your commit message to trigger the appropriate
   # release. Otherwise, a new patch (Z) version will be released.
-  VERSION = '3.4.1'
+  VERSION = '3.7.0'
 end

data/locales/r10k.pot

@@ -1,16 +1,16 @@
 # SOME DESCRIPTIVE TITLE.
-# Copyright (C) 2019 Puppet, Inc.
+# Copyright (C) 2020 Puppet, Inc.
 # This file is distributed under the same license as the r10k package.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2019.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
 #
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: r10k 3.3.3-22-g3035320\n"
+"Project-Id-Version: r10k 3.4.1-57-g2eb088a\n"
 "\n"
 "Report-Msgid-Bugs-To: docs@puppetlabs.com\n"
-"POT-Creation-Date: 2019-12-17 14:55+0000\n"
-"PO-Revision-Date: 2019-12-17 14:55+0000\n"
+"POT-Creation-Date: 2020-07-22 16:41+0000\n"
+"PO-Revision-Date: 2020-07-22 16:41+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -423,26 +423,49 @@ msgstr ""
 msgid "Setting %{name} requires a URL but '%{value}' could not be parsed as a URL"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:72
+#: ../lib/r10k/source/exec.rb:10
+msgid "Environment source %{name} missing required parameter: command"
+msgstr ""
+
+#: ../lib/r10k/source/exec.rb:36
+msgid ""
+"Error parsing command output for exec source %{name}:\n"
+"Not valid JSON: %{j_msg}\n"
+"Not valid YAML: %{y_msg}\n"
+"Stdout:\n"
+"%{out}"
+msgstr ""
+
+#: ../lib/r10k/source/exec.rb:44
+msgid ""
+"Environment source %{name} command %{cmd} did not return valid environment data.\n"
+"Returned: %{data}"
+msgstr ""
+
+#: ../lib/r10k/source/git.rb:77
 msgid "Fetching '%{remote}' to determine current branches."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:75
+#: ../lib/r10k/source/git.rb:80
 msgid "Unable to determine current branches for Git source '%{name}' (%{basedir})"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:100
+#: ../lib/r10k/source/git.rb:105
 msgid "Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}"
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:104
+#: ../lib/r10k/source/git.rb:109
 msgid "Environment %{env_name} contained non-word characters, ignoring it."
 msgstr ""
 
-#: ../lib/r10k/source/git.rb:123 ../lib/r10k/source/svn.rb:113
+#: ../lib/r10k/source/git.rb:128 ../lib/r10k/source/svn.rb:113
 msgid "Branch %{branch} filtered out by ignore_branch_prefixes %{ibp}"
 msgstr ""
 
+#: ../lib/r10k/source/git.rb:139
+msgid "Branch `%{name}:%{branch}` filtered out by filter_command %{cmd}"
+msgstr ""
+
 #: ../lib/r10k/source/yaml.rb:10
 msgid "Couldn't open environments file %{file}: %{err}"
 msgstr ""

data/spec/shared-examples/subprocess-runner.rb

@@ -32,23 +32,29 @@ shared_examples_for "a subprocess runner" do |fixture_root|
     end
   end
 
-  describe "running 'ls' with a different working directory" do
+  describe "running 'ls' or 'dir' with a different working directory" do
     subject do
-      described_class.new(%w[ls]).tap do |o|
-        o.cwd = fixture_root
+      if R10K::Util::Platform.windows?
+        described_class.new(%w[cmd /c dir]).tap do |o|
+          o.cwd = fixture_root
+        end
+      else
+        described_class.new(%w[ls]).tap do |o|
+          o.cwd = fixture_root
+        end
       end
     end
 
     it "returns the contents of the given working directory" do
       result = subject.run
-      expect(result.stdout).to eq 'no-execute.sh'
+      expect(result.stdout).to match('no-execute.sh')
     end
   end
 
   describe "running 'false'" do
     subject { described_class.new(%w[false]) }
 
-    it "sets the exit code to 1" do
+    it "sets the exit code to 1", unless: R10K::Util::Platform.windows? do
       result = subject.run
       expect(result.exit_code).to eq 1
     end

data/spec/unit/action/deploy/environment_spec.rb

@@ -331,6 +331,15 @@ describe R10K::Action::Deploy::Environment do
         expect(subject.instance_variable_get(:@puppet_path)).to eq('/nonexistent')
       end
     end
+
+    describe 'with puppet-conf' do
+
+      subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, []) }
+
+      it 'sets puppet_conf' do
+        expect(subject.instance_variable_get(:@puppet_conf)).to eq('/nonexistent')
+      end
+    end
   end
 
   describe "write_environment_info!" do

data/spec/unit/action/deploy/module_spec.rb

@@ -26,6 +26,10 @@ describe R10K::Action::Deploy::Module do
       described_class.new({ 'puppet-path': '/nonexistent' }, [])
     end
 
+    it 'can accept a puppet-conf option' do
+      described_class.new({ 'puppet-conf': '/nonexistent' }, [])
+    end
+
     it 'can accept a cachedir option' do
       described_class.new({ cachedir: '/nonexistent' }, [])
     end
@@ -70,8 +74,8 @@ describe R10K::Action::Deploy::Module do
 
       before do
         allow(subject).to receive(:visit_environment).and_wrap_original do |original, environment, &block|
-          expect(environment.puppetfile).to receive(:modules).and_return(
-            [R10K::Module::Local.new(environment.name, '/fakedir', [], environment)]
+          expect(environment.puppetfile).to receive(:modules_by_vcs_cachedir).and_return(
+            {none: [R10K::Module::Local.new(environment.name, '/fakedir', [], environment)]}
           )
           original.call(environment, &block)
         end
@@ -128,6 +132,15 @@ describe R10K::Action::Deploy::Module do
     end
   end
 
+  describe 'with puppet-conf' do
+
+    subject { described_class.new({ config: '/some/nonexistent/path', 'puppet-conf': '/nonexistent' }, []) }
+
+    it 'sets puppet_conf' do
+      expect(subject.instance_variable_get(:@puppet_conf)).to eq('/nonexistent')
+    end
+  end
+
   describe 'with cachedir' do
 
     subject { described_class.new({ config: '/some/nonexistent/path', cachedir: '/nonexistent' }, []) }

data/spec/unit/action/puppetfile/install_spec.rb

@@ -19,12 +19,15 @@ describe R10K::Action::Puppetfile::Install do
 
   describe "installing modules" do
     let(:modules) do
-      Array.new(4, R10K::Module::Base.new('author/modname', "/some/nonexistent/path/modname", nil))
+      (1..4).map do |idx|
+        R10K::Module::Base.new("author/modname#{idx}", "/some/nonexistent/path/modname#{idx}", nil)
+      end
     end
 
     before do
       allow(puppetfile).to receive(:purge!)
       allow(puppetfile).to receive(:modules).and_return(modules)
+      allow(puppetfile).to receive(:modules_by_vcs_cachedir).and_return({none: modules})
     end
 
     it "syncs each module in the Puppetfile" do

data/spec/unit/action/runner_spec.rb

@@ -94,7 +94,7 @@ describe R10K::Action::Runner do
                         else
                           { "#{conf_path}": override }
                         end
-            expect(global_settings).to receive(:evaluate).with(overrides).and_call_original
+            expect(global_settings).to receive(:evaluate).with(hash_including(overrides)).and_call_original
             runner.call
           end
         end
@@ -109,7 +109,7 @@ describe R10K::Action::Runner do
                         else
                           { "#{conf_path}": override }
                         end
-            expect(global_settings).to receive(:evaluate).with(overrides).and_call_original
+            expect(global_settings).to receive(:evaluate).with(hash_including(overrides)).and_call_original
             runner.call
           end
         end

data/spec/unit/forge/module_release_spec.rb

@@ -166,33 +166,37 @@ describe R10K::Forge::ModuleRelease do
   end
 
   describe "#cleanup_unpack_path" do
-    it "ignores the unpack_path if it doesn't exist" do
-      expect(unpack_path).to receive(:exist?).and_return false
-      expect(unpack_path).to_not receive(:parent)
+    it "ignores the unpack_path if the parent doesn't exist" do
+      parent = instance_double('Pathname')
+      expect(parent).to receive(:exist?).and_return false
+      expect(parent).to_not receive(:rmtree)
+      expect(unpack_path).to receive(:parent).and_return(parent)
       subject.cleanup_unpack_path
     end
 
     it "removes the containing directory of unpack_path if it exists" do
       parent = instance_double('Pathname')
       expect(parent).to receive(:rmtree)
-      expect(unpack_path).to receive(:exist?).and_return true
-      expect(unpack_path).to receive(:parent).and_return(parent)
+      expect(parent).to receive(:exist?).and_return true
+      expect(unpack_path).to receive(:parent).and_return(parent).exactly(2).times
       subject.cleanup_unpack_path
     end
   end
 
   describe "#cleanup_download_path" do
-    it "ignores the download_path if it doesn't exist" do
-      expect(download_path).to receive(:exist?).and_return false
-      expect(download_path).to_not receive(:parent)
+    it "ignores the download_path if the parent doesn't exist" do
+      parent = instance_double('Pathname')
+      expect(parent).to receive(:exist?).and_return false
+      expect(parent).to_not receive(:rmtree)
+      expect(download_path).to receive(:parent).and_return(parent)
       subject.cleanup_download_path
     end
 
     it "removes the containing directory of download_path if it exists" do
       parent = instance_double('Pathname')
       expect(parent).to receive(:rmtree)
-      expect(download_path).to receive(:exist?).and_return true
-      expect(download_path).to receive(:parent).and_return(parent)
+      expect(parent).to receive(:exist?).and_return true
+      expect(download_path).to receive(:parent).and_return(parent).exactly(2).times
       subject.cleanup_download_path
     end
   end