r10k 3.4.1 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 564ea5e6a1060b6ceb4a5986857b612de2b47179b57998dbdb2415be1e9e795a
-  data.tar.gz: 4f14b580320afc67069326a81d89e7335cdc6ffe1ec58f45edc497150fdb16e3
+  metadata.gz: 89108575df655aa63ddced0692fa8815939d0d472b85914ea29430e1f76985b9
+  data.tar.gz: 343319b5506fb72d905b3a510765d3dac0bb3643188a95fc24cdb83f90e4d490
 SHA512:
-  metadata.gz: 93c8178a7c9dfa287c78d43e89685b68f27e330748d8865c9b9cb6f30b47a9012e33fed10c5b84d0ce37e14c133fc724506dcb57684a4c3f3e48fc2da884ddf2
-  data.tar.gz: b434ab7451f5f064a37de886673e9be6ebfc24628f496f262197a5f2a9c9cdb631922138aa67dd1f6173b1ce515c48bc028fd286939d4134dc0e8f610f4b7676
+  metadata.gz: d034cbfd986f11e9b0a3298a015d75a41db6800a06b8e437b06b1298dac0ea008a6fdd8c20546806faf3d8c58f1f964073a148609bcabb025610aa7e7b9425b6
+  data.tar.gz: 61629bc37a035d75b70a2cf70dd0cd91a0af8286215da2498518e7a6f698091584576c3afb31b734521edae36e4d371b34bc6f2e9ca08824d167d7748c4c646d

data/.travis.yml

@@ -1,7 +1,5 @@
 ---
 language: ruby
-services:
-  - docker
 bundler_args: "--without system"
 script: "bundle exec rspec --color --format documentation spec/unit"
 notifications:
@@ -25,10 +23,20 @@ matrix:
     - stage: r10k tests
       rvm: jruby
     - stage: r10k container tests
+      dist: bionic
       language: ruby
-      rvm: 2.5.5
+      services:
+        # bionic uses 18.06 but need 19.03+ for buildkit so upgrade later in relevant cell
+        - docker
+      rvm: 2.6.5
+      env:
+        # necessary to prevent overwhelming TravisCI build output limits
+        - DOCKER_BUILD_FLAGS="--progress plain"
       script:
         - |
           set -ex
+          sudo apt update -y && sudo apt install -y docker.io
+          sudo systemctl unmask docker.service
+          sudo systemctl start docker
           cd docker && make lint build test
           set +x

data/CHANGELOG.mkd

@@ -4,8 +4,19 @@ CHANGELOG
 Unreleased
 ----
 
+
+3.5.0
+-----
+
+- Add exec environment source type. The exec source type allows for the
+  implementation of external environment sources
+  [#1042](https://github.com/puppetlabs/r10k/pull/1042).
+- Improve atomicity of .r10k-deploy.json writes. Fixes
+  [#813](https://github.com/puppetlabs/r10k/issues/813)
+
 3.4.1
 ----
+
 - Add support for Ruby 2.7
 - (RK-357) Restrict gettext and fast_gettext versions for compatibility with Ruby 2.4
 - Bump cri to 2.15.10

data/azure-pipelines.yml

@@ -17,7 +17,8 @@ variables:
   NAMESPACE: puppet
   CONTAINER_NAME: r10k
   CONTAINER_BUILD_PATH: .
-  LINT_IGNORES: DL3008 DL3018 DL4000 DL4001 DL3028
+  LINT_IGNORES:
+  DOCKER_BUILDKIT: 1
 
 workspace:
   clean: resources

data/doc/dynamic-environments/configuration.mkd

@@ -578,6 +578,20 @@ remote: git@github.com:puppetlabs/control-repo.git
 ref: 8820892
 ```
 
+### Exec environment Source
+
+The exec environment source runs an external command which is expected to return on stdout content compatible with the YAML environment source data format. The command may return the data in JSON or YAML form. The exec environment source is similar in purpose to Puppet's exec node terminus, used to implement external node classifiers (ENCs). R10k's exec source type allows the the implementation of external environment sources.
+
+```yaml
+# r10k.yaml
+---
+sources:
+  puppet:
+    type: exec
+    basedir: /etc/puppetlabs/code/environments
+    command: /usr/local/bin/r10k-environments.sh
+```
+
 ### Environment Modules
 
 The environment modules feature allows module content to be attached to an environment at environment definition time. This happens before modules specified in a Puppetfile are attached to an environment, which does not happen until deploy time. Environment module implementation depends on the environment source type.

data/docker/Makefile

@@ -4,11 +4,12 @@ git_describe = $(shell git describe)
 vcs_ref := $(shell git rev-parse HEAD)
 build_date := $(shell date -u +%FT%T)
 hadolint_available := $(shell hadolint --help > /dev/null 2>&1; echo $$?)
-hadolint_command := hadolint --ignore DL3008 --ignore DL3018 --ignore DL3028 --ignore DL4000 --ignore DL4001
+hadolint_command := hadolint
 hadolint_container := hadolint/hadolint:latest
 export BUNDLE_PATH = $(PWD)/.bundle/gems
 export BUNDLE_BIN = $(PWD)/.bundle/bin
 export GEMFILE = $(PWD)/Gemfile
+export DOCKER_BUILDKIT = 1
 
 ifeq ($(IS_RELEASE),true)
 	VERSION ?= $(shell echo $(git_describe) | sed 's/-.*//')
@@ -36,6 +37,7 @@ endif
 
 build: prep
 	docker build \
+		${DOCKER_BUILD_FLAGS} \
 		--pull \
 		--build-arg vcs_ref=$(vcs_ref) \
 		--build-arg build_date=$(build_date) \

data/docker/r10k/Dockerfile

@@ -1,5 +1,6 @@
 FROM alpine:3.9 as build
 
+# hadolint ignore=DL3018
 RUN apk add --no-cache ruby git && \
     mkdir /workspace
 WORKDIR /workspace
@@ -39,13 +40,15 @@ LABEL org.label-schema.version="$version" \
       org.label-schema.build-date="$build_date"
 
 COPY --from=build /workspace/r10k.gem /
+# ignore apk and gem pinning
+# hadolint ignore=DL3018,DL3028
 RUN chmod a+x /adduser.sh && \
 # Add a puppet user to run r10k as for consistency with puppetserver
     /adduser.sh && \
     chmod +x /docker-entrypoint.sh && \
     chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \
-    apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-dev make gcc musl-dev && \
-    gem install --no-doc /r10k.gem json etc && \
+    apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-json ruby-etc && \
+    gem install --no-doc /r10k.gem && \
     rm -f /r10k.gem
 
 USER puppet

data/lib/r10k/action/deploy/environment.rb

@@ -157,7 +157,10 @@ module R10K
             logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}")
           end
 
-          File.open("#{environment.path}/.r10k-deploy.json", 'w') do |f|
+          # make this file write as atomic as possible in pure ruby
+          final   = "#{environment.path}/.r10k-deploy.json"
+          staging = "#{environment.path}/.r10k-deploy.json~"
+          File.open(staging, 'w') do |f|
             deploy_info = environment.info.merge({
               :started_at => started_at,
               :finished_at => Time.new,
@@ -167,6 +170,7 @@ module R10K
 
             f.puts(JSON.pretty_generate(deploy_info))
           end
+          FileUtils.mv(staging, final)
         end
 
         def undeployable_environment_names(environments, expected_names)

data/lib/r10k/action/runner.rb

@@ -43,10 +43,10 @@ module R10K
         config_settings = settings_from_config(@opts[:config])
 
         overrides = {}
-        overrides[:cachedir] = @opts[:cachedir] unless @opts[:cachedir].nil?
-        overrides[:deploy] = {} if !@opts[:'puppet-path'].nil? || !@opts[:'generate-types'].nil?
-        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] unless @opts[:'puppet-path'].nil?
-        overrides[:deploy][:generate_types] = @opts[:'generate-types'] unless @opts[:'generate-types'].nil?
+        overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
+        overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
+        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+        overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash

data/lib/r10k/source.rb

@@ -37,5 +37,6 @@ module R10K
     require 'r10k/source/svn'
     require 'r10k/source/yaml'
     require 'r10k/source/yamldir'
+    require 'r10k/source/exec'
   end
 end

data/lib/r10k/source/exec.rb

@@ -0,0 +1,51 @@
+require 'r10k/util/subprocess'
+require 'json'
+require 'yaml'
+
+class R10K::Source::Exec < R10K::Source::Hash
+  R10K::Source.register(:exec, self)
+
+  def initialize(name, basedir, options = {})
+    unless @command = options[:command]
+      raise ConfigError, _('Environment source %{name} missing required parameter: command') % {name: name}
+    end
+
+    # We haven't set the environments option yet. We will do that by
+    # overloading the #environments method.
+    super(name, basedir, options)
+  end
+
+  def environments_hash
+    @environments_hash ||= set_environments_hash(run_environments_command)
+  end
+
+  private
+
+  def run_environments_command
+    subproc = R10K::Util::Subprocess.new([@command])
+    subproc.raise_on_fail = true
+    subproc.logger = self.logger
+    procresult = subproc.execute
+
+    begin
+      environments = JSON.parse(procresult.stdout)
+    rescue JSON::ParserError => json_err
+      begin
+        environments = YAML.safe_load(procresult.stdout)
+      rescue Psych::SyntaxError => yaml_err
+        raise R10K::Error, _("Error parsing command output for exec source %{name}:\n" \
+                             "Not valid JSON: %{j_msg}\n" \
+                             "Not valid YAML: %{y_msg}\n" \
+                             "Stdout:\n%{out}") % {name: name, j_msg: json_err.message, y_msg: yaml_err.message, out: procresult.stdout}
+      end
+    end
+
+    unless R10K::Source::Hash.valid_environments_hash?(environments)
+      raise R10K::Error, _("Environment source %{name} command %{cmd} did not return valid environment data.\n" \
+                           'Returned: %{data}') % {name: name, cmd: @command, data: environments}
+    end
+
+    # Return the resulting environments hash
+    environments
+  end
+end

data/lib/r10k/source/hash.rb

@@ -120,6 +120,16 @@
 #
 class R10K::Source::Hash < R10K::Source::Base
 
+  include R10K::Logging
+
+  # @param hash [Hash] A hash to validate.
+  # @return [Boolean] False if the hash is obviously invalid. A true return
+  #   means _maybe_ it's valid.
+  def self.valid_environments_hash?(hash)
+    # TODO: more robust schema valiation
+    hash.is_a?(Hash)
+  end
+
   # @param name [String] The identifier for this source.
   # @param basedir [String] The base directory where the generated environments will be created.
   # @param options [Hash] An additional set of options for this source. The
@@ -131,19 +141,33 @@ class R10K::Source::Hash < R10K::Source::Base
   # @option options [Hash] :environments The hash definition of environments
   def initialize(name, basedir, options = {})
     super(name, basedir, options)
+  end
 
-    @environments_hash = options.delete(:environments) || {}
-
-    @environments_hash.keys.each do |name|
-      # TODO: deal with names that aren't valid
-      ::R10K::Util::SymbolizeKeys.symbolize_keys!(@environments_hash[name])
-      @environments_hash[name][:basedir] = basedir
-      @environments_hash[name][:dirname] = name
+  # Set the environment hash for the source. The environment hash is what the
+  # source uses to generate enviroments.
+  # @param hash [Hash] The hash to sanitize and use as the source's environments.
+  #   Should be formatted for use with R10K::Environment#from_hash.
+  def set_environments_hash(hash)
+    @environments_hash = hash.reduce({}) do |memo,(name,opts)|
+      R10K::Util::SymbolizeKeys.symbolize_keys!(opts)
+      memo.merge({ 
+        name => opts.merge({
+          :basedir => @basedir,
+          :dirname => R10K::Environment::Name.new(name, {prefix: @prefix, source: @name}).dirname
+        })
+      })
     end
   end
 
+  # Return the sanitized environments hash for this source. The environments
+  # hash should contain objects formatted for use with R10K::Environment#from_hash.
+  # If the hash does not exist it will be built based on @options.
+  def environments_hash
+    @environments_hash ||= set_environments_hash(@options.fetch(:environments, {}))
+  end
+
   def environments
-    @environments ||= @environments_hash.map do |name, hash|
+    @environments ||= environments_hash.map do |name, hash|
       R10K::Environment.from_hash(name, hash)
     end
   end

data/lib/r10k/version.rb

@@ -2,5 +2,5 @@ module R10K
   # When updating to a new major (X) or minor (Y) version, include `#major` or
   # `#minor` (respectively) in your commit message to trigger the appropriate
   # release. Otherwise, a new patch (Z) version will be released.
-  VERSION = '3.4.1'
+  VERSION = '3.5.0'
 end

data/locales/r10k.pot

@@ -1,16 +1,16 @@
 # SOME DESCRIPTIVE TITLE.
-# Copyright (C) 2019 Puppet, Inc.
+# Copyright (C) 2020 Puppet, Inc.
 # This file is distributed under the same license as the r10k package.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2019.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2020.
 #
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: r10k 3.3.3-22-g3035320\n"
+"Project-Id-Version: r10k 3.4.1-20-g8fe925b\n"
 "\n"
 "Report-Msgid-Bugs-To: docs@puppetlabs.com\n"
-"POT-Creation-Date: 2019-12-17 14:55+0000\n"
-"PO-Revision-Date: 2019-12-17 14:55+0000\n"
+"POT-Creation-Date: 2020-04-28 18:41+0000\n"
+"PO-Revision-Date: 2020-04-28 18:41+0000\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
 "Language: \n"
@@ -423,6 +423,25 @@ msgstr ""
 msgid "Setting %{name} requires a URL but '%{value}' could not be parsed as a URL"
 msgstr ""
 
+#: ../lib/r10k/source/exec.rb:10
+msgid "Environment source %{name} missing required parameter: command"
+msgstr ""
+
+#: ../lib/r10k/source/exec.rb:36
+msgid ""
+"Error parsing command output for exec source %{name}:\n"
+"Not valid JSON: %{j_msg}\n"
+"Not valid YAML: %{y_msg}\n"
+"Stdout:\n"
+"%{out}"
+msgstr ""
+
+#: ../lib/r10k/source/exec.rb:44
+msgid ""
+"Environment source %{name} command %{cmd} did not return valid environment data.\n"
+"Returned: %{data}"
+msgstr ""
+
 #: ../lib/r10k/source/git.rb:72
 msgid "Fetching '%{remote}' to determine current branches."
 msgstr ""

data/spec/unit/action/puppetfile/install_spec.rb

@@ -19,7 +19,9 @@ describe R10K::Action::Puppetfile::Install do
 
   describe "installing modules" do
     let(:modules) do
-      Array.new(4, R10K::Module::Base.new('author/modname', "/some/nonexistent/path/modname", nil))
+      (1..4).map do |idx|
+        R10K::Module::Base.new("author/modname#{idx}", "/some/nonexistent/path/modname#{idx}", nil)
+      end
     end
 
     before do

data/spec/unit/action/runner_spec.rb

@@ -94,7 +94,7 @@ describe R10K::Action::Runner do
                         else
                           { "#{conf_path}": override }
                         end
-            expect(global_settings).to receive(:evaluate).with(overrides).and_call_original
+            expect(global_settings).to receive(:evaluate).with(hash_including(overrides)).and_call_original
             runner.call
           end
         end
@@ -109,7 +109,7 @@ describe R10K::Action::Runner do
                         else
                           { "#{conf_path}": override }
                         end
-            expect(global_settings).to receive(:evaluate).with(overrides).and_call_original
+            expect(global_settings).to receive(:evaluate).with(hash_including(overrides)).and_call_original
             runner.call
           end
         end

data/spec/unit/source/exec_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+require 'r10k/source'
+require 'json'
+require 'yaml'
+
+describe R10K::Source::Exec do
+
+  let(:environments_hash) do
+    {
+      'production' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'release-141',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.0'
+          }
+        }
+      },
+      'development' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'master',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.1'
+          }
+        }
+      }
+    }
+  end
+
+  describe 'initialize' do
+    context 'with a valid command' do
+      context 'that produces valid output' do
+        it 'accepts json' do
+          allow_any_instance_of(R10K::Util::Subprocess)
+            .to receive(:execute)
+            .and_return(double('result', stdout: environments_hash.to_json))
+
+          source = described_class.new('execsource', '/some/nonexistent/dir', command: '/path/to/command')
+          expect(source.environments.map(&:name)).to contain_exactly('production', 'development')
+        end
+
+        it 'accepts yaml' do
+          allow_any_instance_of(R10K::Util::Subprocess)
+            .to receive(:execute)
+            .and_return(double('result', stdout: environments_hash.to_yaml))
+
+          source = described_class.new('execsource', '/some/nonexistent/dir', command: '/path/to/command')
+          expect(source.environments.map(&:name)).to contain_exactly('production', 'development')
+        end
+
+      end
+
+      context 'that produces invalid output' do
+        it 'raises an error for non-json, non-yaml data' do
+          allow_any_instance_of(R10K::Util::Subprocess)
+            .to receive(:execute)
+            .and_return(double('result', stdout: "one:\ntwo\n"))
+
+          source = described_class.new('execsource', '/some/nonexistent/dir', command: '/path/to/command')
+          expect { source.environments }.to raise_error(/Error parsing command output/)
+        end
+
+        it 'raises an error for yaml data that is not a hash' do
+          allow_any_instance_of(R10K::Util::Subprocess)
+            .to receive(:execute)
+            .and_return(double('result', stdout: "[one, two]"))
+
+          source = described_class.new('execsource', '/some/nonexistent/dir', command: '/path/to/command')
+          expect { source.environments }.to raise_error(R10K::Error, /Environment source execsource.*did not return valid environment data.*one.*two.*/m)
+        end
+      end
+    end
+  end
+end

data/spec/unit/source/hash_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'r10k/source'
+
+describe R10K::Source::Hash do
+
+  describe '.valid_environments_hash?' do
+    it "rejects strings" do
+      expect(R10K::Source::Hash.valid_environments_hash?('200 OK'))
+        .to eq false
+    end
+  end
+
+  let(:environments_hash) do
+    {
+      'production' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'release-141',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.0'
+          }
+        }
+      },
+      'development' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'master',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.1'
+          }
+        }
+      }
+    }
+  end
+
+  describe "with a prefix" do
+    subject do
+      described_class.new('hashsource', '/some/nonexistent/dir',
+                          prefix: 'prefixed', environments: environments_hash)
+    end
+
+    it "prepends environment names with a prefix" do
+      environments = subject.environments
+      expect(environments[0].dirname).to eq 'prefixed_production'
+      expect(environments[1].dirname).to eq 'prefixed_development'
+    end
+  end
+end

data/spec/unit/source/yaml_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'r10k/source'
+
+describe R10K::Source::Yaml do
+
+  let(:environments_hash) do
+    {
+      'production' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'release-141',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.0'
+          }
+        }
+      },
+      'development' => {
+        'remote'  => 'https://git.example.com/puppet/control-repo.git',
+        'ref'     => 'master',
+        'modules' => {
+          'puppetlabs-stdlib' => '6.1.0',
+          'puppetlabs-ntp' => '8.1.0',
+          'example-myapp1' => {
+            'git' => 'https://git.example.com/puppet/example-myapp1.git',
+            'ref' => 'v1.3.1'
+          }
+        }
+      }
+    }
+  end
+
+  describe "with valid yaml file" do
+    it "produces environments" do
+      allow(YAML).to receive(:load_file).with('/envs.yaml').and_return(environments_hash)
+      source = described_class.new('yamlsource', '/some/nonexistent/dir', config: '/envs.yaml')
+      expect(source.environments.map(&:name)).to contain_exactly('production', 'development')
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: r10k
 version: !ruby/object:Gem::Version
-  version: 3.4.1
+  version: 3.5.0
 platform: ruby
 authors:
 - Adrien Thebo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-06 00:00:00.000000000 Z
+date: 2020-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colored2
@@ -408,6 +408,7 @@ files:
 - lib/r10k/settings/uri_definition.rb
 - lib/r10k/source.rb
 - lib/r10k/source/base.rb
+- lib/r10k/source/exec.rb
 - lib/r10k/source/git.rb
 - lib/r10k/source/hash.rb
 - lib/r10k/source/svn.rb
@@ -535,8 +536,11 @@ files:
 - spec/unit/settings/uri_definition_spec.rb
 - spec/unit/settings_spec.rb
 - spec/unit/source/base_spec.rb
+- spec/unit/source/exec_spec.rb
 - spec/unit/source/git_spec.rb
+- spec/unit/source/hash_spec.rb
 - spec/unit/source/svn_spec.rb
+- spec/unit/source/yaml_spec.rb
 - spec/unit/source_spec.rb
 - spec/unit/svn/remote_spec.rb
 - spec/unit/svn/working_dir_spec.rb