r10k 3.5.0 → 3.8.0

data/doc/faq.mkd

@@ -141,7 +141,7 @@ modulepath = modules:external-modules
 
 Lastly, you can simply move your locally versioned modules to a separate
 directory to avoid conflicting over the `/modules` directory entirely. With this
-example as well you can use the `environment.conf file to tell Puppet which
+example as well you can use the `environment.conf` file to tell Puppet which
 directories contain modules.
 
 ```
@@ -149,6 +149,11 @@ directories contain modules.
 modulepath = internal-modules:modules
 ```
 
+#### Does R10K support Local/Private Forge?
+
+Yes. Set the Forge to use _globally_ in `r10k.yaml`. see [Configuration](/doc/dynamic-environments/configuration.mkd#baseurl) for details.
+
+
 #### What does the name mean?
 
 It’s called R10K because I’m terrible at names. When I started working on R10K

data/doc/puppetfile.mkd

@@ -54,6 +54,8 @@ The `forge` setting specifies which server that Forge based modules are fetched
 from. This is currently a noop and is provided for compatibility with
 librarian-puppet.
 
+R10k supports setting the Forge to use _globally_ in `r10k.yaml`. see [Configuration](/doc/dynamic-environments/configuration.mkd#baseurl) for details.
+
 ### moduledir
 
 The `moduledir` setting specifies where modules from the Puppetfile will be

data/docker/Makefile

@@ -1,11 +1,12 @@
 PUPPERWARE_ANALYTICS_STREAM ?= dev
 NAMESPACE ?= puppet
-git_describe = $(shell git describe)
+git_describe = $(shell git describe --tags)
 vcs_ref := $(shell git rev-parse HEAD)
 build_date := $(shell date -u +%FT%T)
 hadolint_available := $(shell hadolint --help > /dev/null 2>&1; echo $$?)
 hadolint_command := hadolint
 hadolint_container := hadolint/hadolint:latest
+alpine_version := 3.9
 export BUNDLE_PATH = $(PWD)/.bundle/gems
 export BUNDLE_BIN = $(PWD)/.bundle/bin
 export GEMFILE = $(PWD)/Gemfile
@@ -13,6 +14,14 @@ export DOCKER_BUILDKIT = 1
 
 ifeq ($(IS_RELEASE),true)
 	VERSION ?= $(shell echo $(git_describe) | sed 's/-.*//')
+	PUBLISHED_VERSION ?= $(shell curl --silent 'https://rubygems.org/api/v1/gems/r10k.json' | jq '."version"' | tr -d '"')
+	CONTAINER_EXISTS = $(shell DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect $(NAMESPACE)/r10k:$(VERSION) > /dev/null 2>&1; echo $$?)
+ifeq ($(CONTAINER_EXISTS),0)
+	SKIP_BUILD ?= true
+else ifneq ($(VERSION),$(PUBLISHED_VERSION))
+	SKIP_BUILD ?= true
+endif
+
 	LATEST_VERSION ?= latest
 	dockerfile := release.Dockerfile
 	dockerfile_context := r10k
@@ -26,6 +35,10 @@ endif
 prep:
 	@git fetch --unshallow 2> /dev/null ||:
 	@git fetch origin 'refs/tags/*:refs/tags/*'
+ifeq ($(SKIP_BUILD),true)
+	@echo "SKIP_BUILD is true, exiting with 1"
+	@exit 1
+endif
 
 lint:
 ifeq ($(hadolint_available),0)
@@ -36,9 +49,10 @@ else
 endif
 
 build: prep
+	docker pull alpine:$(alpine_version)
 	docker build \
 		${DOCKER_BUILD_FLAGS} \
-		--pull \
+		--build-arg alpine_version=$(alpine_version) \
 		--build-arg vcs_ref=$(vcs_ref) \
 		--build-arg build_date=$(build_date) \
 		--build-arg version=$(VERSION) \

data/docker/r10k/Dockerfile

@@ -1,4 +1,5 @@
-FROM alpine:3.9 as build
+ARG alpine_version=3.9
+FROM alpine:${alpine_version} as build
 
 # hadolint ignore=DL3018
 RUN apk add --no-cache ruby git && \
@@ -8,7 +9,7 @@ COPY . /workspace
 RUN gem build r10k.gemspec && \
     mv r10k*.gem r10k.gem
 
-FROM alpine:3.9
+FROM alpine:${alpine_version}
 
 ARG vcs_ref
 ARG build_date
@@ -16,6 +17,11 @@ ARG version="3.1.0"
 # Used by entrypoint to submit metrics to Google Analytics.
 # Published images should use "production" for this build_arg.
 ARG pupperware_analytics_stream="dev"
+# required to schedule runs of "r10k" in K8s
+ARG supercronic_version="0.1.9"
+ARG supercronic_sha1sum="5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85"
+ARG supercronic="supercronic-linux-amd64"
+ARG supercronic_url="https://github.com/aptible/supercronic/releases/download/v$supercronic_version/$supercronic"
 
 LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
       org.label-schema.vendor="Puppet" \
@@ -32,7 +38,7 @@ COPY docker/r10k/docker-entrypoint.d /docker-entrypoint.d
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["help"]
 
-# dyanmic LABELs and ENV vars placed lower for the sake of Docker layer caching
+# dynamic LABELs and ENV vars placed lower for the sake of Docker layer caching
 ENV PUPPERWARE_ANALYTICS_STREAM="$pupperware_analytics_stream"
 
 LABEL org.label-schema.version="$version" \
@@ -40,16 +46,21 @@ LABEL org.label-schema.version="$version" \
       org.label-schema.build-date="$build_date"
 
 COPY --from=build /workspace/r10k.gem /
+SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
 # ignore apk and gem pinning
 # hadolint ignore=DL3018,DL3028
-RUN chmod a+x /adduser.sh && \
+RUN chmod a+x /adduser.sh /docker-entrypoint.sh && \
 # Add a puppet user to run r10k as for consistency with puppetserver
     /adduser.sh && \
-    chmod +x /docker-entrypoint.sh && \
     chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \
     apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-json ruby-etc && \
     gem install --no-doc /r10k.gem && \
-    rm -f /r10k.gem
+    rm -f /r10k.gem && \
+    curl --fail --silent --show-error --location --remote-name "$supercronic_url" && \
+    echo "${supercronic_sha1sum}  ${supercronic}" | sha1sum -c - && \
+    chmod +x "$supercronic" && \
+    mv "$supercronic" "/usr/local/bin/${supercronic}" && \
+    ln -s "/usr/local/bin/${supercronic}" /usr/local/bin/supercronic
 
 USER puppet
 WORKDIR /home/puppet

data/docker/r10k/release.Dockerfile

@@ -1,4 +1,5 @@
-FROM alpine:3.9
+ARG alpine_version=3.9
+FROM alpine:${alpine_version}
 
 ARG vcs_ref
 ARG build_date
@@ -6,6 +7,11 @@ ARG version="3.1.0"
 # Used by entrypoint to submit metrics to Google Analytics.
 # Published images should use "production" for this build_arg.
 ARG pupperware_analytics_stream="dev"
+# required to schedule runs of "r10k" in K8s
+ARG supercronic_version="0.1.9"
+ARG supercronic_sha1sum="5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85"
+ARG supercronic="supercronic-linux-amd64"
+ARG supercronic_url="https://github.com/aptible/supercronic/releases/download/v$supercronic_version/$supercronic"
 
 LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
       org.label-schema.vendor="Puppet" \
@@ -16,7 +22,7 @@ LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
       org.label-schema.schema-version="1.0" \
       org.label-schema.dockerfile="/release.Dockerfile"
 
-COPY docker-entrypoint.sh /
+COPY adduser.sh docker-entrypoint.sh /
 COPY docker-entrypoint.d /docker-entrypoint.d
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
@@ -29,8 +35,21 @@ LABEL org.label-schema.version="$version" \
       org.label-schema.vcs-ref="$vcs_ref" \
       org.label-schema.build-date="$build_date"
 
-RUN chmod +x /docker-entrypoint.sh && \
+SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
+# ignore apk and gem pinning
+# hadolint ignore=DL3018,DL3028
+RUN chmod a+x /adduser.sh /docker-entrypoint.sh && \
+    /adduser.sh && \
+    chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \
     apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-dev make gcc musl-dev && \
-    gem install --no-doc r10k:"$version" json etc
+    gem install --no-doc r10k:"$version" json etc && \
+    curl --fail --silent --show-error --location --remote-name "$supercronic_url" && \
+    echo "${supercronic_sha1sum}  ${supercronic}" | sha1sum -c - && \
+    chmod +x "$supercronic" && \
+    mv "$supercronic" "/usr/local/bin/${supercronic}" && \
+    ln -s "/usr/local/bin/${supercronic}" /usr/local/bin/supercronic
+
+USER puppet
+WORKDIR /home/puppet
 
 COPY release.Dockerfile /

data/integration/tests/git_source/git_source_repeated_remote.rb

@@ -0,0 +1,68 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+test_name 'Verify the same remote can be used in more than one object'
+
+env_path = on(master, puppet('config print environmentpath')).stdout.rstrip
+r10k_fqp = get_r10k_fqp(master)
+git_environments_path = '/root/environments'
+git_repo_path = '/git_repos'
+git_repo_name = 'environments'
+git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git")
+code_dir = "#{env_path}/production"
+
+last_commit = git_last_commit(master, git_environments_path)
+git_provider = ENV['GIT_PROVIDER']
+r10k_config_path = get_r10k_config_file_path(master)
+r10k_config_bak_path = "#{r10k_config_path}.bak"
+#In-line files
+r10k_conf = <<-CONF
+cachedir: '/var/cache/r10k'
+git:
+  provider: '#{git_provider}'
+sources:
+  control:
+    basedir: "#{env_path}"
+    remote: "#{git_control_remote}"
+CONF
+
+# Install the same module in two different places
+puppetfile = <<-EOS
+mod 'prod_apache',
+  :git => 'git://github.com/puppetlabs/puppetlabs-apache.git',
+  :branch => 'master'
+
+mod 'test_apache',
+  :git => 'git://github.com/puppetlabs/puppetlabs-apache.git',
+  :branch => 'master'
+EOS
+
+teardown do
+  step 'Restore Original "r10k" Config'
+  on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}")
+
+  clean_up_r10k(master, last_commit, git_environments_path)
+end
+
+step 'Stub the forge'
+stub_forge_on(master)
+
+step 'Backup Current "r10k" Config'
+on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}")
+
+step 'Update the "r10k" Config'
+create_remote_file(master, r10k_config_path, r10k_conf)
+
+step 'Ask r10k to deploy'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Add puppetfile with repeated remote'
+create_remote_file(master, "#{git_environments_path}/Puppetfile", puppetfile)
+git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path)
+
+step 'Deploy r10k'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Verify module was installed in both places'
+on(master, "test -d #{code_dir}/modules/prod_apache")
+on(master, "test -d #{code_dir}/modules/test_apache")

data/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb

@@ -24,7 +24,7 @@ stage_env_notify_message = 'This is a different message'
 stage_env_notify_message_regex = /#{stage_env_notify_message}/
 
 #Verification for "test" Environment
-test_env_error_message_regex = /Error:.*Could not find environment 'test'/
+test_env_error_message_regex = /Error:.*Could not.*environment '?test'?/
 
 #Verification for "temp" Environment
 test_env_notify_message_regex = /I am in the temp environment/

data/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb

@@ -15,7 +15,7 @@ initial_env_names = ['production', 'stage']
 
 #Verification
 notify_message_regex = /I am in the production environment/
-stage_env_error_message_regex = /Error:.*Could not find environment 'stage'/
+stage_env_error_message_regex = /Error:.*Could not.*environment '?stage'?/
 
 #Manifest
 site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp')

data/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb

@@ -22,7 +22,7 @@ site_pp = create_site_pp(master_certname, '  include helloworld')
 notify_message_prod_env_regex = /I am in the production environment/
 notify_message_test_env_regex = /I am in the test environment/
 removal_message_test_env_regex = /Removing unmanaged path.*test/
-error_message_regex = /Could not retrieve catalog from remote server/
+error_message_regex = /Could not retrieve (catalog from remote server|information from environment test)/
 
 #Teardown
 teardown do

data/lib/r10k/action/deploy/environment.rb

@@ -188,6 +188,9 @@ module R10K
                       'no-force': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
+                      'puppet-conf': :self,
+                      'private-key': :self,
+                      'oauth-token': :self,
                       'default-branch-override': :self)
         end
       end

data/lib/r10k/action/deploy/module.rb

@@ -76,7 +76,10 @@ module R10K
                       cachedir: :self,
                       'no-force': :self,
                       'generate-types': :self,
-                      'puppet-path': :self)
+                      'puppet-path': :self,
+                      'puppet-conf': :self,
+                      'private-key': :self,
+                      'oauth-token': :self)
         end
       end
     end

data/lib/r10k/action/runner.rb

@@ -46,6 +46,7 @@ module R10K
         overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
         overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
         overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+        overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] unless @opts[:'puppet-conf'].nil?
         overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
@@ -54,6 +55,10 @@ module R10K
           newval
         end
 
+        # Credentials from the CLI override both the global and per-repo
+        # credentials from the config, and so need to be handled specially
+        with_overrides = add_credential_overrides(with_overrides)
+
         @settings = R10K::Settings.global_settings.evaluate(with_overrides)
 
         R10K::Initializers::GlobalInitializer.new(@settings).call
@@ -91,6 +96,35 @@ module R10K
 
         results
       end
+
+      def add_credential_overrides(overrides)
+        sshkey_path = @opts[:'private-key']
+        token_path = @opts[:'oauth-token']
+
+        if sshkey_path && token_path
+          raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy."
+        end
+
+        if sshkey_path
+          overrides[:git] ||= {}
+          overrides[:git][:private_key] = sshkey_path
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:private_key] = sshkey_path
+            end
+          end
+        elsif token_path
+          overrides[:git] ||= {}
+          overrides[:git][:oauth_token] = token_path
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:oauth_token] = token_path
+            end
+          end
+        end
+
+        overrides
+      end
     end
   end
 end

data/lib/r10k/cli/deploy.rb

@@ -21,7 +21,7 @@ module R10K::CLI
 (https://puppet.com/docs/puppet/latest/environments_about.html).
         DESCRIPTION
 
-        required nil, :cachedir, 'Specify a cachedir, overriding the value in config'
+        option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required
         flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
         flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment'
         option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd|
@@ -31,6 +31,9 @@ module R10K::CLI
             exit 1
           end
         end
+        option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required
+        option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required
+        option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required
 
         run do |opts, args, cmd|
           puts cmd.help(:verbose => opts[:verbose])
@@ -61,7 +64,8 @@ scheduled. On subsequent deployments, Puppetfile deployment will default to off.
           DESCRIPTION
 
           flag :p, :puppetfile, 'Deploy modules from a puppetfile'
-          required nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile'
+          option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile',
+                 argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Environment)
         end
@@ -81,7 +85,7 @@ It will load the Puppetfile configurations out of all environments, and will
 try to deploy the given module names in all environments.
           DESCRIPTION
 
-          required :e, :environment, 'Update the modules in the given environment'
+          option :e, :environment, 'Update the modules in the given environment', argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Module)
         end
@@ -99,7 +103,8 @@ try to deploy the given module names in all environments.
           flag :p, :puppetfile, 'Display Puppetfile modules'
           flag nil, :detail, 'Display detailed information'
           flag nil, :fetch, 'Update available environment lists from all remote sources'
-          required nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml'
+          option nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml',
+                 argument: :required
 
           runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Display)
         end

data/lib/r10k/cli/puppetfile.rb

@@ -30,8 +30,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           name    'install'
           usage   'install'
           summary 'Install all modules from a Puppetfile'
-          required nil, :moduledir, 'Path to install modules to'
-          required nil, :puppetfile, 'Path to puppetfile'
+          option nil, :moduledir, 'Path to install modules to', argument: :required
+          option nil, :puppetfile, 'Path to puppetfile', argument: :required
           flag     nil, :force, 'Force locally changed files to be overwritten'
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Install)
         end
@@ -45,7 +45,7 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           usage 'check'
           summary 'Try and load the Puppetfile to verify the syntax is correct.'
 
-          required nil, :puppetfile, 'Path to Puppetfile'
+          option nil, :puppetfile, 'Path to Puppetfile', argument: :required
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Check)
         end
       end
@@ -58,8 +58,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
           usage 'purge'
           summary 'Purge unmanaged modules from a Puppetfile managed directory'
 
-          required nil, :moduledir, 'Path to install modules to'
-          required nil, :puppetfile, 'Path to Puppetfile'
+          option nil, :moduledir, 'Path to install modules to', argument: :required
+          option nil, :puppetfile, 'Path to Puppetfile', argument: :required
           runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Purge)
         end
       end