r10k 3.4.0 → 3.6.0

data/doc/faq.mkd

@@ -141,7 +141,7 @@ modulepath = modules:external-modules
 
 Lastly, you can simply move your locally versioned modules to a separate
 directory to avoid conflicting over the `/modules` directory entirely. With this
-example as well you can use the `environment.conf file to tell Puppet which
+example as well you can use the `environment.conf` file to tell Puppet which
 directories contain modules.
 
 ```
@@ -149,6 +149,11 @@ directories contain modules.
 modulepath = internal-modules:modules
 ```
 
+#### Does R10K support Local/Private Forge?
+
+Yes. Set the Forge to use _globally_ in `r10k.yaml`. see [Configuration](/doc/dynamic-environments/configuration.mkd#baseurl) for details.
+
+
 #### What does the name mean?
 
 It’s called R10K because I’m terrible at names. When I started working on R10K

data/doc/puppetfile.mkd

@@ -54,6 +54,8 @@ The `forge` setting specifies which server that Forge based modules are fetched
 from. This is currently a noop and is provided for compatibility with
 librarian-puppet.
 
+R10k supports setting the Forge to use _globally_ in `r10k.yaml`. see [Configuration](/doc/dynamic-environments/configuration.mkd#baseurl) for details.
+
 ### moduledir
 
 The `moduledir` setting specifies where modules from the Puppetfile will be

data/docker/Makefile

@@ -1,17 +1,27 @@
 PUPPERWARE_ANALYTICS_STREAM ?= dev
 NAMESPACE ?= puppet
-git_describe = $(shell git describe)
+git_describe = $(shell git describe --tags)
 vcs_ref := $(shell git rev-parse HEAD)
 build_date := $(shell date -u +%FT%T)
 hadolint_available := $(shell hadolint --help > /dev/null 2>&1; echo $$?)
-hadolint_command := hadolint --ignore DL3008 --ignore DL3018 --ignore DL3028 --ignore DL4000 --ignore DL4001
+hadolint_command := hadolint
 hadolint_container := hadolint/hadolint:latest
+alpine_version := 3.9
 export BUNDLE_PATH = $(PWD)/.bundle/gems
 export BUNDLE_BIN = $(PWD)/.bundle/bin
 export GEMFILE = $(PWD)/Gemfile
+export DOCKER_BUILDKIT = 1
 
 ifeq ($(IS_RELEASE),true)
 	VERSION ?= $(shell echo $(git_describe) | sed 's/-.*//')
+	PUBLISHED_VERSION ?= $(shell curl --silent 'https://rubygems.org/api/v1/gems/r10k.json' | jq '."version"' | tr -d '"')
+	CONTAINER_EXISTS = $(shell DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect $(NAMESPACE)/r10k:$(VERSION) > /dev/null 2>&1; echo $$?)
+ifeq ($(CONTAINER_EXISTS),0)
+	SKIP_BUILD ?= true
+else ifneq ($(VERSION),$(PUBLISHED_VERSION))
+	SKIP_BUILD ?= true
+endif
+
 	LATEST_VERSION ?= latest
 	dockerfile := release.Dockerfile
 	dockerfile_context := r10k
@@ -25,6 +35,10 @@ endif
 prep:
 	@git fetch --unshallow 2> /dev/null ||:
 	@git fetch origin 'refs/tags/*:refs/tags/*'
+ifeq ($(SKIP_BUILD),true)
+	@echo "SKIP_BUILD is true, exiting with 1"
+	@exit 1
+endif
 
 lint:
 ifeq ($(hadolint_available),0)
@@ -35,8 +49,10 @@ else
 endif
 
 build: prep
+	docker pull alpine:$(alpine_version)
 	docker build \
-		--pull \
+		${DOCKER_BUILD_FLAGS} \
+		--build-arg alpine_version=$(alpine_version) \
 		--build-arg vcs_ref=$(vcs_ref) \
 		--build-arg build_date=$(build_date) \
 		--build-arg version=$(VERSION) \

data/docker/r10k/Dockerfile

@@ -1,5 +1,7 @@
-FROM alpine:3.9 as build
+ARG alpine_version=3.9
+FROM alpine:${alpine_version} as build
 
+# hadolint ignore=DL3018
 RUN apk add --no-cache ruby git && \
     mkdir /workspace
 WORKDIR /workspace
@@ -7,7 +9,7 @@ COPY . /workspace
 RUN gem build r10k.gemspec && \
     mv r10k*.gem r10k.gem
 
-FROM alpine:3.9
+FROM alpine:${alpine_version}
 
 ARG vcs_ref
 ARG build_date
@@ -15,6 +17,11 @@ ARG version="3.1.0"
 # Used by entrypoint to submit metrics to Google Analytics.
 # Published images should use "production" for this build_arg.
 ARG pupperware_analytics_stream="dev"
+# required to schedule runs of "r10k" in K8s
+ARG supercronic_version="0.1.9"
+ARG supercronic_sha1sum="5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85"
+ARG supercronic="supercronic-linux-amd64"
+ARG supercronic_url="https://github.com/aptible/supercronic/releases/download/v$supercronic_version/$supercronic"
 
 LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
       org.label-schema.vendor="Puppet" \
@@ -31,7 +38,7 @@ COPY docker/r10k/docker-entrypoint.d /docker-entrypoint.d
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["help"]
 
-# dyanmic LABELs and ENV vars placed lower for the sake of Docker layer caching
+# dynamic LABELs and ENV vars placed lower for the sake of Docker layer caching
 ENV PUPPERWARE_ANALYTICS_STREAM="$pupperware_analytics_stream"
 
 LABEL org.label-schema.version="$version" \
@@ -39,14 +46,21 @@ LABEL org.label-schema.version="$version" \
       org.label-schema.build-date="$build_date"
 
 COPY --from=build /workspace/r10k.gem /
-RUN chmod a+x /adduser.sh && \
+SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
+# ignore apk and gem pinning
+# hadolint ignore=DL3018,DL3028
+RUN chmod a+x /adduser.sh /docker-entrypoint.sh && \
 # Add a puppet user to run r10k as for consistency with puppetserver
     /adduser.sh && \
-    chmod +x /docker-entrypoint.sh && \
     chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \
-    apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-dev make gcc musl-dev && \
-    gem install --no-doc /r10k.gem json etc && \
-    rm -f /r10k.gem
+    apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-json ruby-etc && \
+    gem install --no-doc /r10k.gem && \
+    rm -f /r10k.gem && \
+    curl --fail --silent --show-error --location --remote-name "$supercronic_url" && \
+    echo "${supercronic_sha1sum}  ${supercronic}" | sha1sum -c - && \
+    chmod +x "$supercronic" && \
+    mv "$supercronic" "/usr/local/bin/${supercronic}" && \
+    ln -s "/usr/local/bin/${supercronic}" /usr/local/bin/supercronic
 
 USER puppet
 WORKDIR /home/puppet

data/docker/r10k/release.Dockerfile

@@ -1,4 +1,5 @@
-FROM alpine:3.9
+ARG alpine_version=3.9
+FROM alpine:${alpine_version}
 
 ARG vcs_ref
 ARG build_date
@@ -6,6 +7,11 @@ ARG version="3.1.0"
 # Used by entrypoint to submit metrics to Google Analytics.
 # Published images should use "production" for this build_arg.
 ARG pupperware_analytics_stream="dev"
+# required to schedule runs of "r10k" in K8s
+ARG supercronic_version="0.1.9"
+ARG supercronic_sha1sum="5ddf8ea26b56d4a7ff6faecdd8966610d5cb9d85"
+ARG supercronic="supercronic-linux-amd64"
+ARG supercronic_url="https://github.com/aptible/supercronic/releases/download/v$supercronic_version/$supercronic"
 
 LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
       org.label-schema.vendor="Puppet" \
@@ -16,7 +22,7 @@ LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
       org.label-schema.schema-version="1.0" \
       org.label-schema.dockerfile="/release.Dockerfile"
 
-COPY docker-entrypoint.sh /
+COPY adduser.sh docker-entrypoint.sh /
 COPY docker-entrypoint.d /docker-entrypoint.d
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
@@ -29,8 +35,21 @@ LABEL org.label-schema.version="$version" \
       org.label-schema.vcs-ref="$vcs_ref" \
       org.label-schema.build-date="$build_date"
 
-RUN chmod +x /docker-entrypoint.sh && \
+SHELL ["/bin/ash", "-eo", "pipefail", "-c"]
+# ignore apk and gem pinning
+# hadolint ignore=DL3018,DL3028
+RUN chmod a+x /adduser.sh /docker-entrypoint.sh && \
+    /adduser.sh && \
+    chown -R puppet: /docker-entrypoint.d /docker-entrypoint.sh && \
     apk add --no-cache ruby openssh-client git ruby-rugged curl ruby-dev make gcc musl-dev && \
-    gem install --no-doc r10k:"$version" json etc
+    gem install --no-doc r10k:"$version" json etc && \
+    curl --fail --silent --show-error --location --remote-name "$supercronic_url" && \
+    echo "${supercronic_sha1sum}  ${supercronic}" | sha1sum -c - && \
+    chmod +x "$supercronic" && \
+    mv "$supercronic" "/usr/local/bin/${supercronic}" && \
+    ln -s "/usr/local/bin/${supercronic}" /usr/local/bin/supercronic
+
+USER puppet
+WORKDIR /home/puppet
 
 COPY release.Dockerfile /

data/integration/Rakefile

@@ -44,8 +44,8 @@ EOS
 
   t.add_env do |env|
     env.name = 'pe_dist_dir'
-    env.message = 'The location to download PE from example "http://enterprise.delivery.puppetlabs.net/20XX.X/ci-ready"'
-    ENV['pe_dist_dir'] ||= "http://enterprise.delivery.puppetlabs.net/#{ENV['PE_FAMILY']}/ci-ready"
+    env.message = 'The location to download PE from, for example "https://artifactory.delivery.puppetlabs.net/artifactory/generic_enterprise__local/20XX.X/ci-ready"'
+    ENV['pe_dist_dir'] ||= "https://artifactory.delivery.puppetlabs.net/artifactory/generic_enterprise__local/#{ENV['PE_FAMILY']}/ci-ready"
   end
 
   t.add_env do |env|

data/integration/tests/git_source/git_source_repeated_remote.rb

@@ -0,0 +1,68 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+test_name 'Verify the same remote can be used in more than one object'
+
+env_path = on(master, puppet('config print environmentpath')).stdout.rstrip
+r10k_fqp = get_r10k_fqp(master)
+git_environments_path = '/root/environments'
+git_repo_path = '/git_repos'
+git_repo_name = 'environments'
+git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git")
+code_dir = "#{env_path}/production"
+
+last_commit = git_last_commit(master, git_environments_path)
+git_provider = ENV['GIT_PROVIDER']
+r10k_config_path = get_r10k_config_file_path(master)
+r10k_config_bak_path = "#{r10k_config_path}.bak"
+#In-line files
+r10k_conf = <<-CONF
+cachedir: '/var/cache/r10k'
+git:
+  provider: '#{git_provider}'
+sources:
+  control:
+    basedir: "#{env_path}"
+    remote: "#{git_control_remote}"
+CONF
+
+# Install the same module in two different places
+puppetfile = <<-EOS
+mod 'prod_apache',
+  :git => 'git://github.com/puppetlabs/puppetlabs-apache.git',
+  :branch => 'master'
+
+mod 'test_apache',
+  :git => 'git://github.com/puppetlabs/puppetlabs-apache.git',
+  :branch => 'master'
+EOS
+
+teardown do
+  step 'Restore Original "r10k" Config'
+  on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}")
+
+  clean_up_r10k(master, last_commit, git_environments_path)
+end
+
+step 'Stub the forge'
+stub_forge_on(master)
+
+step 'Backup Current "r10k" Config'
+on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}")
+
+step 'Update the "r10k" Config'
+create_remote_file(master, r10k_config_path, r10k_conf)
+
+step 'Ask r10k to deploy'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Add puppetfile with repeated remote'
+create_remote_file(master, "#{git_environments_path}/Puppetfile", puppetfile)
+git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path)
+
+step 'Deploy r10k'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Verify module was installed in both places'
+on(master, "test -d #{code_dir}/modules/prod_apache")
+on(master, "test -d #{code_dir}/modules/test_apache")

data/lib/r10k/action/deploy/environment.rb

@@ -157,7 +157,10 @@ module R10K
             logger.debug("Unable to get environment module deploy data for .r10k-deploy.json at #{environment.path}")
           end
 
-          File.open("#{environment.path}/.r10k-deploy.json", 'w') do |f|
+          # make this file write as atomic as possible in pure ruby
+          final   = "#{environment.path}/.r10k-deploy.json"
+          staging = "#{environment.path}/.r10k-deploy.json~"
+          File.open(staging, 'w') do |f|
             deploy_info = environment.info.merge({
               :started_at => started_at,
               :finished_at => Time.new,
@@ -167,6 +170,7 @@ module R10K
 
             f.puts(JSON.pretty_generate(deploy_info))
           end
+          FileUtils.mv(staging, final)
         end
 
         def undeployable_environment_names(environments, expected_names)

data/lib/r10k/action/deploy/module.rb

@@ -72,7 +72,11 @@ module R10K
         end
 
         def allowed_initialize_opts
-          super.merge(environment: true, 'no-force': :self, 'generate-types': :self, 'puppet-path': :self)
+          super.merge(environment: true,
+                      cachedir: :self,
+                      'no-force': :self,
+                      'generate-types': :self,
+                      'puppet-path': :self)
         end
       end
     end

data/lib/r10k/action/runner.rb

@@ -43,10 +43,10 @@ module R10K
         config_settings = settings_from_config(@opts[:config])
 
         overrides = {}
-        overrides[:cachedir] = @opts[:cachedir] unless @opts[:cachedir].nil?
-        overrides[:deploy] = {} if !@opts[:'puppet-path'].nil? || !@opts[:'generate-types'].nil?
-        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] unless @opts[:'puppet-path'].nil?
-        overrides[:deploy][:generate_types] = @opts[:'generate-types'] unless @opts[:'generate-types'].nil?
+        overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
+        overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
+        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+        overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash

data/lib/r10k/cli/deploy.rb

@@ -18,7 +18,7 @@ module R10K::CLI
 
         description <<-DESCRIPTION
 `r10k deploy` implements the Git branch to Puppet environment workflow
-(https://puppetlabs.com/blog/git-workflow-and-puppet-environments/).
+(https://puppet.com/docs/puppet/latest/environments_about.html).
         DESCRIPTION
 
         required nil, :cachedir, 'Specify a cachedir, overriding the value in config'

data/lib/r10k/forge/module_release.rb

@@ -212,14 +212,14 @@ module R10K
 
       # Remove the temporary directory used for unpacking the module.
       def cleanup_unpack_path
-        if unpack_path.exist?
+        if unpack_path.parent.exist?
           unpack_path.parent.rmtree
         end
       end
 
       # Remove the downloaded module release.
       def cleanup_download_path
-        if download_path.exist?
+        if download_path.parent.exist?
           download_path.parent.rmtree
         end
       end

data/lib/r10k/git/cache.rb

@@ -99,10 +99,8 @@ class R10K::Git::Cache
 
   alias cached? exist?
 
-  private
-
   # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @remote.gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-')
   end
 end

data/lib/r10k/git/stateful_repository.rb

@@ -12,6 +12,10 @@ class R10K::Git::StatefulRepository
   #   @api private
   attr_reader :repo
 
+  # @!attribute [r] cache
+  #   @api private
+  attr_reader :cache
+
   extend Forwardable
   def_delegators :@repo, :head, :tracked_paths
 

data/lib/r10k/module/base.rb

@@ -99,6 +99,14 @@ class R10K::Module::Base
     raise NotImplementedError
   end
 
+  # Return the module's cachedir. Subclasses that implement a cache
+  # will override this to return a real directory location.
+  #
+  # @return [String, :none]
+  def cachedir
+    :none
+  end
+
   private
 
   def parse_title(title)

data/lib/r10k/module/git.rb

@@ -57,6 +57,10 @@ class R10K::Module::Git < R10K::Module::Base
     @repo.status(version)
   end
 
+  def cachedir
+    @repo.cache.sanitized_dirname
+  end
+
   private
 
   def validate_ref(desired, default)

data/lib/r10k/puppetfile.rb

@@ -10,7 +10,7 @@ class Puppetfile
 
   include R10K::Settings::Mixin
 
-  def_setting_attr :pool_size, 1
+  def_setting_attr :pool_size, 4
 
   include R10K::Logging
 
@@ -42,6 +42,11 @@ class Puppetfile
   #   @return [Boolean] Overwrite any locally made changes
   attr_accessor :force
 
+  # @!attribute [r] modules_by_vcs_cachedir
+  #   @api private Only exposed for testing purposes
+  #   @return [Hash{:none, String => Array<R10K::Module>}]
+  attr_reader :modules_by_vcs_cachedir
+
   # @param [String] basedir
   # @param [String] moduledir The directory to install the modules, default to #{basedir}/modules
   # @param [String] puppetfile_path The path to the Puppetfile, default to #{basedir}/Puppetfile
@@ -58,6 +63,7 @@ class Puppetfile
 
     @modules = []
     @managed_content = {}
+    @modules_by_vcs_cachedir = {}
     @forge   = 'forgeapi.puppetlabs.com'
 
     @loaded = false
@@ -137,6 +143,9 @@ class Puppetfile
     mod.origin = 'Puppetfile'
 
     @managed_content[install_path] << mod.name
+    cachedir = mod.cachedir
+    @modules_by_vcs_cachedir[cachedir] ||= []
+    @modules_by_vcs_cachedir[cachedir] << mod
     @modules << mod
   end
 
@@ -145,7 +154,9 @@ class Puppetfile
   def managed_directories
     self.load unless @loaded
 
-    @managed_content.keys
+    dirs = @managed_content.keys
+    dirs.delete(real_basedir)
+    dirs
   end
 
   # Returns an array of the full paths to all the content being managed.
@@ -212,15 +223,22 @@ class Puppetfile
   def modules_queue(visitor)
     Queue.new.tap do |queue|
       visitor.visit(:puppetfile, self) do
-        modules.each { |mod| queue << mod }
+        modules_by_cachedir = modules_by_vcs_cachedir.clone
+        modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
+
+        modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
+        modules_by_cachedir.values.each {|mods| queue << mods }
       end
     end
   end
+  public :modules_queue
 
   def visitor_thread(visitor, mods_queue)
     Thread.new do
       begin
-        while mod = mods_queue.pop(true) do mod.accept(visitor) end
+        while mods = mods_queue.pop(true) do
+          mods.each {|mod| mod.accept(visitor) }
+        end
       rescue ThreadError => e
         logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
         Thread.exit
@@ -248,8 +266,6 @@ class Puppetfile
   end
 
   def validate_install_path(path, modname)
-    real_basedir = Pathname.new(basedir).cleanpath.to_s
-
     unless /^#{Regexp.escape(real_basedir)}.*/ =~ path
       raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}")
     end
@@ -257,6 +273,10 @@ class Puppetfile
     true
   end
 
+  def real_basedir
+    Pathname.new(basedir).cleanpath.to_s
+  end
+
   class DSL
     # A barebones implementation of the Puppetfile DSL
     #