r10k 3.12.0 → 3.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f378346234a9e8a8d92b3acb64735f88ca6b8e8edd60aa33adad9a09d7efd537
-  data.tar.gz: 1b94403eb2e5cb420939d85b73ec299c709418afabb45705f70b1cfb8606a1f5
+  metadata.gz: 51cffcbc4e3340cf55e51da3a60e66e4c4b851d17bdd566d257213e6a4244bdc
+  data.tar.gz: 10c04f87924b3c5f4eb84cba797879a0a1ac09a6fbc3e0f598c53788d1d5f9b2
 SHA512:
-  metadata.gz: a3eda38f3fc6e79ea62082c7c71bd65383089dce82e87dc6473c1f278f1560239f3e2189df28ef23a9ce8e47f3446cdd63e8794344ee62895914353bdcdec6ee
-  data.tar.gz: 0dc89955ede8226bf59c7cd4f8e03d9a89688700f5767d0578884fce97643c937f3363dd0d965404fd2b900128b181874dfb129952d64891a3700516239a2af6
+  metadata.gz: 8684fc987a031207807de406810ec0529ef00ab8cad321923451a51d5d869ec2aee3cbca411ca37dbe1bbc7ec3c4a0e7daa58df099b0bc1c74005e5c914e0ba2
+  data.tar.gz: b8515912cdc5099dfd5c238da01663f7352185bd8454363cdce4bf562138302002fd563fe7783c212071fee2a8855f5abc78dd7d66c1dc006595613b697cd8eb

data/.github/workflows/docker.yml

@@ -3,7 +3,7 @@ name: Docker test and publish
 on:
   push:
     branches:
-      - master
+      - main
 
 jobs:
   build-and-publish:

data/.github/workflows/rspec_tests.yml

@@ -16,9 +16,9 @@ jobs:
           - {os: ubuntu-18.04, ruby: 2.6}
           - {os: ubuntu-18.04, ruby: 2.7}
           - {os: ubuntu-18.04, ruby: jruby-9.2.10.0}
-          - {os: windows-2016, ruby: 2.5}
-          - {os: windows-2016, ruby: 2.6}
-          - {os: windows-2016, ruby: 2.7}
+          - {os: windows-2019, ruby: 2.5}
+          - {os: windows-2019, ruby: 2.6}
+          - {os: windows-2019, ruby: 2.7}
 
     runs-on: ${{ matrix.cfg.os }}
     steps:
@@ -59,7 +59,7 @@ jobs:
           bundle --version
 
           # Run tests
-          bundle exec rspec --color --format documentation spec/unit
+          bundle exec rspec --color --format documentation spec
 
       - name: Run tests on Linux
         if: runner.os == 'Linux'
@@ -78,4 +78,4 @@ jobs:
           fi
 
           # Run tests
-          bundle exec rspec --color --format documentation spec/unit
+          bundle exec rspec --color --format documentation spec

data/.github/workflows/stale.yml

@@ -15,5 +15,7 @@ jobs:
         days-before-close: 7
         stale-issue-message: 'This issue has been marked stale because it has had no activity for 60 days. The Puppet Team is actively prioritizing existing bugs and new features, if this issue is still important to you please comment and we will add this to our backlog to complete. Otherwise, it will be closed in 7 days.'
         stale-issue-label: 'stale'
+        exempt-issue-labels: 'community interest'
         stale-pr-message: "This PR has been marked stale because it has had no activity for 60 days. If you are still interested in getting this merged, please comment and we'll try to move it forward. Otherwise, it will be closed in 7 days."
         stale-pr-label: 'stale'
+        exempt-pr-labels: 'community interest'

data/.gitignore

@@ -7,3 +7,4 @@ coverage
 integration/log
 integration/junit
 integration/configs
+r10k.log

data/.travis.yml

@@ -1,7 +1,7 @@
 ---
 language: ruby
 bundler_args: "--without system"
-script: "bundle exec rspec --color --format documentation spec/unit"
+script: "bundle exec rspec --color --format documentation spec"
 notifications:
   email: false
 sudo: false

data/CHANGELOG.mkd

@@ -4,6 +4,39 @@ CHANGELOG
 Unreleased
 ----------
 
+3.14.1
+------
+
+- (RK-395) Make `exclude_spec` from a Puppetfile the priority override [#1271](https://github.com/puppetlabs/r10k/issues/1271)
+- (RK-394) Fix `force` always resolving to true for `puppetfile install` [#1269](https://github.com/puppetlabs/r10k/issues/1265)
+- (RK-393) Bug fix: not all spec directories are deleted when :exclude_spec is true [#1267](https://github.com/puppetlabs/r10k/pull/1267)
+- Refactor internal module creation to always expect a hash, even for Forge modules, which can be specified in the Puppetfile with just a version string. [#1170](https://github.com/puppetlabs/r10k/pull/1170)
+
+3.14.0
+------
+
+- Record unprocessed environment name, so that `strip_component` does not cause truncated environment names to be used as git branches, resulting in errors or incorrect deploys. [#1240](https://github.com/puppetlabs/r10k/pull/1240)
+- (CODEMGMT-1294) Resync repos with unresolvable refs [#1239](https://github.com/puppetlabs/r10k/pull/1239)
+- (RK-378) Restore access to the environment name from the Puppetfile [#1241](https://github.com/puppetlabs/r10k/pull/1241)
+- (CODEMGMT-1300) Ensure the remote url in rugged cache directories is current [#1245](https://github.com/puppetlabs/r10k/pull/1245)
+- Add support for tarball module type, allowing module content to be packaged and sourced from generic fileservers [#1244](https://github.com/puppetlabs/r10k/pull/1244)
+- Add experimental support for tarball environment type, allowing whole environments to be packaged and sourced from generic fileservers [#1244](https://github.com/puppetlabs/r10k/pull/1244)
+
+3.13.0
+------
+
+- Restore Ruby 3 compatibility [#1234](https://github.com/puppetlabs/r10k/pull/1234)
+- (RK-381) Do not recurse into symlinked dirs when finding files to purge. [#1233](https://github.com/puppetlabs/r10k/pull/1233)
+- Purge should remove unmanaged directories, in addition to unmanaged files. [#1222](https://github.com/puppetlabs/r10k/pull/1222)
+- Rename experimental environment type "bare" to "plain". [#1228](https://github.com/puppetlabs/r10k/pull/1228)
+- Add support for specifying additional logging ouputs. [#1230](https://github.com/puppetlabs/r10k/issues/1230)
+
+3.12.1
+------
+
+- Fix requiring individual R10K::Actions without having already required 'r10k'. [#1223](https://github.com/puppetlabs/r10k/issues/1223)
+- Fix evaluation of Puppetfiles that include local modules. [#1224](https://github.com/puppetlabs/r10k/pull/1224)
+
 3.12.0
 ------
 

data/doc/dynamic-environments/configuration.mkd

@@ -128,6 +128,18 @@ git:
   private_key: "/etc/puppetlabs/r10k/ssh/id_rsa"
 ```
 
+#### oauth_token
+
+The oauth_token setting is only used by the Rugged git provider.
+
+The oauth_token option specifies the path to the default access token for Git HTTPS remotes.
+Public git repositories can be accessed via HTTPS without authentication, but the oauth_token setting may be set if any non-public HTTPS remotes are used.
+
+```yaml
+git:
+  oauth_token: "/etc/puppetlabs/r10k/token"
+```
+
 #### repositories
 
 The repositories option allows configuration to be set on a per-remote basis. Each entry is a map of
@@ -145,6 +157,18 @@ git:
       private_key: "/etc/puppetlabs/r10k/ssh/id_rsa-protected-repo-deploy-key"
 ```
 
+##### oauth_token
+
+A repository specific access token to use for HTTPS connections for the given repository URL. This
+overrides the global oauth_token setting.
+
+```yaml
+git:
+  repositories:
+    - remote: "https://tessier-ashpool.freeside/protected-repo.git"
+      oauth_token: "/etc/puppetlabs/r10k/protected-repo-deploy-token"
+```
+
 ##### proxy
 
 The 'proxy' setting allows you to set or override the global proxy setting for a single, specific
@@ -349,12 +373,13 @@ deploy:
 
 #### exclude_spec
 
-During module deployment, r10k's default behavior is to deploy the spec directory.
-This behavior can be configured for all modules using the `exclude_spec` setting in the r10k config.
-This config setting can be overridden in each module definition in the Puppetfile
-or more globally via the CLI for deploys. The following example sets all modules to
-not deploy the spec dir.
-
+During module deployment, r10k's default behavior is to deploy the spec directory. Setting
+`exclude_spec` to true will deploy modules without their spec directory. This behavior
+can be configured for all modules using the `exclude_spec` setting in the r10k config.
+It can also be passed as a CLI argument for `deploy environment/module`, overriding the
+r10k config. Setting this per module in a `Puppetfile` will override the default, r10k config,
+and cli flag for that module. The following example sets all modules to not deploy the spec
+dir via the r10k config.
 
 ```yaml
 deploy:
@@ -760,16 +785,16 @@ modules:
     version: 62d07f2
 ```
 
-### Bare Environment Type
+### Plain Environment Type
 
 A "control repository" typically contains a hiera.yaml, an environment.conf, a manifests/site.pp file, and a few other things. However, none of these are strictly necessary for an environment to be functional if modules can be deployed to it.
 
-The bare environment type allows sources that support environment modules to operate without a control repo being required. Modules can be deployed directly.
+The plain environment type allows sources that support environment modules to operate without a control repo being required. Modules can be deployed directly.
 
 ```yaml
 ---
 production:
-  type: bare
+  type: plain
   modules:
     puppetlabs-stdlib:
       type: forge
@@ -783,7 +808,7 @@ production:
       version: 62d07f2
 
 development:
-  type: bare
+  type: plain
   modules:
     puppetlabs-stdlib:
       type: forge
@@ -796,3 +821,27 @@ development:
       source: https://github.com/reidmv/reidmv-xampl.git
       version: 62d07f2
 ```
+
+### Tarball Environment Type
+
+The tarball environment type allows an environment to be deployed from a tarball archive, rather than a Git repository. When using a tarball environment type, a source location for the tarball is required. Optionally, the tarball's sha256 checksum may be specified as the version. It is highly recommended to include a version specifier. If a version specifier is not included, r10k will never invalidate a cached copy of the tarball's source.
+
+Tarball environment sources will be unpacked directly into the environment root.
+
+```yaml
+---
+production:
+  type: tarball
+  source: https://repo.example.com/projects/puppet/env-2.36.1.tar.gz
+  version: 99a906c99c2f144de43f2ae500509a7474ed11c583fb623efa8e5b377a3157f0 # sha256digest
+
+development:
+  type: tarball
+  source: https://repo.example.com/projects/puppet/env-6128ada.tar.gz
+  version: 6128ada158622cd90f8e1360fb7c2c3830a812d1ec26ddf0db7eb16d61b7293f # sha256digest
+  modules:
+    reidmv-xampl:
+      type: git
+      source: https://github.com/reidmv/reidmv-xampl.git
+      version: 62d07f2
+```

data/doc/git/providers.mkd

@@ -56,6 +56,28 @@ git:
       private_key: '/root/.ssh/private_repo_id'
 ```
 
+### HTTPS Configuration
+
+Public HTTPS based Git repositories can be accessed with no additional settings.
+For repos that do require authentication, the 'oauth_token' option may be provided.
+
+```yaml
+git:
+  oauth_token: '/etc/puppetlabs/r10k/token'
+```
+
+If you have per repository access tokens you can add them with the repositories list.
+
+```yaml
+git:
+  # default access token
+  oauth_token: '/etc/puppetlabs/r10k/token'
+  repositories:
+    - remote: "https://github.com/my_org/private_repo.git"
+      # access token for this repo only
+      oauth_token: '/etc/puppetlabs/r10k/private_repo_token'
+```
+
 #### Supported transports with Rugged
 
 Rugged compiles libgit2 and and the Ruby bindings when the gem is installed. You

data/doc/puppetfile.mkd

@@ -239,6 +239,19 @@ credentials may be visible in the process table when r10k is running. If you
 choose to supply SVN credentials make sure that the system running r10k is
 appropriately secured.
 
+### Tarball
+
+Modules can be installed from tarball archives. A tarball module must specify a source URL to retreive the tarball content from. A tarball module may optionally specify a sha256 checksum as the module version.
+
+    mod 'puppetlabs-apache',
+      type: 'tarball',
+      source: 'https://repo.example.com/puppet/modules/puppetlabs-apache-7.0.0.tar.gz',
+      version: 'aedd6dc1a5136c6a1a1ec2f285df2a70b0fe4c9effb254b5a1f58116e4c1659e' # sha256 digest
+
+If no version is specified, a tarball will be downloaded from the given source and cached. The cache will not be invalidated until the source URL is changed, or a sha256 checksum version is provided.
+
+Tarball module content will be unpacked directly into an appropriately named module directory. For example, the puppetlabs-apache-7.0.0.tar.gz archive in the example above will be unpacked into `<environment-dir>/modules/apache/`.
+
 ### Local
 
 In the event you want to store locally written modules in your r10k-managed

data/docker/Makefile

@@ -6,7 +6,7 @@ build_date := $(shell date -u +%FT%T)
 hadolint_available := $(shell hadolint --help > /dev/null 2>&1; echo $$?)
 hadolint_command := hadolint
 hadolint_container := ghcr.io/hadolint/hadolint:latest
-alpine_version := 3.9
+alpine_version := 3.14
 export BUNDLE_PATH = $(PWD)/.bundle/gems
 export BUNDLE_BIN = $(PWD)/.bundle/bin
 export GEMFILE = $(PWD)/Gemfile

data/docker/r10k/Dockerfile

@@ -1,4 +1,4 @@
-ARG alpine_version=3.9
+ARG alpine_version=3.14
 FROM alpine:${alpine_version} as build
 
 # hadolint ignore=DL3018

data/docker/r10k/release.Dockerfile

@@ -1,4 +1,4 @@
-ARG alpine_version=3.9
+ARG alpine_version=3.14
 FROM alpine:${alpine_version}
 
 ARG vcs_ref

data/lib/r10k/action/base.rb

@@ -1,5 +1,5 @@
-require 'r10k/util/setopts'
 require 'r10k/logging'
+require 'r10k/util/setopts'
 
 module R10K
   module Action

data/lib/r10k/action/deploy/deploy_helpers.rb

@@ -1,8 +1,12 @@
+require 'r10k/logging'
+
 module R10K
   module Action
     module Deploy
       module DeployHelpers
 
+        include R10K::Logging
+
         # Ensure that a config file has been found (and presumably loaded) and exit
         # with a helpful error if it hasn't.
         #

data/lib/r10k/action/deploy/display.rb

@@ -1,6 +1,6 @@
-require 'r10k/deployment'
 require 'r10k/action/base'
 require 'r10k/action/deploy/deploy_helpers'
+require 'r10k/deployment'
 
 module R10K
   module Action

data/lib/r10k/action/deploy/environment.rb

@@ -1,10 +1,9 @@
-require 'r10k/util/setopts'
-require 'r10k/util/cleaner'
-require 'r10k/deployment'
-require 'r10k/logging'
-require 'r10k/action/visitor'
 require 'r10k/action/base'
 require 'r10k/action/deploy/deploy_helpers'
+require 'r10k/action/visitor'
+require 'r10k/deployment'
+require 'r10k/util/setopts'
+
 require 'json'
 
 module R10K
@@ -13,6 +12,7 @@ module R10K
       class Environment < R10K::Action::Base
 
         include R10K::Action::Deploy::DeployHelpers
+        include R10K::Action::Visitor
 
         # Deprecated
         attr_reader :force
@@ -81,8 +81,6 @@ module R10K
           @visit_ok
         end
 
-        include R10K::Action::Visitor
-
         private
 
         def read_purge_allowlist (whitelist, allowlist)

data/lib/r10k/action/deploy/module.rb

@@ -1,7 +1,7 @@
-require 'r10k/deployment'
-require 'r10k/action/visitor'
 require 'r10k/action/base'
 require 'r10k/action/deploy/deploy_helpers'
+require 'r10k/action/visitor'
+require 'r10k/deployment'
 
 module R10K
   module Action
@@ -9,6 +9,7 @@ module R10K
       class Module < R10K::Action::Base
 
         include R10K::Action::Deploy::DeployHelpers
+        include R10K::Action::Visitor
 
         # Deprecated
         attr_reader :force
@@ -69,8 +70,6 @@ module R10K
           @visit_ok
         end
 
-        include R10K::Action::Visitor
-
         private
 
         def visit_deployment(deployment)

data/lib/r10k/action/runner.rb

@@ -51,6 +51,12 @@ module R10K
           overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
           overrides[:deploy][:exclude_spec] = @opts[:'exclude-spec'] if @opts.key?(:'exclude-spec')
         end
+        # If the log level has been given as an argument, ensure that output happens on stderr
+        if @opts.key?(:loglevel)
+          overrides[:logging] = {}
+          overrides[:logging][:level] = @opts[:loglevel]
+          overrides[:logging][:disable_default_stderr] = false
+        end
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash

data/lib/r10k/action/visitor.rb

@@ -1,4 +1,5 @@
 require 'r10k/errors/formatting'
+require 'r10k/logging'
 
 module R10K
   module Action
@@ -13,6 +14,8 @@ module R10K
     # @api private
     module Visitor
 
+      include R10K::Logging
+
       # Dispatch to the type specific visitor method
       #
       # @param type [Symbol] The object type to dispatch for

data/lib/r10k/environment/bare.rb

@@ -1,13 +1,10 @@
-class R10K::Environment::Bare < R10K::Environment::WithModules
+class R10K::Environment::Bare < R10K::Environment::Plain
 
   R10K::Environment.register(:bare, self)
 
-  def sync
-    path.mkpath
-  end
-
-  def status
-    :not_applicable
+  def initialize(name, basedir, dirname, options = {})
+    logger.warn _('"bare" environment type is deprecated; please use "plain" instead (environment: %{name})') % {name: name}
+    super
   end
 
   def signature

data/lib/r10k/environment/name.rb

@@ -6,17 +6,22 @@ module R10K
     class Name
 
       # @!attribute [r] name
-      #   @return [String] The unmodified name of the environment
+      #   @return [String] The functional name of the environment derived from inputs and options.
       attr_reader :name
 
+      # @!attribute [r] original_name
+      #   @return [String] The unmodified name originally given to create the object.
+      attr_reader :original_name
+
       INVALID_CHARACTERS = %r[\W]
 
-      def initialize(name, opts)
+      def initialize(original_name, opts)
         @source  = opts[:source]
         @prefix  = opts[:prefix]
         @invalid = opts[:invalid]
 
-        @name = derive_name(name, opts[:strip_component])
+        @name = derive_name(original_name, opts[:strip_component])
+        @original_name = original_name
         @opts = opts
 
         case @invalid
@@ -71,8 +76,8 @@ module R10K
 
       private
 
-      def derive_name(name, strip_component)
-        return name unless strip_component
+      def derive_name(original_name, strip_component)
+        return original_name unless strip_component
 
         unless strip_component.is_a?(String)
           raise _('Improper configuration value given for strip_component setting in %{src} source. ' \
@@ -82,11 +87,11 @@ module R10K
 
         if %r{^/.*/$}.match(strip_component)
           regex = Regexp.new(strip_component[1..-2])
-          name.gsub(regex, '')
-        elsif name.start_with?(strip_component)
-          name[strip_component.size..-1]
+          original_name.gsub(regex, '')
+        elsif original_name.start_with?(strip_component)
+          original_name[strip_component.size..-1]
         else
-          name
+          original_name
         end
       end
 

data/lib/r10k/environment/plain.rb

@@ -0,0 +1,16 @@
+class R10K::Environment::Plain < R10K::Environment::WithModules
+
+  R10K::Environment.register(:plain, self)
+
+  def sync
+    path.mkpath
+  end
+
+  def status
+    :not_applicable
+  end
+
+  def signature
+    'plain-default'
+  end
+end

data/lib/r10k/environment/tarball.rb

@@ -0,0 +1,78 @@
+require 'r10k/util/setopts'
+require 'r10k/tarball'
+require 'r10k/environment'
+
+class R10K::Environment::Tarball < R10K::Environment::WithModules
+
+  R10K::Environment.register(:tarball, self)
+
+  # @!attribute [r] tarball
+  #   @api private
+  #   @return [R10K::Tarball]
+  attr_reader :tarball
+
+  include R10K::Util::Setopts
+
+  # Initialize the given tarball environment.
+  #
+  # @param name [String] The unique name describing this environment.
+  # @param basedir [String] The base directory where this environment will be created.
+  # @param dirname [String] The directory name for this environment.
+  # @param options [Hash] An additional set of options for this environment.
+  #
+  # @param options [String] :source Where to get the tarball from
+  # @param options [String] :version The sha256 digest of the tarball
+  def initialize(name, basedir, dirname, options = {})
+    super
+    setopts(options, {
+      # Standard option interface
+      :type      => ::R10K::Util::Setopts::Ignore,
+      :source    => :self,
+      :version   => :checksum,
+
+      # Type-specific options
+      :checksum => :self,
+    })
+
+    @tarball = R10K::Tarball.new(name, @source, checksum: @checksum)
+  end
+
+  def path
+    @path ||= Pathname.new(File.join(@basedir, @dirname))
+  end
+
+  def sync
+    tarball.get unless tarball.cache_valid?
+    case status
+    when :absent, :mismatched
+      tarball.unpack(path.to_s)
+      # Untracked files left behind from previous extractions are expected to
+      # be deleted by r10k's purge facility.
+    end
+  end
+
+  def status
+    if not path.exist?
+      :absent
+    elsif not (tarball.cache_valid? && tarball.insync?(path.to_s, ignore_untracked_files: true))
+      :mismatched
+    else
+      :insync
+    end
+  end
+
+  def signature
+    @checksum || @tarball.cache_checksum
+  end
+
+  include R10K::Util::Purgeable
+
+  # Returns an array of the full paths to all the content being managed.
+  # @note This implements a required method for the Purgeable mixin
+  # @return [Array<String>]
+  def desired_contents
+    desired = []
+    desired += @tarball.paths.map { |entry| File.join(@full_path, entry) }
+    desired += super
+  end
+end

data/lib/r10k/environment/with_modules.rb

@@ -96,7 +96,7 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   def load_modules(module_hash)
     module_hash.each do |name, args|
       if !args.is_a?(Hash)
-        args = { version: args }
+        args = { type: 'forge', version: args }
       end
 
       add_module(name, args)

data/lib/r10k/environment.rb

@@ -30,8 +30,10 @@ module R10K
 
     require 'r10k/environment/base'
     require 'r10k/environment/with_modules'
+    require 'r10k/environment/plain'
     require 'r10k/environment/bare'
     require 'r10k/environment/git'
     require 'r10k/environment/svn'
+    require 'r10k/environment/tarball'
   end
 end

data/lib/r10k/errors.rb

@@ -58,4 +58,9 @@ module R10K
       str.gsub(/^/, prefix)
     end
   end
+
+  # An error class for configuration errors
+  #
+  class ConfigError < Error
+  end
 end

data/lib/r10k/forge/module_release.rb

@@ -1,5 +1,6 @@
 require 'r10k/logging'
 require 'r10k/settings/mixin'
+require 'r10k/util/cacheable'
 require 'fileutils'
 require 'tmpdir'
 require 'puppet_forge'
@@ -13,7 +14,7 @@ module R10K
 
       def_setting_attr :proxy
       def_setting_attr :baseurl
-      def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/cache': '/root/.r10k/cache')
+      def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir
 
       include R10K::Logging