r10k 3.5.1 → 3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f2418e73622a4149507cc7e2dde74b68a448689da2a7287b19107fbef0e184e
-  data.tar.gz: 95acaeb5cb7ad0969ec3f2a3545ab82490bafb9d0fca5da444125bbe780b9836
+  metadata.gz: 7e658209eb66330ab2c2cea4dd4fac0690b6700042e639e5ff11d2fffb120f1b
+  data.tar.gz: 870249f76ef6c2b289e3255e6c752d6fc725009a8e37f8b4219f7496a7c0a439
 SHA512:
-  metadata.gz: dd666a18fe5df938ab5a151cf411ac335e0f3f151428fe8589010ff699b832609fc58e189c46f52894d9d6b9d8e63156ed928bd14d2ed02d5d9a113fc0c5feb0
-  data.tar.gz: 40307661c96a21d00595271fa21836175c993dc937504b82b449218e39cd3804000a7c24c32a52aa570098b450fd35417573190dd9d603ac087e217f391aacb4
+  metadata.gz: a01dfae93025bac535c4e18d5f15b800c7bc1df1aa79cf0a2ea565c034767d5c5f0b0e4b41b7954e59310090a25ca67e28e92ab416958b2f91864a3da873d93a
+  data.tar.gz: 0631fe1846185c2808bd14be2cca3b182402991476950bae18da0fb19129eb7139fbec3b1f9a3bf1568f09931d0438c9a014380378cce71b01d7b9022364cb7c

data/.github/pull_request_template.md

@@ -1 +1,4 @@
-Please add all notable changes to the "Unreleased" section of the CHANGELOG.
+Please add all notable changes to the "Unreleased" section of the CHANGELOG in the format:
+```
+- Summary of changes. [Issue or PR #](link to issue or PR)
+```

data/.github/workflows/docker.yml

@@ -29,7 +29,10 @@ jobs:
       - run: gem install bundler
       - name: Build container
         working-directory: docker
-        run: make lint build test
+        run: |
+          docker system prune --all --force --volumes
+          docker builder prune --force --keep-storage=10GB
+          make lint build test
       - name: Publish container
         working-directory: docker
         run: |

data/.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: Tag and release
 on:
   push:
     branches:
-      - master
+      - main
     paths:
       - 'lib/r10k/version.rb'
 
@@ -15,10 +15,11 @@ jobs:
         with:
           fetch-depth: '0'
       - name: Bump version and push tag
-        uses: anothrNick/github-tag-action@1.17.2
+        uses: anothrNick/github-tag-action@1.35.0
         env:
           GITHUB_TOKEN: ${{ secrets.PUPPET_RELEASE_GH_TOKEN }}
           DEFAULT_BUMP: patch
+          TAG_CONTEXT: branch
           WITH_V: false
           # Uncomment this if the tag and version file become out-of-sync and
           # you need to tag at a specific version.

data/.github/workflows/rspec_tests.yml

@@ -0,0 +1,81 @@
+name: Rspec tests
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  rspec_tests:
+    name: ${{ matrix.cfg.os }}(ruby ${{ matrix.cfg.ruby }})
+    strategy:
+      matrix:
+        cfg:
+          - {os: ubuntu-18.04, ruby: 2.4}
+          - {os: ubuntu-18.04, ruby: 2.5}
+          - {os: ubuntu-18.04, ruby: 2.6}
+          - {os: ubuntu-18.04, ruby: 2.7}
+          - {os: ubuntu-18.04, ruby: jruby-9.2.9.0}
+          - {os: windows-2016, ruby: 2.5}
+          - {os: windows-2016, ruby: 2.6}
+          - {os: windows-2016, ruby: 2.7}
+
+    runs-on: ${{ matrix.cfg.os }}
+    steps:
+      - name: Checkout current PR
+        uses: actions/checkout@v2
+
+      - name: Install ruby version ${{ matrix.cfg.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.cfg.ruby }}
+
+      - name: Install bundler and gems
+        run: |
+          gem install bundler
+          bundle config set without packaging documentation
+          bundle install --jobs 4 --retry 3
+
+      - name: Run tests on Windows
+        if: runner.os == 'Windows'
+        run: |
+          # https://github.com/ruby/ruby/pull/2791/files#diff-ff5ff976e81bebd977f0834e60416abbR97-R100
+          # Actions uses UTF8, causes test failures, similar to normal OS setup
+          $PSDefaultParameterValues['*:Encoding'] = 'utf8'
+          [Console]::OutputEncoding = [System.Text.Encoding]::GetEncoding("IBM437")
+          [Console]::InputEncoding  = [System.Text.Encoding]::GetEncoding("IBM437")
+          $Env:LOG_SPEC_ORDER = 'true'
+
+          # debug information
+          chcp
+          Get-WinSystemLocale
+          Get-ChildItem Env: | % { Write-Output "$($_.Key): $($_.Value)"  }
+          # list current OpenSSL install
+          gem list openssl
+          ruby -ropenssl -e 'puts \"OpenSSL Version - #{OpenSSL::OPENSSL_VERSION}\"; puts \"OpenSSL Library Version - #{OpenSSL::OPENSSL_LIBRARY_VERSION}\"'
+          Get-Content Gemfile.lock
+          ruby -v
+          gem --version
+          bundle --version
+
+          # Run tests
+          bundle exec rspec --color --format documentation spec/unit
+
+      - name: Run tests on Linux
+        if: runner.os == 'Linux'
+        run: |
+          # debug information
+          cat Gemfile.lock
+          ruby -v
+          gem --version
+          bundle --version
+
+          if [[ ${{ matrix.cfg.ruby }} =~ "jruby" ]]; then
+            export _JAVA_OPTIONS='-Xmx1024m -Xms512m'
+
+            # workaround for PUP-10683
+            sudo apt remove rpm
+          fi
+
+          # Run tests
+          bundle exec rspec --color --format documentation spec/unit

data/.travis.yml

@@ -29,7 +29,8 @@ matrix:
         - docker
       rvm: 2.6.6
       env:
-        - DOCKER_COMPOSE_VERSION=1.25.5
+        - DOCKER_COMPOSE_VERSION=1.28.6
+        - DOCKER_BUILDX_VERSION=0.5.1
         # necessary to prevent overwhelming TravisCI build output limits
         - DOCKER_BUILD_FLAGS="--progress plain"
       before_install:
@@ -37,9 +38,15 @@ matrix:
         - curl --location https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname --kernel-name`-`uname --machine` > docker-compose
         - chmod +x docker-compose
         - sudo mv docker-compose /usr/local/bin
+        - mkdir -vp ~/.docker/cli-plugins
+        - curl --location https://github.com/docker/buildx/releases/download/v${DOCKER_BUILDX_VERSION}/buildx-v${DOCKER_BUILDX_VERSION}.linux-amd64 > ~/.docker/cli-plugins/docker-buildx
+        - chmod +x ~/.docker/cli-plugins/docker-buildx
+        - docker buildx create --name travis_builder --use
       script:
         - set -e
         - cd docker
         - make lint
         - make build
         - make test
+      after_script:
+        - docker buildx rm travis_builder

data/CHANGELOG.mkd

@@ -2,14 +2,50 @@ CHANGELOG
 =========
 
 Unreleased
-----
+----------
+
+3.9.0
+-----
+
+- Add '--modules' flag to `deploy` subcommand as a replacement to '--puppetfile', deprecate '--puppetfile'. [#1147](https://github.com/puppetlabs/r10k/pull/1147)
+- Deprecate 'purge_whitelist' and favor usage of 'purge_allowlist'. [#1144](https://github.com/puppetlabs/r10k/pull/1144)
+- Add 'strip\_component' environment source configuration setting, to allow deploying Git branches named like "env/production" as Puppet environments named like "production". [#1128](https://github.com/puppetlabs/r10k/pull/1128)
+- A warning will be emitted when the user supplies conflicting arguments to module definitions in a Puppetfile, such as when specifying both :commit and :branch [#1130](https://github.com/puppetlabs/r10k/pull/1130)
+- Add optional standard module and environment specification interface: name, type, source, version. These options can be used when specifying environments and/or modules in a yaml/exec source, as well as when specifying modules in a Puppetfile. Providing the standard interface simplifies integrations with external services [#1131](https://github.com/puppetlabs/r10k/pull/1131)
+- Pin cri to 2.15.10 to maintain support for Ruby 2.3 and 2.4 [#1121](https://github.com/puppetlabs/r10k/issues/1121)
+
+3.8.0
+-----
+
+- When a forge module fails name validation the offending name will now be printed in the error message. [#1126](https://github.com/puppetlabs/r10k/pull/1126)
+- Module ref resolution will now fall back to the normal default branch if the default branch override cannot be resolved. [#1122](https://github.com/puppetlabs/r10k/pull/1122)
+- Experimental feature change: conflicts between environment-defined modules and Puppetfile-defined modules now default to logging a warning and deploying the environment module version, overriding the Puppetfile. Previously, conflicts would result in an error. The behavior is now configurable via the `module_conflicts` environment setting [#1107](https://github.com/puppetlabs/r10k/pull/1107)
+
+3.7.0
+-----
+
+- Use %LOCALAPPDATA% for Windows cachedir [#1049](https://github.com/puppetlabs/r10k/issues/1049)
+- Add support for defining custom puppet.conf when generating types [#993](https://github.com/puppetlabs/r10k/pull/993)
+- Update test cases to account for error message changes in Puppet 7
+
+3.6.0
+-----
+
+- Add filter_command configuration option for git repositories. (Thanks to [mhumpula](https://github.com/mhumpula) for the feature.) [#823](https://github.com/puppetlabs/r10k/pull/823)
+- Increase default pool_size to 4, allowing modules to be downloaded on 4 threads concurrently. [#1038](https://github.com/puppetlabs/r10k/issues/1038)
+- Ensure that modules that share a cachedir download serially, to avoid cache corruption. [#1058](https://github.com/puppetlabs/r10k/issues/1058)
+- Don't purge root when using `puppetfile install`. [#1084](https://github.com/puppetlabs/r10k/issues/1084)
+
+3.5.2
+-----
+
+- (RK-319) Clean up tmp directories used for downloading modules
 
 3.5.1
 -----
 
 - Upgrade Rugged Gemfile dependency for local development to ~> 1.0
 
-
 3.5.0
 -----
 
@@ -171,6 +207,11 @@ Ruby >= 2.3 official and documented.
     - Unable to parse Puppetfile
   - Various perfomance improvements
 
+2.6.9
+----
+
+- Don't purge root when using `puppetfile install`. [#1084](https://github.com/puppetlabs/r10k/issues/1084)
+
 2.6.8
 ----
 

data/CODEOWNERS

@@ -1,2 +1,2 @@
-*        @puppetlabs/puppetserver-maintainers @adrienthebo @dhollinger
-/docker/ @puppetlabs/pupperware
+*        @puppetlabs/puppetserver-maintainers
+/docker/ @puppetlabs/platform-services

data/README.mkd

@@ -17,8 +17,8 @@ implementation of Puppet [environments][workflow].
 Requirements
 ------------
 
-R10k supports the Ruby versions `>= 2.3.0`. It's tested on Ruby 2.3.0 up to
-Ruby 2.6.0 + Jruby.
+R10k supports the Ruby versions `>= 2.4.0`. It's tested on Ruby 2.4.0 up to
+Ruby 2.7.0 + Jruby.
 
 R10k requires additional components, depending on how you plan on managing
 environments and modules.
@@ -98,6 +98,14 @@ When localizing the strings found in R10k, follow the prescribed
 The workflow describes the rake tasks provided to generate the .po files for
 each locale.
 
+Releasing
+---------
+To release a new version of the r10k gem, ensure the [changelog](CHANGELOG.mkd) is up to date and open a pull request updating the [version file](lib/r10k/version.rb). When the PR is merged, a new release of the gem will be triggered.
+
+By default, a patch (Z) release will be triggered. To release a new major (X) or minor (Y) version, include `#major` or `#minor` (respectively) in your commit message to trigger the appropriate release.
+
+NOTE: This currently only works for the default branch. If you would like to release from a different branch, please contact the [CODEOWNERS](CODEOWNERS).
+
 Getting help
 ------------
 
@@ -113,6 +121,7 @@ Please see the CHANGELOG for a listing of the (very awesome) contributors.
 
 ## Maintenance
 
-See [MAINTAINERS](MAINTAINERS) (json file)
+See [CODEOWNERS](CODEOWNERS) for active repo maintainers.
+
+Open [issues](https://github.com/puppetlabs/r10k/issues) directly in the r10k repo.
 
-Tickets: File at https://tickets.puppet.com/browse/RK

data/doc/common-patterns.mkd

@@ -42,3 +42,4 @@ publicly available hook. These include:
 * [Reaktor](https://github.com/pzim/reaktor)
 * [zack/r10k's Webhooks](https://forge.puppetlabs.com/zack/r10k#webhook-support)
 (Puppet Enterprise only)
+* [Simple Puppet Provisioner](https://github.com/mbaynton/SimplePuppetProvisioner)

data/doc/dynamic-environments/configuration.mkd

@@ -76,9 +76,9 @@ The proxy server being used will be logged at the "debug" level when r10k runs.
 ### pool_size
 
 The pool_size setting is a number to determine how many threads should be spawn
-while updating modules. The default value is 1, which means the default behaviour
-is to update modules in a serial manner. Increasing this number should bring
-some performance gains.
+while updating modules. The default value is 4, which means modules will be updated
+in parallel. If this causes issues, change this setting to 1 to cause modules to be
+updated serially.
 
 ### git
 
@@ -249,9 +249,9 @@ found which is neither committed to the control repo branch that maps to that
 environment, nor declared in a Puppetfile committed to that branch.
 
 Enabling this purge level will cause r10k to load and parse the Puppetfile for
-the environment even without the `--puppetfile` flag being set. However,
+the environment even without the `--modules` flag being set. However,
 Puppetfile content will still only be deployed if the environment is new or
-the `--puppetfile` flag is set. Additionally, no environment-level content
+the `--modules` flag is set. Additionally, no environment-level content
 will be purged if any errors are encountered while evaluating the Puppetfile
 or deploying its contents.
 
@@ -266,31 +266,31 @@ managed by a Puppetfile include the configured `moduledir` (which defaults to
 "modules") as well as alternate directories specified as an `install_path`
 option to any Puppetfile content declarations.
 
-#### purge\_whitelist
+#### purge\_allowlist
 
-The `purge_whitelist` setting exempts the specified filename patterns from
+The `purge_allowlist` setting exempts the specified filename patterns from
 being purged. This setting is currently only considered during `environment`
 level purging. (See above.) Given value must be a list of shell style filename
 patterns in string format.
 
 See the Ruby [documentation for the `fnmatch` method](http://ruby-doc.org/core-2.2.0/File.html#method-c-fnmatch)
 for more details on valid patterns. Note that the `FNM_PATHNAME` and
-`FNM_DOTMATCH` flags are in effect when r10k considers the whitelist.
+`FNM_DOTMATCH` flags are in effect when r10k considers the allowlist.
 
 Patterns are relative to the root of the environment being purged and *do
-not match recursively* by default. For example, a whitelist value of
+not match recursively* by default. For example, a allowlist value of
 `*myfile*` would only preserve a matching file at the root of the
 environment. To preserve the file throughout the deployed environment,
 a recursive pattern such as `**/*myfile*` would be required.
 
-Files matching a whitelist pattern may still be removed if they exist in
+Files matching a allowlist pattern may still be removed if they exist in
 a folder that is otherwise subject to purging. In this case, an additional
-whitelist rule to preserve the containing folder is required.
+allowlist rule to preserve the containing folder is required.
 
 ```yaml
 ---
 deploy:
-  purge_whitelist: [ 'custom.json', '**/*.xpp' ]
+  purge_allowlist: [ 'custom.json', '**/*.xpp' ]
 ```
 
 
@@ -327,6 +327,15 @@ deploy:
   puppet_path: '/usr/local/bin/puppet'
 ```
 
+#### puppet\_conf
+
+The path to the puppet.conf file used for generating types. Defaults to `/etc/puppetlabs/puppet/puppet.conf`.
+
+```yaml
+deploy:
+  puppet_conf: '/opt/puppet/conf/puppet.conf'
+```
+
 Source options
 --------------
 
@@ -385,6 +394,23 @@ sources:
 * if `false` (default) environment folder will not be prefixed
 * if `String` environment folder will be prefixed with the `prefix` value.
 
+### strip\_component
+
+The 'strip\_component' setting allows parts of environment names from a source to have a transformation applied, removing a part of the name before turning them into Puppet environments. This is primarily useful for VCS sources (e.g.  Git), because it allows branch names to use prefixes or organizing name components such as "env/production", "env/development", but deploy Puppet environments from these branches named without the leading "env/" component. E.g. "production", "development".
+
+```yaml
+---
+sources:
+  mysource:
+    basedir: '/etc/puppet/environments'
+    strip_component: 'env/'
+```
+
+#### strip\_component behaviour
+
+* if `string` environment names will have this prefix removed, if the prefix is present. Note that when string values are used, names can only have prefix components removed.
+* if `/regex/` the regex will be matched against environment names and if a match is found, the matching name component will be removed.
+
 ### ignore_branch_prefixes
 
 The 'ignore_branch_prefixes' setting causes environments to be ignored which match in part or whole
@@ -411,6 +437,29 @@ sources:
       - 'dev'
 ```
 
+### filter_command
+
+You can filter out any branch based on the result of the command specified as
+'filter_command'. Currently it only works with git repository. Non zero return
+status of the command results in a branch beeing removed. The command is passed
+additional environment variables
+
+* GIT_DIR – path to the cached git repository
+* R10K_BRANCH – branch which is being filtered
+* R10K_NAME – source name from r10k configuration
+
+This can be used for example for filtering out the branches with invalid gpg signature of their latest commit
+
+```yaml
+---
+sources:
+  mysource:
+    basedir: '/etc/puppet/environments'
+    filter_command: 'git verify-commit $R10K_BRANCH 2> /dev/null'
+```
+
+Beware that if the production branch of manifests is filtered out, you will end up with empty environment.
+
 Examples
 --------
 
@@ -596,7 +645,7 @@ sources:
 
 The environment modules feature allows module content to be attached to an environment at environment definition time. This happens before modules specified in a Puppetfile are attached to an environment, which does not happen until deploy time. Environment module implementation depends on the environment source type.
 
-For the YAML environment source type, attach modules to an environment by specifying a modules key for the environment, and providing a hash of modules to attach. Each module accepts the same arguments accepted by the `mod` method in a Puppetfile.
+For the YAML environment source type, attach modules to an environment by specifying a modules key for the environment, and providing a hash of modules to attach. Each module accepts the same arguments accepted by the `mod` method in a Puppetfile. For ease of reading and consistency, however, it is perferred to use the generic type, source, and version options over implementation-specific formats and options such as "ref" and "git".
 
 The example below includes two Forge modules and one module sourced from a Git repository. The two environments are almost identical. However, a new version of the stdlib module has been deployed in development (6.2.0), that has not yet been deployed to production.
 
@@ -604,25 +653,35 @@ The example below includes two Forge modules and one module sourced from a Git r
 ---
 production:
   type: git
-  remote: git@github.com:puppetlabs/control-repo.git
-  ref: 8820892
+  source: git@github.com:puppetlabs/control-repo.git
+  version: 8820892
   modules:
-    puppetlabs-stdlib: 6.0.0
-    puppetlabs-concat: 6.1.0
+    puppetlabs-stdlib:
+      type: forge
+      version: 6.0.0
+    puppetlabs-concat:
+      type: forge
+      version: 6.1.0
     reidmv-xampl:
-      git: https://github.com/reidmv/reidmv-xampl.git
-      ref: 62d07f2
+      type: git
+      source: https://github.com/reidmv/reidmv-xampl.git
+      version: 62d07f2
 
 development:
   type: git
-  remote: git@github.com:puppetlabs/control-repo.git
-  ref: 8820892
+  source: git@github.com:puppetlabs/control-repo.git
+  version: 8820892
   modules:
-    puppetlabs-stdlib: 6.2.0
-    puppetlabs-concat: 6.1.0
+    puppetlabs-stdlib:
+      type: forge
+      version: 6.2.0
+    puppetlabs-concat:
+      type: forge
+      version: 6.1.0
     reidmv-xampl:
-      git: https://github.com/reidmv/reidmv-xampl.git
-      ref: 62d07f2
+      type: git
+      source: https://github.com/reidmv/reidmv-xampl.git
+      version: 62d07f2
 ```
 
 An example of a single environment definition for the YAMLdir environment source type:
@@ -631,14 +690,49 @@ An example of a single environment definition for the YAMLdir environment source
 # production.yaml
 ---
 type: git
-remote: git@github.com:puppetlabs/control-repo.git
-ref: 8820892
+source: git@github.com:puppetlabs/control-repo.git
+version: 8820892
+modules:
+  puppetlabs-stdlib:
+    type: forge
+    version: 6.0.0
+  puppetlabs-concat:
+    type: forge
+    version: 6.1.0
+  reidmv-xampl:
+    type: git
+    source: https://github.com/reidmv/reidmv-xampl.git
+    version: 62d07f2
+```
+
+#### Puppetfile module conflicts
+
+When a module is defined in an environment and also in a Puppetfile, the default behavior is for the environment definition of the module to take precedence, a warning to be logged, and the Puppetfile definition to be ignored. The behavior is configurable to optionally skip the warning, or allow a hard failure instead. Use the `module_conflicts` option in an environment definition to control this.
+
+Available `module_conflicts` options:
+
+* `override_and_warn` (default): the version of the module defined by the environment will be used, and the version defined in the Puppetfile will be ignored. A warning will be printed.
+* `override`: the version of the module defined by the environment will be used, and the version defined in the Puppetfile will be ignored.
+* `error`: an error will be raised alerting the user to the conflict. The environment will not be deployed.
+
+```yaml
+# production.yaml
+---
+type: git
+source: git@github.com:puppetlabs/control-repo.git
+version: 8820892
+module_conflicts: override_and_warn
 modules:
-  puppetlabs-stdlib: 6.0.0
-  puppetlabs-concat: 6.1.0
+  puppetlabs-stdlib:
+    type: forge
+    version: 6.0.0
+  puppetlabs-concat:
+    type: forge
+    version: 6.1.0
   reidmv-xampl:
-    git: https://github.com/reidmv/reidmv-xampl.git
-    ref: 62d07f2
+    type: git
+    source: https://github.com/reidmv/reidmv-xampl.git
+    version: 62d07f2
 ```
 
 ### Bare Environment Type
@@ -652,18 +746,28 @@ The bare environment type allows sources that support environment modules to ope
 production:
   type: bare
   modules:
-    puppetlabs-stdlib: 6.0.0
-    puppetlabs-concat: 6.1.0
+    puppetlabs-stdlib:
+      type: forge
+      version: 6.0.0
+    puppetlabs-concat:
+      type: forge
+      version: 6.1.0
     reidmv-xampl:
-      git: https://github.com/reidmv/reidmv-xampl.git
-      ref: 62d07f2
+      type: git
+      source: https://github.com/reidmv/reidmv-xampl.git
+      version: 62d07f2
 
 development:
   type: bare
   modules:
-    puppetlabs-stdlib: 6.0.0
-    puppetlabs-concat: 6.1.0
+    puppetlabs-stdlib:
+      type: forge
+      version: 6.0.0
+    puppetlabs-concat:
+      type: forge
+      version: 6.1.0
     reidmv-xampl:
-      git: https://github.com/reidmv/reidmv-xampl.git
-      ref: 62d07f2
+      type: git
+      source: https://github.com/reidmv/reidmv-xampl.git
+      version: 62d07f2
 ```