r10k 3.4.0 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fa591e05f2bcb8e3e7acb9b4ac018ef5a3471832bca677a49c5a409f358b5a8
-  data.tar.gz: 9d27ceb63bc53563b419140a3de96b744171bb790d8fb12a2a4c73e4b3feb2cf
+  metadata.gz: 4e54158e0a651c799246229971ba62ad36bd40a670ca84e61479aa5280306c8a
+  data.tar.gz: 153cc105ce9276d79e2501dc99accb398bba8fec423fbf0726a1b183993a2748
 SHA512:
-  metadata.gz: 1cc2808331cbcb0f52ce418eaea05995a96ddec0bc5e523b7179969d4f51609745912ae7a8e621104a704a40f71a7502bdc809d10948ea39961cb288a81ddd9c
-  data.tar.gz: 418fcc583a83105d5953cd1df74e4ea09735b427765c67f2600d357c3daade952a7d1242f102d1bf2c34b71aa8bcd7cfeca7416274d97911b2916a28b7b03571
+  metadata.gz: 264ea2591c062d460edab02b8edf7a59c9ebc08ebea27ee2b103e43bdf71c936b797094014d66b776573196d4a9e86dfc9f0fd345626f27d7e25c8170ebaa83a
+  data.tar.gz: 224e40f3f13466ab2ea3296bc3741b10d591ea65c97ba60b3dd81078f70149770c07c46ac1464a44fd59af0d20dbc8bbf1a5e3990e5d7921cc05403772d99640

data/.github/pull_request_template.md

@@ -0,0 +1,4 @@
+Please add all notable changes to the "Unreleased" section of the CHANGELOG in the format:
+```
+- (JIRA ticket) Summary of changes. [Issue or PR #](link to issue or PR)
+```

data/.github/workflows/docker.yml

@@ -10,6 +10,7 @@ jobs:
     env:
       PUPPERWARE_ANALYTICS_STREAM: production
       IS_LATEST: true
+      # Still need these env vars for README publishing
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
 
@@ -17,6 +18,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@master
+      - uses: azure/docker-login@v1
+        with: # This doesn't seem to work unless we point directly to the secrets
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: Set up Ruby 2.6
         uses: actions/setup-ruby@v1
         with:
@@ -28,5 +33,24 @@ jobs:
       - name: Publish container
         working-directory: docker
         run: |
-          docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD"
           make publish
+      - name: Build release container
+        env:
+          IS_RELEASE: true
+        working-directory: docker
+        run: |
+          if make prep; then
+            make lint build test
+          else
+            echo "Skipping release container building and testing"
+          fi
+      - name: Publish release container
+        env:
+          IS_RELEASE: true
+        working-directory: docker
+        run: |
+          if make prep; then
+            make publish
+          else
+            echo "Skipping release container publishing"
+          fi

data/.github/workflows/release.yml

@@ -0,0 +1,36 @@
+name: Tag and release
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'lib/r10k/version.rb'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - name: Bump version and push tag
+        uses: anothrNick/github-tag-action@1.17.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUPPET_RELEASE_GH_TOKEN }}
+          DEFAULT_BUMP: patch
+          WITH_V: false
+          # Uncomment this if the tag and version file become out-of-sync and
+          # you need to tag at a specific version.
+          # CUSTOM_TAG:
+      - name: Build gem
+        uses: scarhand/actions-ruby@master
+        with:
+          args: build *.gemspec
+      - name: Publish gem
+        uses: scarhand/actions-ruby@master
+        env:
+          RUBYGEMS_AUTH_TOKEN: ${{ secrets.RUBYGEMS_AUTH_TOKEN }}
+        with:
+          args: push *.gem
+

data/.travis.yml

@@ -1,17 +1,17 @@
 ---
 language: ruby
-services:
-  - docker
 bundler_args: "--without system"
 script: "bundle exec rspec --color --format documentation spec/unit"
 notifications:
   email: false
 sudo: false
 jdk:
-  - openjdk8
+  - openjdk11
 before_install: gem install bundler -v '< 2' --no-document
 matrix:
   include:
+    - stage: r10k tests
+      rvm: 2.7.0
     - stage: r10k tests
       rvm: 2.6.5
     - stage: r10k tests
@@ -23,10 +23,23 @@ matrix:
     - stage: r10k tests
       rvm: jruby
     - stage: r10k container tests
+      dist: focal
       language: ruby
-      rvm: 2.5.5
+      services:
+        - docker
+      rvm: 2.6.6
+      env:
+        - DOCKER_COMPOSE_VERSION=1.25.5
+        # necessary to prevent overwhelming TravisCI build output limits
+        - DOCKER_BUILD_FLAGS="--progress plain"
+      before_install:
+        - sudo rm /usr/local/bin/docker-compose
+        - curl --location https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname --kernel-name`-`uname --machine` > docker-compose
+        - chmod +x docker-compose
+        - sudo mv docker-compose /usr/local/bin
       script:
-        - |
-          set -ex
-          cd docker && make lint build test
-          set +x
+        - set -e
+        - cd docker
+        - make lint
+        - make build
+        - make test

data/CHANGELOG.mkd

@@ -1,6 +1,45 @@
 CHANGELOG
 =========
 
+Unreleased
+----------
+
+3.6.0
+-----
+
+- Add filter_command configuration option for git repositories. (Thanks to [mhumpula](https://github.com/mhumpula) for the feature.) [#823](https://github.com/puppetlabs/r10k/pull/823)
+- Increase default pool_size to 4, allowing modules to be downloaded on 4 threads concurrently. [#1038](https://github.com/puppetlabs/r10k/issues/1038)
+- Ensure that modules that share a cachedir download serially, to avoid cache corruption. [#1058](https://github.com/puppetlabs/r10k/issues/1058)
+- Don't purge root when using `puppetfile install`. [#1084](https://github.com/puppetlabs/r10k/issues/1084)
+
+3.5.2
+-----
+
+- (RK-319) Clean up tmp directories used for downloading modules
+
+3.5.1
+-----
+
+- Upgrade Rugged Gemfile dependency for local development to ~> 1.0
+
+3.5.0
+-----
+
+- Add exec environment source type. The exec source type allows for the
+  implementation of external environment sources
+  [#1042](https://github.com/puppetlabs/r10k/pull/1042).
+- Improve atomicity of .r10k-deploy.json writes. Fixes
+  [#813](https://github.com/puppetlabs/r10k/issues/813)
+
+3.4.1
+-----
+
+- Add support for Ruby 2.7
+- (RK-357) Restrict gettext and fast_gettext versions for compatibility with Ruby 2.4
+- Bump cri to 2.15.10
+- (RK-360) Allow overriding the cachedir in `deploy module`
+- Bump puppet_forge to 2.3.0
+
 3.4.0
 -----
 
@@ -144,6 +183,27 @@ Ruby >= 2.3 official and documented.
     - Unable to parse Puppetfile
   - Various perfomance improvements
 
+2.6.9
+----
+
+- Don't purge root when using `puppetfile install`. [#1084](https://github.com/puppetlabs/r10k/issues/1084)
+
+2.6.8
+----
+
+### Changes
+
+(RK-357) Restrict gettext and fast_gettext versions for compatibility with Ruby 2.4.
+(RK-358) Update puppet_forge to ~> 2.3.0.
+
+2.6.7
+----
+
+### Changes
+
+(CDPE-1813) Add module deploy info to .r10k-deploy.json.
+(RK-351) Update minitar to ~> 0.9.0.
+
 2.6.6
 ----
 
@@ -356,7 +416,7 @@ Puppetfile should be installed to. See the [Puppetfile documentation](https://gi
 
 You can now configure how r10k purges unmanaged content after a deployment. The
 default behavior should be unchanged but there is a new "purge\_levels" configuration
-option that can be used to enable new behavior or de-activate certain existing 
+option that can be used to enable new behavior or de-activate certain existing
 behaviors. See the relevant [configuration documentation](https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#purge_levels) for more details.
 
 (RK-223) Ability to track control repo branch from content declarations.
@@ -403,7 +463,7 @@ fix.)
 Previously, r10k only supported the use of HTTP proxies for connecting to the Puppet
 Forge. With these changes, r10k can now be configured to use an HTTP proxy for both
 Forge and Git operations. Configuration can be specified globally, for Forge or Git
-only, or on a per-Git repository basis. See [configuration documentation](https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd) 
+only, or on a per-Git repository basis. See [configuration documentation](https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd)
 for more details.
 
 ### Bug Fixes
@@ -424,9 +484,9 @@ for SSH.
 (RK-241) "deploy display" action does not properly format wrapped exceptions
 
 The "deploy display" action was not capturing and logging exceptions in the same way as
-other related actions. This meant that in many cases, when an error occurred, the 
+other related actions. This meant that in many cases, when an error occurred, the
 underlying cause was not being shown. Specifically, the "deploy display" action was
-not benefitting from the improved error messaging for unreadable SSH keys which was 
+not benefitting from the improved error messaging for unreadable SSH keys which was
 added in r10k 2.2.0 as part of RK-220.
 
 2.2.1

data/CODEOWNERS

@@ -1,2 +1,2 @@
-*        @puppetlabs/puppetserver-maintainers
+*        @puppetlabs/puppetserver-maintainers @adrienthebo @dhollinger
 /docker/ @puppetlabs/pupperware

data/Gemfile

@@ -2,7 +2,7 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 gemspec
 
 group :extra do
-  gem 'rugged', '>= 0.24.0', '< 0.24.6', :platforms => :ruby
+  gem 'rugged', '~> 1.0', :platforms => :ruby
 end
 
 group :development do

data/README.mkd

@@ -8,11 +8,11 @@ Puppet environment and module deployment
 Description
 -----------
 
-[workflow]: http://puppetlabs.com/blog/git-workflow-and-puppet-environments/
+[workflow]: https://puppet.com/docs/puppet/latest/environments_about.html
 
 R10k provides a general purpose toolset for deploying Puppet environments and
 modules. It implements the [Puppetfile](doc/puppetfile.mkd) format and provides a native
-implementation of Puppet [dynamic environments][workflow].
+implementation of Puppet [environments][workflow].
 
 Requirements
 ------------
@@ -98,6 +98,14 @@ When localizing the strings found in R10k, follow the prescribed
 The workflow describes the rake tasks provided to generate the .po files for
 each locale.
 
+Releasing
+---------
+To release a new version of the r10k gem, ensure the [changelog](CHANGELOG.mkd) is up to date and open a pull request updating the [version file](lib/r10k/version.rb). When the PR is merged, a new release of the gem will be triggered.
+
+By default, a patch (Z) release will be triggered. To release a new major (X) or minor (Y) version, include `#major` or `#minor` (respectively) in your commit message to trigger the appropriate release.
+
+NOTE: This currently only works for the default branch. If you would like to release from a different branch, please contact the [CODEOWNERS](CODEOWNERS).
+
 Getting help
 ------------
 
@@ -113,6 +121,7 @@ Please see the CHANGELOG for a listing of the (very awesome) contributors.
 
 ## Maintenance
 
-See [MAINTAINERS](MAINTAINERS) (json file)
+See [CODEOWNERS](CODEOWNERS) for active repo maintainers.
+
+Open [issues](https://github.com/puppetlabs/r10k/issues) directly in the r10k repo.
 
-Tickets: File at https://tickets.puppet.com/browse/RK

data/azure-pipelines.yml

@@ -17,7 +17,9 @@ variables:
   NAMESPACE: puppet
   CONTAINER_NAME: r10k
   CONTAINER_BUILD_PATH: .
-  LINT_IGNORES: DL3008 DL3018 DL4000 DL4001 DL3028
+  LINT_IGNORES:
+  DOCKER_BUILDKIT: 1
+  BUILD_OPTIONS: --build-arg alpine_version=3.9
 
 workspace:
   clean: resources
@@ -52,7 +54,7 @@ steps:
 
 - powershell: |
     . "$(bundle show pupperware)/ci/build.ps1"
-    Build-Container -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE -PathOrUri $ENV:CONTAINER_BUILD_PATH
+    Build-Container -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE -PathOrUri $ENV:CONTAINER_BUILD_PATH -AdditionalOptions ($ENV:BUILD_OPTIONS -split ' ')
   displayName: Build $(CONTAINER_NAME) Container
   timeoutInMinutes: 5
   name: build_container

data/doc/dynamic-environments/configuration.mkd

@@ -48,7 +48,7 @@ cachedir: '/var/cache/r10k'
 
 
 The cachedir setting defaults to `~/.r10k`. If the HOME environment variable is
-unset r10k will assume that r10k is being run with the Puppet [`prerun_command`](http://docs.puppetlabs.com/references/latest/configuration.html#preruncommand)
+unset r10k will assume that r10k is being run with the Puppet [`prerun_command`](https://puppet.com/docs/puppet/latest/configuration.html#preruncommand)
 setting and will set the cachedir default to `/root/.r10k`.
 
 ### proxy
@@ -76,9 +76,9 @@ The proxy server being used will be logged at the "debug" level when r10k runs.
 ### pool_size
 
 The pool_size setting is a number to determine how many threads should be spawn
-while updating modules. The default value is 1, which means the default behaviour
-is to update modules in a serial manner. Increasing this number should bring
-some performance gains.
+while updating modules. The default value is 4, which means modules will be updated
+in parallel. If this causes issues, change this setting to 1 to cause modules to be
+updated serially.
 
 ### git
 
@@ -411,6 +411,29 @@ sources:
       - 'dev'
 ```
 
+### filter_command
+
+You can filter out any branch based on the result of the command specified as
+'filter_command'. Currently it only works with git repository. Non zero return
+status of the command results in a branch beeing removed. The command is passed
+additional environment variables
+
+* GIT_DIR – path to the cached git repository
+* R10K_BRANCH – branch which is being filtered
+* R10K_NAME – source name from r10k configuration
+
+This can be used for example for filtering out the branches with invalid gpg signature of their latest commit
+
+```yaml
+---
+sources:
+  mysource:
+    basedir: '/etc/puppet/environments'
+    filter_command: 'git verify-commit $R10K_BRANCH 2> /dev/null'
+```
+
+Beware that if the production branch of manifests is filtered out, you will end up with empty environment.
+
 Examples
 --------
 
@@ -578,6 +601,20 @@ remote: git@github.com:puppetlabs/control-repo.git
 ref: 8820892
 ```
 
+### Exec environment Source
+
+The exec environment source runs an external command which is expected to return on stdout content compatible with the YAML environment source data format. The command may return the data in JSON or YAML form. The exec environment source is similar in purpose to Puppet's exec node terminus, used to implement external node classifiers (ENCs). R10k's exec source type allows the the implementation of external environment sources.
+
+```yaml
+# r10k.yaml
+---
+sources:
+  puppet:
+    type: exec
+    basedir: /etc/puppetlabs/code/environments
+    command: /usr/local/bin/r10k-environments.sh
+```
+
 ### Environment Modules
 
 The environment modules feature allows module content to be attached to an environment at environment definition time. This happens before modules specified in a Puppetfile are attached to an environment, which does not happen until deploy time. Environment module implementation depends on the environment source type.

data/doc/dynamic-environments/git-environments.mkd

@@ -29,7 +29,7 @@ seamlessly reflected in Puppet environments. This means that creating a new Git
 branch creates a new Puppet environment, updating a Git branch will update that
 environment, and deleting a Git branch will remove that environment.
 
-R10k supports both [directory and config file environments](https://docs.puppetlabs.com/puppet/latest/reference/environments.html).
+R10k supports both [directory and config file environments](https://puppet.com/docs/puppet/latest/env_environments.html).
 Ensure that the basedir for your sources and your puppet config align.
 
 How it works

data/doc/dynamic-environments/master-configuration.mkd

@@ -1,70 +1,40 @@
 Puppet master configuration
 ===========================
 
-In order to use dynamic environments, your Puppet masters will need to be
+In order to use environments, your Puppet masters will need to be
 configured to load manifests and modules relative to the requested environment.
-The following settings should be configured in puppet.conf.
 
-## Puppet <= 3.4.x
+This is the default behavior in Puppet 4+
 
-In Puppet 3.4 and earlier, dynamic environments are achieved by interpolating
-the `$environment` variable in the modulepath, manifestdir, and manifest
-settings. When environments is loaded the settings are dynamically set based on
-the name of the environment, thus allowing environments to be created on the
-fly.
+This behavior is controled by the following settings
+(listed with their default values):
 
-    [master]
-        modulepath = /etc/puppet/environments/$environment/modules:/etc/puppet/environments/$environment/dist
+```
+codedir = /etc/puppetlabs/code
+environmentpath = $codedir/environments
+basemodulepath = $codedir/modules:/opt/puppetlabs/puppet/share/modules
+```
 
-        # If you use a top level manifest dir
-        manifestdir = /etc/puppet/environments/$environment/manifests
-        # If you use a specific site wide manifest
-        manifest = /etc/puppet/environments/$environment/manifests/nodes.pp
+The environment requested by the agent or assigned to it by an ENC is looked
+for at `$environmentpath/$environment`.
 
-## Puppet = 3.5.x
+That environment may have an environment.conf file in its base directory that
+specifies its own modulepath. If not, the default computed modulepath for that
+environment is `$environmentpath/$environment/modules:$basemodulepath`.
 
-Puppet 3.5.0 adds initial support for "directory environments", where
-environments are created by enumerating directories in `$environmentpath`.
-This new configuration expect that all modules are in the root of the
-directories in the environment path.
+These configuration variables are documented at
+[Puppet Configuration Reference](https://puppet.com/docs/puppet/latest/configuration.html)
+and their interaction in forming the modulepath is documented at
+[Directories and the Modulepath](https://puppet.com/docs/puppet/latest/dirs_modulepath.html).
+More information can be found about environments in general at
+[Creating Environmnets](https://puppet.com/docs/puppet/latest/environments_creating.html).
 
-    [master]
-    # None of modulepath, manifestdir, or manifest should be enabled
-    # See http://docs.puppetlabs.com/puppet/latest/reference/environments.html#enabling-directory-environments
-    #   for more information on the changes
-        environmentpath = $confdir/environments
+The evolution to using environments in this way was a gradual one from community
+conventions to supported feature and there were several intermediate stages that
+had different configuration requirements within the puppet.conf. Some Puppet
+documentation from the 3.x and 4.x series may no longer be applicable to the
+above which solidified in Puppet 4.x.
 
-Note that these settings go into the `[master]` section; you don't need to
-explicitly configure an environment section for each environment you want to
-use. (But you can if you want.)
-
-## Puppet >= 3.6.0
-
-[environmentconf]: http://docs.puppetlabs.com/puppet/latest/reference/config_file_environment.html
-
-Puppet 3.6.0 adds more fine grained control over how directory environments are
-configured. Each directory based environment can have an
-[`environment.conf`][environmentconf] file in the root of that environment that
-can specify the manifest, modulepath, config_version, and environment_timeout
-for that specific environment.
-
-    # puppet.conf
-    [master]
-    # None of modulepath, manifestdir, or manifest should be enabled
-    # See http://docs.puppetlabs.com/puppet/latest/reference/environments.html#enabling-directory-environments
-    #   for more information on the changes
-        environmentpath = $confdir/environments
-
-- - -
-
-    # environment.conf
-    manifest  = site.pp
-    moduledir = modules:dist
-
-A deployed environment with a Puppetfile will look something like this:
-
-    .
-    ├── Puppetfile
-    ├── environment.conf
-    ├── dist
-    └── modules
+If you need to upgrade away from any intermediate setup see
+[Environments in Puppet 3.8](https://puppet.com/docs/puppet/3.8/environments.html)
+for examples that may help.