r10k 3.3.2 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c887307303f3824d75c36c71bba941d2727eb5ab
-  data.tar.gz: fde4db259354fcec049b96538bf2440f4466e021
+SHA256:
+  metadata.gz: 2f2418e73622a4149507cc7e2dde74b68a448689da2a7287b19107fbef0e184e
+  data.tar.gz: 95acaeb5cb7ad0969ec3f2a3545ab82490bafb9d0fca5da444125bbe780b9836
 SHA512:
-  metadata.gz: 3296cd805a5494b27e8bda6ff93644eae2b31feb73246bf5fccd1ba470cfe6a53bd62f6440f65fa8484cbb158c6b7c16b8534bfea5a0b92336c664fd03f44bb0
-  data.tar.gz: 4ea29b8f13acddc4fbc89dc7d84ed458967ab95383fd6362f03a01d4f51f890aef944f4cc7a97d708f167dd1fd6e4dd6c0131976017e5fa36a58b0ba28f22e03
+  metadata.gz: dd666a18fe5df938ab5a151cf411ac335e0f3f151428fe8589010ff699b832609fc58e189c46f52894d9d6b9d8e63156ed928bd14d2ed02d5d9a113fc0c5feb0
+  data.tar.gz: 40307661c96a21d00595271fa21836175c993dc937504b82b449218e39cd3804000a7c24c32a52aa570098b450fd35417573190dd9d603ac087e217f391aacb4

data/.github/pull_request_template.md

@@ -0,0 +1 @@
+Please add all notable changes to the "Unreleased" section of the CHANGELOG.

data/.github/workflows/docker.yml

@@ -0,0 +1,56 @@
+name: Docker test and publish
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build-and-publish:
+    env:
+      PUPPERWARE_ANALYTICS_STREAM: production
+      IS_LATEST: true
+      # Still need these env vars for README publishing
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@master
+      - uses: azure/docker-login@v1
+        with: # This doesn't seem to work unless we point directly to the secrets
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.6.x
+      - run: gem install bundler
+      - name: Build container
+        working-directory: docker
+        run: make lint build test
+      - name: Publish container
+        working-directory: docker
+        run: |
+          make publish
+      - name: Build release container
+        env:
+          IS_RELEASE: true
+        working-directory: docker
+        run: |
+          if make prep; then
+            make lint build test
+          else
+            echo "Skipping release container building and testing"
+          fi
+      - name: Publish release container
+        env:
+          IS_RELEASE: true
+        working-directory: docker
+        run: |
+          if make prep; then
+            make publish
+          else
+            echo "Skipping release container publishing"
+          fi

data/.github/workflows/release.yml

@@ -0,0 +1,36 @@
+name: Tag and release
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'lib/r10k/version.rb'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - name: Bump version and push tag
+        uses: anothrNick/github-tag-action@1.17.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUPPET_RELEASE_GH_TOKEN }}
+          DEFAULT_BUMP: patch
+          WITH_V: false
+          # Uncomment this if the tag and version file become out-of-sync and
+          # you need to tag at a specific version.
+          # CUSTOM_TAG:
+      - name: Build gem
+        uses: scarhand/actions-ruby@master
+        with:
+          args: build *.gemspec
+      - name: Publish gem
+        uses: scarhand/actions-ruby@master
+        env:
+          RUBYGEMS_AUTH_TOKEN: ${{ secrets.RUBYGEMS_AUTH_TOKEN }}
+        with:
+          args: push *.gem
+

data/.travis.yml

@@ -1,17 +1,19 @@
 ---
 language: ruby
-services:
-  - docker
 bundler_args: "--without system"
 script: "bundle exec rspec --color --format documentation spec/unit"
 notifications:
   email: false
 sudo: false
 jdk:
-  - openjdk8
+  - openjdk11
 before_install: gem install bundler -v '< 2' --no-document
 matrix:
   include:
+    - stage: r10k tests
+      rvm: 2.7.0
+    - stage: r10k tests
+      rvm: 2.6.5
     - stage: r10k tests
       rvm: 2.5.0
     - stage: r10k tests
@@ -21,10 +23,23 @@ matrix:
     - stage: r10k tests
       rvm: jruby
     - stage: r10k container tests
+      dist: focal
       language: ruby
-      rvm: 2.5.5
+      services:
+        - docker
+      rvm: 2.6.6
+      env:
+        - DOCKER_COMPOSE_VERSION=1.25.5
+        # necessary to prevent overwhelming TravisCI build output limits
+        - DOCKER_BUILD_FLAGS="--progress plain"
+      before_install:
+        - sudo rm /usr/local/bin/docker-compose
+        - curl --location https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname --kernel-name`-`uname --machine` > docker-compose
+        - chmod +x docker-compose
+        - sudo mv docker-compose /usr/local/bin
       script:
-        - |
-          set -ex
-          cd docker && make lint build test
-          set +x
+        - set -e
+        - cd docker
+        - make lint
+        - make build
+        - make test

data/CHANGELOG.mkd

@@ -1,6 +1,46 @@
 CHANGELOG
 =========
 
+Unreleased
+----
+
+3.5.1
+-----
+
+- Upgrade Rugged Gemfile dependency for local development to ~> 1.0
+
+
+3.5.0
+-----
+
+- Add exec environment source type. The exec source type allows for the
+  implementation of external environment sources
+  [#1042](https://github.com/puppetlabs/r10k/pull/1042).
+- Improve atomicity of .r10k-deploy.json writes. Fixes
+  [#813](https://github.com/puppetlabs/r10k/issues/813)
+
+3.4.1
+-----
+
+- Add support for Ruby 2.7
+- (RK-357) Restrict gettext and fast_gettext versions for compatibility with Ruby 2.4
+- Bump cri to 2.15.10
+- (RK-360) Allow overriding the cachedir in `deploy module`
+- Bump puppet_forge to 2.3.0
+
+3.4.0
+-----
+
+  - Switch to supported `colored2` gem
+  - Add YAML environment source types [#983](https://github.com/puppetlabs/r10k/pull/983), [#1002](https://github.com/puppetlabs/r10k/pull/1002). YAML source types use configuration files to define environments, rather than VCS branches, and support deploying modules to an environment without modifying a control-repo. YAML environment source types are experimental in this release
+
+3.3.3
+-----
+
+### Changes
+
+  - Update minitar version to 0.9, to match the version installed with puppet-agent
+
 3.3.2
 -----
 
@@ -131,6 +171,22 @@ Ruby >= 2.3 official and documented.
     - Unable to parse Puppetfile
   - Various perfomance improvements
 
+2.6.8
+----
+
+### Changes
+
+(RK-357) Restrict gettext and fast_gettext versions for compatibility with Ruby 2.4.
+(RK-358) Update puppet_forge to ~> 2.3.0.
+
+2.6.7
+----
+
+### Changes
+
+(CDPE-1813) Add module deploy info to .r10k-deploy.json.
+(RK-351) Update minitar to ~> 0.9.0.
+
 2.6.6
 ----
 
@@ -343,7 +399,7 @@ Puppetfile should be installed to. See the [Puppetfile documentation](https://gi
 
 You can now configure how r10k purges unmanaged content after a deployment. The
 default behavior should be unchanged but there is a new "purge\_levels" configuration
-option that can be used to enable new behavior or de-activate certain existing 
+option that can be used to enable new behavior or de-activate certain existing
 behaviors. See the relevant [configuration documentation](https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#purge_levels) for more details.
 
 (RK-223) Ability to track control repo branch from content declarations.
@@ -390,7 +446,7 @@ fix.)
 Previously, r10k only supported the use of HTTP proxies for connecting to the Puppet
 Forge. With these changes, r10k can now be configured to use an HTTP proxy for both
 Forge and Git operations. Configuration can be specified globally, for Forge or Git
-only, or on a per-Git repository basis. See [configuration documentation](https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd) 
+only, or on a per-Git repository basis. See [configuration documentation](https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd)
 for more details.
 
 ### Bug Fixes
@@ -411,9 +467,9 @@ for SSH.
 (RK-241) "deploy display" action does not properly format wrapped exceptions
 
 The "deploy display" action was not capturing and logging exceptions in the same way as
-other related actions. This meant that in many cases, when an error occurred, the 
+other related actions. This meant that in many cases, when an error occurred, the
 underlying cause was not being shown. Specifically, the "deploy display" action was
-not benefitting from the improved error messaging for unreadable SSH keys which was 
+not benefitting from the improved error messaging for unreadable SSH keys which was
 added in r10k 2.2.0 as part of RK-220.
 
 2.2.1

data/CODEOWNERS

@@ -1 +1,2 @@
+*        @puppetlabs/puppetserver-maintainers @adrienthebo @dhollinger
 /docker/ @puppetlabs/pupperware

data/Gemfile

@@ -2,7 +2,7 @@ source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 gemspec
 
 group :extra do
-  gem 'rugged', '>= 0.24.0', '< 0.24.6', :platforms => :ruby
+  gem 'rugged', '~> 1.0', :platforms => :ruby
 end
 
 group :development do

data/README.mkd

@@ -8,16 +8,17 @@ Puppet environment and module deployment
 Description
 -----------
 
-[workflow]: http://puppetlabs.com/blog/git-workflow-and-puppet-environments/
+[workflow]: https://puppet.com/docs/puppet/latest/environments_about.html
 
 R10k provides a general purpose toolset for deploying Puppet environments and
 modules. It implements the [Puppetfile](doc/puppetfile.mkd) format and provides a native
-implementation of Puppet [dynamic environments][workflow].
+implementation of Puppet [environments][workflow].
 
 Requirements
 ------------
 
-R10k supports the Ruby versions `>= 2.3.0`.
+R10k supports the Ruby versions `>= 2.3.0`. It's tested on Ruby 2.3.0 up to
+Ruby 2.6.0 + Jruby.
 
 R10k requires additional components, depending on how you plan on managing
 environments and modules.

data/azure-pipelines.yml

@@ -17,7 +17,8 @@ variables:
   NAMESPACE: puppet
   CONTAINER_NAME: r10k
   CONTAINER_BUILD_PATH: .
-  LINT_IGNORES: DL3008 DL3018 DL4000 DL4001 DL3028
+  LINT_IGNORES:
+  DOCKER_BUILDKIT: 1
 
 workspace:
   clean: resources
@@ -31,26 +32,30 @@ steps:
     $gempath = Join-Path -Path (Get-Location) -ChildPath 'docker/.bundle/gems'
     bundle config --local gemfile $gemfile
     bundle config --local path $gempath
-    bundle install
+    bundle install --with test
   displayName: Fetch Dependencies
+  timeoutInMinutes: 1
   name: fetch_deps
 
 - powershell: |
     . "$(bundle show pupperware)/ci/build.ps1"
     Write-HostDiagnostics
   displayName: Diagnostic Host Information
+  timeoutInMinutes: 2
   name: hostinfo
 
 - powershell: |
     . "$(bundle show pupperware)/ci/build.ps1"
     Lint-Dockerfile -Name $ENV:CONTAINER_NAME -Ignore ($ENV:LINT_IGNORES -split ' ')
   displayName: Lint $(CONTAINER_NAME) Dockerfile
+  timeoutInMinutes: 1
   name: lint_dockerfile
 
 - powershell: |
     . "$(bundle show pupperware)/ci/build.ps1"
     Build-Container -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE -PathOrUri $ENV:CONTAINER_BUILD_PATH
   displayName: Build $(CONTAINER_NAME) Container
+  timeoutInMinutes: 5
   name: build_container
 
 - powershell: |
@@ -63,6 +68,7 @@ steps:
     . "$(bundle show pupperware)/ci/build.ps1"
     Invoke-ContainerTest -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE
   displayName: Test $(CONTAINER_NAME)
+  timeoutInMinutes: 5
   name: test_container
 
 - task: PublishTestResults@2
@@ -76,5 +82,5 @@ steps:
     . "$(bundle show pupperware)/ci/build.ps1"
     Clear-BuildState -Name $ENV:CONTAINER_NAME -Namespace $ENV:NAMESPACE
   displayName: Container Cleanup
-  timeoutInMinutes: 3
+  timeoutInMinutes: 4
   condition: always()

data/bin/r10k

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'r10k/cli'
-require 'colored'
+require 'colored2'
 
 begin
   R10K::CLI.command.run(ARGV)

data/doc/dynamic-environments/configuration.mkd

@@ -48,7 +48,7 @@ cachedir: '/var/cache/r10k'
 
 
 The cachedir setting defaults to `~/.r10k`. If the HOME environment variable is
-unset r10k will assume that r10k is being run with the Puppet [`prerun_command`](http://docs.puppetlabs.com/references/latest/configuration.html#preruncommand)
+unset r10k will assume that r10k is being run with the Puppet [`prerun_command`](https://puppet.com/docs/puppet/latest/configuration.html#preruncommand)
 setting and will set the cachedir default to `/root/.r10k`.
 
 ### proxy
@@ -263,7 +263,7 @@ After Puppetfile content for a given environment is deployed, r10k will
 recursively remove any content found in a directory managed by the Puppetfile
 which is not also declared in that Puppetfile. Directories considered to be
 managed by a Puppetfile include the configured `moduledir` (which defaults to
-"modules") as well as alternate directories specified as an `install\_path`
+"modules") as well as alternate directories specified as an `install_path`
 option to any Puppetfile content declarations.
 
 #### purge\_whitelist
@@ -509,3 +509,161 @@ This will create the following directory structure:
 |-- app1_production  # app1 data repository, production branch
 |-- app1_develop     # app1 data repository, develop branch
 ```
+
+Experimental Features
+---------------------
+
+### YAML Environment Source
+
+Dynamically deploying Puppet content based on the state of version control repositories can be powerful and efficient for development workflows. The linkage however is not advantageous when trying to build precision controls over deployment of previously-developed and tested content.
+
+The YAML environment source type allows for a clear separation of tooling between development workflow, and deployment workflow. Development workflow creates new commits in the version control system. Deployment workflow consumes them.
+
+To use the YAML environment source, configure r10k's sources with at least one entry using the yaml type.
+
+```yaml
+# r10k.yaml
+---
+sources:
+  puppet:
+    type: yaml
+    basedir: /etc/puppetlabs/code/environments
+    config: /etc/puppetlabs/r10k/environments.yaml # default
+```
+
+When using the YAML source type, every environment is enumerated in a single yaml file. Each environment specifies a type, source, and version (typically a Git ref) to deploy. In the following example, two environments are defined, which are identical to each other.
+
+```yaml
+---
+production:
+  type: git
+  remote: git@github.com:puppetlabs/control-repo.git
+  ref: 8820892
+
+development:
+  type: git
+  remote: git@github.com:puppetlabs/control-repo.git
+  ref: 8820892
+```
+
+### YAMLdir Environment Source
+
+Like the YAML environment source, but implemented as a conf.d pattern.
+
+```yaml
+# r10k.yaml
+---
+sources:
+  puppet:
+    type: yamldir
+    basedir: /etc/puppetlabs/code/environments
+    config: /etc/puppetlabs/r10k/environments.d # default
+```
+
+Each environment is defined in a yaml file placed in the configuration directory. The filename, without the .yaml extension, will be the name of the environment.
+
+```
+/etc/puppetlabs/r10k/environments.d
+├── production.yaml
+└── development.yaml
+```
+
+The contents of the file should be a hash specifying the enviornment type, and all other applicable environment options.
+
+```yaml
+# production.yaml
+---
+type: git
+remote: git@github.com:puppetlabs/control-repo.git
+ref: 8820892
+```
+
+### Exec environment Source
+
+The exec environment source runs an external command which is expected to return on stdout content compatible with the YAML environment source data format. The command may return the data in JSON or YAML form. The exec environment source is similar in purpose to Puppet's exec node terminus, used to implement external node classifiers (ENCs). R10k's exec source type allows the the implementation of external environment sources.
+
+```yaml
+# r10k.yaml
+---
+sources:
+  puppet:
+    type: exec
+    basedir: /etc/puppetlabs/code/environments
+    command: /usr/local/bin/r10k-environments.sh
+```
+
+### Environment Modules
+
+The environment modules feature allows module content to be attached to an environment at environment definition time. This happens before modules specified in a Puppetfile are attached to an environment, which does not happen until deploy time. Environment module implementation depends on the environment source type.
+
+For the YAML environment source type, attach modules to an environment by specifying a modules key for the environment, and providing a hash of modules to attach. Each module accepts the same arguments accepted by the `mod` method in a Puppetfile.
+
+The example below includes two Forge modules and one module sourced from a Git repository. The two environments are almost identical. However, a new version of the stdlib module has been deployed in development (6.2.0), that has not yet been deployed to production.
+
+```yaml
+---
+production:
+  type: git
+  remote: git@github.com:puppetlabs/control-repo.git
+  ref: 8820892
+  modules:
+    puppetlabs-stdlib: 6.0.0
+    puppetlabs-concat: 6.1.0
+    reidmv-xampl:
+      git: https://github.com/reidmv/reidmv-xampl.git
+      ref: 62d07f2
+
+development:
+  type: git
+  remote: git@github.com:puppetlabs/control-repo.git
+  ref: 8820892
+  modules:
+    puppetlabs-stdlib: 6.2.0
+    puppetlabs-concat: 6.1.0
+    reidmv-xampl:
+      git: https://github.com/reidmv/reidmv-xampl.git
+      ref: 62d07f2
+```
+
+An example of a single environment definition for the YAMLdir environment source type:
+
+```yaml
+# production.yaml
+---
+type: git
+remote: git@github.com:puppetlabs/control-repo.git
+ref: 8820892
+modules:
+  puppetlabs-stdlib: 6.0.0
+  puppetlabs-concat: 6.1.0
+  reidmv-xampl:
+    git: https://github.com/reidmv/reidmv-xampl.git
+    ref: 62d07f2
+```
+
+### Bare Environment Type
+
+A "control repository" typically contains a hiera.yaml, an environment.conf, a manifests/site.pp file, and a few other things. However, none of these are strictly necessary for an environment to be functional if modules can be deployed to it.
+
+The bare environment type allows sources that support environment modules to operate without a control repo being required. Modules can be deployed directly.
+
+```yaml
+---
+production:
+  type: bare
+  modules:
+    puppetlabs-stdlib: 6.0.0
+    puppetlabs-concat: 6.1.0
+    reidmv-xampl:
+      git: https://github.com/reidmv/reidmv-xampl.git
+      ref: 62d07f2
+
+development:
+  type: bare
+  modules:
+    puppetlabs-stdlib: 6.0.0
+    puppetlabs-concat: 6.1.0
+    reidmv-xampl:
+      git: https://github.com/reidmv/reidmv-xampl.git
+      ref: 62d07f2
+```