r10k 3.13.0 → 3.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13a5d7be1f8e4f5794ad944ef12cfbe32d6a7e704371e8f8137227507857600a
-  data.tar.gz: a9060539b639c566371a1c23c28d19910d63649dedeca67ab4b9237d0a43bfab
+  metadata.gz: da22a28bcd5de19a996ff2c869ba074c402e393d7a3caa2528bb342c75192234
+  data.tar.gz: 9bf7067d8412b9d158f2d47ad96499f90183c5dcade26b97fede2a344b06dca0
 SHA512:
-  metadata.gz: aa2a7a28f9d30468bd17d6e54b4e6202a3290779391efe95409a16648f2eb81a5dd7483964b9c7d3bad8fafbb808fa99a86b6a7c2c0cdac5a5f1d5140e20efd0
-  data.tar.gz: 1ee18ff19d30c76a6f00a9650eff77ccf0cae1f60c789a5ae8c90fd725a8c4197f587d4bc1d6a6288b8452ba1c3db07cdfa30030d7fad7e130639283b5f56722
+  metadata.gz: a2a529ce1b781f858253f1d929e4cf521806a8f11ce9733a4f2dc92dcfd4792d4afcbad9290a814d10a14be645eb0ada9551c542def43f6acc48d28931783bc2
+  data.tar.gz: 31d2a9e5d85f29da1c6f62d77d7bbd83c7ed8c4687bb5035f7185c8b37042e2707e59b623d6603af5d533ee79c44380d4a3a4a1a2a28ddd06af9c723c6f1c1bb

data/.github/workflows/docker.yml

@@ -3,7 +3,7 @@ name: Docker test and publish
 on:
   push:
     branches:
-      - master
+      - main
 
 jobs:
   build-and-publish:

data/.github/workflows/rspec_tests.yml

@@ -59,7 +59,7 @@ jobs:
           bundle --version
 
           # Run tests
-          bundle exec rspec --color --format documentation spec/unit
+          bundle exec rspec --color --format documentation spec
 
       - name: Run tests on Linux
         if: runner.os == 'Linux'
@@ -78,4 +78,4 @@ jobs:
           fi
 
           # Run tests
-          bundle exec rspec --color --format documentation spec/unit
+          bundle exec rspec --color --format documentation spec

data/.travis.yml

@@ -1,7 +1,7 @@
 ---
 language: ruby
 bundler_args: "--without system"
-script: "bundle exec rspec --color --format documentation spec/unit"
+script: "bundle exec rspec --color --format documentation spec"
 notifications:
   email: false
 sudo: false

data/CHANGELOG.mkd

@@ -4,6 +4,16 @@ CHANGELOG
 Unreleased
 ----------
 
+3.14.0
+------
+
+- Record unprocessed environment name, so that `strip_component` does not cause truncated environment names to be used as git branches, resulting in errors or incorrect deploys. [#1240](https://github.com/puppetlabs/r10k/pull/1240)
+- (CODEMGMT-1294) Resync repos with unresolvable refs [#1239](https://github.com/puppetlabs/r10k/pull/1239)
+- (RK-378) Restore access to the environment name from the Puppetfile [#1241](https://github.com/puppetlabs/r10k/pull/1241)
+- (CODEMGMT-1300) Ensure the remote url in rugged cache directories is current [#1245](https://github.com/puppetlabs/r10k/pull/1245)
+- Add support for tarball module type, allowing module content to be packaged and sourced from generic fileservers [#1244](https://github.com/puppetlabs/r10k/pull/1244)
+- Add experimental support for tarball environment type, allowing whole environments to be packaged and sourced from generic fileservers [#1244](https://github.com/puppetlabs/r10k/pull/1244)
+
 3.13.0
 ------
 

data/doc/dynamic-environments/configuration.mkd

@@ -796,3 +796,27 @@ development:
       source: https://github.com/reidmv/reidmv-xampl.git
       version: 62d07f2
 ```
+
+### Tarball Environment Type
+
+The tarball environment type allows an environment to be deployed from a tarball archive, rather than a Git repository. When using a tarball environment type, a source location for the tarball is required. Optionally, the tarball's sha256 checksum may be specified as the version. It is highly recommended to include a version specifier. If a version specifier is not included, r10k will never invalidate a cached copy of the tarball's source.
+
+Tarball environment sources will be unpacked directly into the environment root.
+
+```yaml
+---
+production:
+  type: tarball
+  source: https://repo.example.com/projects/puppet/env-2.36.1.tar.gz
+  version: 99a906c99c2f144de43f2ae500509a7474ed11c583fb623efa8e5b377a3157f0 # sha256digest
+
+development:
+  type: tarball
+  source: https://repo.example.com/projects/puppet/env-6128ada.tar.gz
+  version: 6128ada158622cd90f8e1360fb7c2c3830a812d1ec26ddf0db7eb16d61b7293f # sha256digest
+  modules:
+    reidmv-xampl:
+      type: git
+      source: https://github.com/reidmv/reidmv-xampl.git
+      version: 62d07f2
+```

data/doc/puppetfile.mkd

@@ -239,6 +239,19 @@ credentials may be visible in the process table when r10k is running. If you
 choose to supply SVN credentials make sure that the system running r10k is
 appropriately secured.
 
+### Tarball
+
+Modules can be installed from tarball archives. A tarball module must specify a source URL to retreive the tarball content from. A tarball module may optionally specify a sha256 checksum as the module version.
+
+    mod 'puppetlabs-apache',
+      type: 'tarball',
+      source: 'https://repo.example.com/puppet/modules/puppetlabs-apache-7.0.0.tar.gz',
+      version: 'aedd6dc1a5136c6a1a1ec2f285df2a70b0fe4c9effb254b5a1f58116e4c1659e' # sha256 digest
+
+If no version is specified, a tarball will be downloaded from the given source and cached. The cache will not be invalidated until the source URL is changed, or a sha256 checksum version is provided.
+
+Tarball module content will be unpacked directly into an appropriately named module directory. For example, the puppetlabs-apache-7.0.0.tar.gz archive in the example above will be unpacked into `<environment-dir>/modules/apache/`.
+
 ### Local
 
 In the event you want to store locally written modules in your r10k-managed

data/lib/r10k/environment/name.rb

@@ -6,17 +6,22 @@ module R10K
     class Name
 
       # @!attribute [r] name
-      #   @return [String] The unmodified name of the environment
+      #   @return [String] The functional name of the environment derived from inputs and options.
       attr_reader :name
 
+      # @!attribute [r] original_name
+      #   @return [String] The unmodified name originally given to create the object.
+      attr_reader :original_name
+
       INVALID_CHARACTERS = %r[\W]
 
-      def initialize(name, opts)
+      def initialize(original_name, opts)
         @source  = opts[:source]
         @prefix  = opts[:prefix]
         @invalid = opts[:invalid]
 
-        @name = derive_name(name, opts[:strip_component])
+        @name = derive_name(original_name, opts[:strip_component])
+        @original_name = original_name
         @opts = opts
 
         case @invalid
@@ -71,8 +76,8 @@ module R10K
 
       private
 
-      def derive_name(name, strip_component)
-        return name unless strip_component
+      def derive_name(original_name, strip_component)
+        return original_name unless strip_component
 
         unless strip_component.is_a?(String)
           raise _('Improper configuration value given for strip_component setting in %{src} source. ' \
@@ -82,11 +87,11 @@ module R10K
 
         if %r{^/.*/$}.match(strip_component)
           regex = Regexp.new(strip_component[1..-2])
-          name.gsub(regex, '')
-        elsif name.start_with?(strip_component)
-          name[strip_component.size..-1]
+          original_name.gsub(regex, '')
+        elsif original_name.start_with?(strip_component)
+          original_name[strip_component.size..-1]
         else
-          name
+          original_name
         end
       end
 

data/lib/r10k/environment/tarball.rb

@@ -0,0 +1,78 @@
+require 'r10k/util/setopts'
+require 'r10k/tarball'
+require 'r10k/environment'
+
+class R10K::Environment::Tarball < R10K::Environment::WithModules
+
+  R10K::Environment.register(:tarball, self)
+
+  # @!attribute [r] tarball
+  #   @api private
+  #   @return [R10K::Tarball]
+  attr_reader :tarball
+
+  include R10K::Util::Setopts
+
+  # Initialize the given tarball environment.
+  #
+  # @param name [String] The unique name describing this environment.
+  # @param basedir [String] The base directory where this environment will be created.
+  # @param dirname [String] The directory name for this environment.
+  # @param options [Hash] An additional set of options for this environment.
+  #
+  # @param options [String] :source Where to get the tarball from
+  # @param options [String] :version The sha256 digest of the tarball
+  def initialize(name, basedir, dirname, options = {})
+    super
+    setopts(options, {
+      # Standard option interface
+      :type      => ::R10K::Util::Setopts::Ignore,
+      :source    => :self,
+      :version   => :checksum,
+
+      # Type-specific options
+      :checksum => :self,
+    })
+
+    @tarball = R10K::Tarball.new(name, @source, checksum: @checksum)
+  end
+
+  def path
+    @path ||= Pathname.new(File.join(@basedir, @dirname))
+  end
+
+  def sync
+    tarball.get unless tarball.cache_valid?
+    case status
+    when :absent, :mismatched
+      tarball.unpack(path.to_s)
+      # Untracked files left behind from previous extractions are expected to
+      # be deleted by r10k's purge facility.
+    end
+  end
+
+  def status
+    if not path.exist?
+      :absent
+    elsif not (tarball.cache_valid? && tarball.insync?(path.to_s, ignore_untracked_files: true))
+      :mismatched
+    else
+      :insync
+    end
+  end
+
+  def signature
+    @checksum || @tarball.cache_checksum
+  end
+
+  include R10K::Util::Purgeable
+
+  # Returns an array of the full paths to all the content being managed.
+  # @note This implements a required method for the Purgeable mixin
+  # @return [Array<String>]
+  def desired_contents
+    desired = []
+    desired += @tarball.paths.map { |entry| File.join(@full_path, entry) }
+    desired += super
+  end
+end

data/lib/r10k/environment.rb

@@ -34,5 +34,6 @@ module R10K
     require 'r10k/environment/bare'
     require 'r10k/environment/git'
     require 'r10k/environment/svn'
+    require 'r10k/environment/tarball'
   end
 end

data/lib/r10k/forge/module_release.rb

@@ -1,5 +1,6 @@
 require 'r10k/logging'
 require 'r10k/settings/mixin'
+require 'r10k/util/cacheable'
 require 'fileutils'
 require 'tmpdir'
 require 'puppet_forge'
@@ -13,7 +14,7 @@ module R10K
 
       def_setting_attr :proxy
       def_setting_attr :baseurl
-      def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/cache': '/root/.r10k/cache')
+      def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir
 
       include R10K::Logging
 

data/lib/r10k/git/cache.rb

@@ -3,6 +3,7 @@ require 'r10k/git'
 require 'r10k/settings'
 require 'r10k/instance_cache'
 require 'forwardable'
+require 'r10k/util/cacheable'
 
 # Cache Git repository mirrors for object database reuse.
 #
@@ -15,18 +16,9 @@ require 'forwardable'
 class R10K::Git::Cache
 
   include R10K::Settings::Mixin
+  include R10K::Util::Cacheable
 
-  #@api private
-  def self.determine_cache_root
-    if R10K::Util::Platform.windows?
-      File.join(ENV['LOCALAPPDATA'], 'r10k', 'git')
-    else
-      File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git')
-    end
-  end
-  private_class_method :determine_cache_root
-
-  def_setting_attr :cache_root, determine_cache_root
+  def_setting_attr :cache_root, R10K::Util::Cacheable.default_cachedir('git')
 
   @instance_cache = R10K::InstanceCache.new(self)
 
@@ -109,8 +101,7 @@ class R10K::Git::Cache
 
   alias cached? exist?
 
-  # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @sanitized_dirname ||= @remote.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= super(@remote)
   end
 end

data/lib/r10k/git/rugged/base_repository.rb

@@ -20,7 +20,8 @@ class R10K::Git::Rugged::BaseRepository
     else
       object.oid
     end
-  rescue ::Rugged::ReferenceError
+  rescue ::Rugged::ReferenceError, ::Rugged::OdbError => e
+    logger.debug2(_("Unable to resolve %{pattern}: %{e} ") % {pattern: pattern, e: e })
     nil
   end
 
@@ -60,6 +61,16 @@ class R10K::Git::Rugged::BaseRepository
     remotes_hash
   end
 
+  # Update a remote URL
+  # @param [String] The remote URL of the git repository
+  # @param [String] An optional remote name for the git repository
+  def update_remote(remote, remote_name='origin')
+    if @_rugged_repo
+      logger.debug2(_("Remote URL is different from cache, updating %{orig} to %{update}") % {orig: remotes[remote_name], update: remote})
+      @_rugged_repo.remotes.set_url(remote_name, remote)
+    end
+  end
+
   private
 
   def with_repo(opts={})

data/lib/r10k/git/rugged/cache.rb

@@ -8,4 +8,12 @@ class R10K::Git::Rugged::Cache < R10K::Git::Cache
   def self.bare_repository
     R10K::Git::Rugged::BareRepository
   end
+
+  # Update the remote URL if the cache differs from the current configuration
+  def sync!
+    if cached? && @repo.remotes['origin'] != @remote
+      @repo.update_remote(@remote)
+    end
+    super
+  end
 end

data/lib/r10k/git/stateful_repository.rb

@@ -86,6 +86,8 @@ class R10K::Git::StatefulRepository
       :mismatched
     elsif !(@repo.origin == @remote)
       :mismatched
+    elsif @repo.head.nil?
+      :mismatched
     elsif @repo.dirty?
       :dirty
     elsif !(@repo.head == @cache.resolve(ref))

data/lib/r10k/initializers.rb

@@ -4,6 +4,7 @@ require 'r10k/git'
 require 'r10k/git/cache'
 
 require 'r10k/forge/module_release'
+require 'r10k/tarball'
 
 module R10K
   module Initializers
@@ -36,10 +37,13 @@ module R10K
 
         with_setting(:cachedir) { |value| R10K::Git::Cache.settings[:cache_root] = value }
         with_setting(:cachedir) { |value| R10K::Forge::ModuleRelease.settings[:cache_root] = value }
+        with_setting(:cachedir) { |value| R10K::Tarball.settings[:cache_root] = value }
         with_setting(:pool_size) { |value| R10K::Puppetfile.settings[:pool_size] = value }
+        with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value }
 
         with_setting(:git) { |value| GitInitializer.new(value).call }
         with_setting(:forge) { |value| ForgeInitializer.new(value).call }
+        with_setting(:tarball) { |value| TarballInitializer.new(value).call }
       end
     end
 
@@ -79,5 +83,11 @@ module R10K
         with_setting(:authorization_token) { |value| PuppetForge::Connection.authorization = value }
       end
     end
+
+    class TarballInitializer < BaseInitializer
+      def call
+        with_setting(:proxy) { |value| R10K::Tarball.settings[:proxy] = value }
+      end
+    end
   end
 end

data/lib/r10k/module/tarball.rb

@@ -0,0 +1,101 @@
+require 'r10k/module'
+require 'r10k/util/setopts'
+require 'r10k/tarball'
+
+# This class defines a tarball source module implementation
+class R10K::Module::Tarball < R10K::Module::Base
+
+  R10K::Module.register(self)
+
+  def self.implement?(name, args)
+    args.is_a?(Hash) && args[:type].to_s == 'tarball'
+  rescue
+    false
+  end
+
+  def self.statically_defined_version(name, args)
+    args[:version] || args[:checksum]
+  end
+
+  # @!attribute [r] tarball
+  #   @api private
+  #   @return [R10K::Tarball]
+  attr_reader :tarball
+
+  include R10K::Util::Setopts
+
+  def initialize(name, dirname, opts, environment=nil)
+    super
+    setopts(opts, {
+      # Standard option interface
+      :source    => :self,
+      :version   => :checksum,
+      :type      => ::R10K::Util::Setopts::Ignore,
+      :overrides => :self,
+
+      # Type-specific options
+      :checksum => :self,
+    })
+
+    @tarball = R10K::Tarball.new(name, @source, checksum: @checksum)
+  end
+
+  # Return the status of the currently installed module.
+  #
+  # @return [Symbol]
+  def status
+    if not path.exist?
+      :absent
+    elsif not (tarball.cache_valid? && tarball.insync?(path.to_s))
+      :mismatched
+    else
+      :insync
+    end
+  end
+
+  # Synchronize this module with the indicated state.
+  # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
+  def sync(opts={})
+    tarball.get unless tarball.cache_valid?
+    if should_sync?
+      case status
+      when :absent
+        tarball.unpack(path.to_s)
+      when :mismatched
+        path.rmtree
+        tarball.unpack(path.to_s)
+      end
+      maybe_delete_spec_dir
+      true
+    else
+      false
+    end
+  end
+
+  # Return the desired version of this module
+  def version
+    @checksum || '(present)'
+  end
+
+  # Return the properties of the module
+  #
+  # @return [Hash]
+  # @abstract
+  def properties
+    {
+      :expected => version,
+      :actual   => ((state = status) == :insync) ? version : state,
+      :type     => :tarball,
+    }
+  end
+
+  # Tarball caches are files, not directories. An important purpose of this
+  # method is to indicate where the cache "path" is, for locking/parallelism,
+  # so for the Tarball module type, the relevant path location is returned.
+  #
+  # @return [String] The path this module will cache its tarball source to
+  def cachedir
+    tarball.cache_path
+  end
+end

data/lib/r10k/module.rb

@@ -53,4 +53,5 @@ module R10K::Module
   require 'r10k/module/local'
   require 'r10k/module/forge'
   require 'r10k/module/definition'
+  require 'r10k/module/tarball'
 end

data/lib/r10k/module_loader/puppetfile.rb

@@ -16,7 +16,8 @@ module R10K
 
       attr_accessor :default_branch_override, :environment
       attr_reader :modules, :moduledir, :puppetfile_path,
-        :managed_directories, :desired_contents, :purge_exclusions
+        :managed_directories, :desired_contents, :purge_exclusions,
+        :environment_name
 
       # @param basedir [String] The path that contains the moduledir &
       #     Puppetfile by default. May be an environment, project, or
@@ -40,6 +41,7 @@ module R10K
         @puppetfile_path  = resolve_path(@basedir, puppetfile)
         @overrides   = overrides
         @environment = environment
+        @environment_name = @environment&.name
         @default_branch_override = @overrides.dig(:environments, :default_branch_override)
         @allow_puppetfile_forge = @overrides.dig(:forge, :allow_puppetfile_override)
 
@@ -165,6 +167,13 @@ module R10K
         @modules << mod
       end
 
+      # @deprecated
+      # @return [String] The base directory that contains the Puppetfile
+      def basedir
+        logger.warn _('"basedir" is deprecated. Please use "environment_name" instead. "basedir" will be removed in a future version.')
+        @basedir
+      end
+
      private
 
       def empty_load_output

data/lib/r10k/source/git.rb

@@ -100,26 +100,26 @@ class R10K::Source::Git < R10K::Source::Base
 
   def generate_environments
     envs = []
-    branch_names.each do |bn|
-      if bn.valid?
-        envs << R10K::Environment::Git.new(bn.name,
+    environment_names.each do |en|
+      if en.valid?
+        envs << R10K::Environment::Git.new(en.name,
                                            @basedir,
-                                           bn.dirname,
+                                           en.dirname,
                                            {remote: remote,
-                                            ref: bn.name,
+                                            ref: en.original_name,
                                             puppetfile_name: puppetfile_name,
                                             overrides: @options[:overrides]})
-      elsif bn.correct?
-       logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: bn.name.inspect, corrected_env_name: bn.dirname}
-        envs << R10K::Environment::Git.new(bn.name,
+      elsif en.correct?
+       logger.warn _("Environment %{env_name} contained non-word characters, correcting name to %{corrected_env_name}") % {env_name: en.name.inspect, corrected_env_name: en.dirname}
+        envs << R10K::Environment::Git.new(en.name,
                                            @basedir,
-                                           bn.dirname,
+                                           en.dirname,
                                            {remote: remote,
-                                            ref: bn.name,
+                                            ref: en.original_name,
                                             puppetfile_name: puppetfile_name,
                                             overrides: @options[:overrides]})
-      elsif bn.validate?
-       logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: bn.name.inspect}
+      elsif en.validate?
+       logger.error _("Environment %{env_name} contained non-word characters, ignoring it.") % {env_name: en.name.inspect}
       end
     end
 
@@ -157,22 +157,22 @@ class R10K::Source::Git < R10K::Source::Base
 
   private
 
-  def branch_names
+  def environment_names
     opts = {prefix: @prefix,
             invalid: @invalid_branches,
             source: @name,
             strip_component: @strip_component}
-    branches = @cache.branches
+    branch_names = @cache.branches
     if @ignore_branch_prefixes && !@ignore_branch_prefixes.empty?
-      branches = filter_branches_by_regexp(branches, @ignore_branch_prefixes)
+      branch_names = filter_branches_by_regexp(branch_names, @ignore_branch_prefixes)
     end
 
     if @filter_command && !@filter_command.empty?
-      branches = filter_branches_by_command(branches, @filter_command)
+      branch_names = filter_branches_by_command(branch_names, @filter_command)
     end
 
-    branches.map do |branch|
-      R10K::Environment::Name.new(branch, opts)
+    branch_names.map do |branch_name|
+      R10K::Environment::Name.new(branch_name, opts)
     end
   end
 end