r10k 2.2.2 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 79e256f57d1cd80b724c490632856750dcf368f4
-  data.tar.gz: 146b029e45575f462cf7438b4a795c9e68fcb5c7
+  metadata.gz: 79a29967cc56c3178634ae325404b4b201aa9841
+  data.tar.gz: 5ddfd39f4a437e52ed87adc06f08945a8c2eef07
 SHA512:
-  metadata.gz: 8479890443ec53087bf5b8469d2d04fd6c0c325f6215a428b536a9bad1a91e67ac7d67d5dccf4786936e39bdc192b09925bfb74a39f262cfedbcd2fc881453d0
-  data.tar.gz: baf085c12deb3a95664a3db9c90475dfba51ca6c9d39c8d0945c2572d040b0fb181daaf40bc74ebec06cf77dd1a4e1e596da472b0b63ae76e7cebf4d97c271e5
+  metadata.gz: 1da246a1e621b9b47f9a5baca4641160c8a71a0ba66f86849f28bac73a1341b586987b154e5e4b97aca66f074fae813c4a153119f2ccc508f34b8fe2ea4c1241
+  data.tar.gz: 64f55efe4f54dae8da9d6ff5c8d5ed28201dd80ca1202e73c9af22c9867e309cb009d3c4bc6454b3b88870656bb751bed68866816d711ad47c64c0dc295d83db

data/CHANGELOG.mkd

@@ -1,6 +1,31 @@
 CHANGELOG
 =========
 
+2.3.0
+-----
+
+2016/05/17
+
+### New Features
+
+(RK-236/RK-237) Added HTTP proxy support for Git operations.
+
+Previously, r10k only supported the use of HTTP proxies for connecting to the Puppet
+Forge. With these changes, r10k can now be configured to use an HTTP proxy for both
+Forge and Git operations. Configuration can be specified globally, for Forge or Git
+only, or on a per-Git repository basis. See [configuration documentation](https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd) 
+for more details.
+
+### Bug Fixes
+
+(RK-238) When r10k encounters and ignores invalid file types in a module archive, it
+will now log the message at the DEBUG1 level instead of at WARN.
+
+(RK-243) In certain cases, when using the "rugged" Git provider, specifying invalid HTTP
+credentials for a repository could result in an infinite loop. Authentication retry
+for HTTP repositories is now capped at 50 attempts which matches the existing behavior
+for SSH.
+
 2.2.2
 -----
 

data/Gemfile

@@ -3,7 +3,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :extra do
-  gem 'rugged', '~> 0.21.4', :platforms => :ruby
+  gem 'rugged', '~> 0.24.0', :platforms => :ruby
 end
 
 group :development do

data/doc/dynamic-environments/configuration.mkd

@@ -52,6 +52,28 @@ The cachedir setting defaults to `~/.r10k`. If the HOME environment variable is
 unset r10k will assume that r10k is being run with the Puppet [`prerun_command`][prerun_command]
 setting and will set the cachedir default to `/root/.r10k`.
 
+### proxy
+
+The 'proxy' setting configures a proxy server to use for all operations which occur over
+an HTTP(S) transport. You can override this setting for Git or Forge operations only by
+setting the 'proxy' setting under the 'git' or 'forge' settings. You can also override
+for a specific Git repository by setting a proxy in the 'repositories' list of the 'git'
+setting. By default, r10k will look for and use the first environment variable it finds
+in this list: HTTPS\_PROXY, https\_proxy, HTTP\_PROXY, http\_proxy. If no proxy setting
+is found in the environment, this setting will default to use no proxy.
+
+```yaml
+proxy: 'http://proxy.example.com:3128'
+```
+
+r10k also supports using authenticated proxies with either Basic or Digest authentication:
+
+```yaml
+proxy: 'http://user:password@proxy.example.com:3128'
+```
+
+The proxy server being used will be logged at the "debug" level when r10k runs.
+
 ### git
 
 The 'git' setting is a hash that contains Git specific settings.
@@ -68,6 +90,12 @@ git:
 See the [git provider documentation](../git/providers.mkd) for more information
 regarding Git providers.
 
+#### proxy
+
+The 'proxy' setting allows you to set or override the global proxy setting specifically
+for Git operations that use an HTTP(S) transport. See the global proxy setting documentation
+for more information and examples.
+
 #### username
 
 The username setting is only used by the Rugged git provider.
@@ -107,10 +135,15 @@ overrides the global private_key setting.
 ```yaml
 git:
   repositories:
-    "ssh://tessier-ashpool.freeside/protected-repo.git":
+    - remote: "ssh://tessier-ashpool.freeside/protected-repo.git"
       private_key: "/etc/puppetlabs/r10k/ssh/id_rsa-protected-repo-deploy-key"
 ```
 
+##### proxy
+
+The 'proxy' setting allows you to set or override the global proxy setting for a single, specific
+repository. See the global proxy setting documentation for more information and examples.
+
 ### forge
 
 The 'forge' setting is a hash that contains settings for downloading modules
@@ -118,16 +151,8 @@ from the Puppet Forge.
 
 #### proxy
 
-The proxy option sets an optional HTTP proxy to use when downloading modules
-from the Forge.
-
-```yaml
-forge:
-  proxy: 'http://my-site.proxy:3128'
-```
-
-If no proxy is given, r10k will check the environment variables 'HTTPS_PROXY',
-'https_proxy', 'HTTP_PROXY', and 'http_proxy' in that order for a proxy URL.
+The 'proxy' setting allows you to set or override the global proxy setting for all Forge
+interactions. See the global proxy setting documentation for more information and examples.
 
 #### baseurl
 

data/integration/tests/basic_functionality/proxy_with_pe_only_module.rb

@@ -0,0 +1,128 @@
+require 'git_utils'
+require 'r10k_utils'
+require 'master_manipulator'
+test_name 'RK-242 '#'- C87652 - Specify the proxy in the r10k.yaml'
+
+confine(:to, :platform => ['el', 'sles'])
+
+#Init
+master_platform = fact_on(master, 'osfamily')
+master_certname = on(master, puppet('config', 'print', 'certname')).stdout.rstrip
+env_path = on(master, puppet('config print environmentpath')).stdout.rstrip
+r10k_fqp = get_r10k_fqp(master)
+
+git_repo_path = '/git_repos'
+git_repo_name = 'environments'
+git_control_remote = File.join(git_repo_path, "#{git_repo_name}.git")
+git_environments_path = '/root/environments'
+last_commit = git_last_commit(master, git_environments_path)
+git_provider = ENV['GIT_PROVIDER']
+
+local_files_root_path = ENV['FILES'] || 'files'
+
+git_manifest_template_path = File.join(local_files_root_path, 'pre-suite', 'git_config.pp.erb')
+git_manifest = ERB.new(File.read(git_manifest_template_path)).result(binding)
+
+r10k_config_path = get_r10k_config_file_path(master)
+r10k_config_bak_path = "#{r10k_config_path}.bak"
+
+case master_platform
+  when 'RedHat'
+    pkg_manager = 'yum'
+  when 'Suse'
+    pkg_manager = 'zypper'
+end
+
+install_squid = "#{pkg_manager} install -y squid"
+remove_squid = "#{pkg_manager} remove -y squid"
+squid_log = "/var/log/squid/access.log"
+
+#In-line files
+r10k_conf = <<-CONF
+cachedir: '/var/cache/r10k'
+git:
+  provider: '#{git_provider}'
+sources:
+  control:
+    basedir: "#{env_path}"
+    remote: "#{git_control_remote}"
+forge:
+  proxy: "http://#{master.hostname}:3128"
+CONF
+
+#Manifest
+site_pp_path = File.join(git_environments_path, 'manifests', 'site.pp')
+site_pp = create_site_pp(master_certname, '  include peonly')
+
+#Verification
+squid_log_regex = /CONNECT forgeapi.puppetlabs.com:443/
+notify_message_regex = /I am in the production environment, this is a PE only module/
+
+#Teardown
+teardown do
+  step 'remove license file'
+  on(master, 'rm -f /etc/puppetlabs/license.key')
+
+  step 'Restore "git" Package'
+  on(master, puppet('apply'), :stdin => git_manifest, :acceptable_exit_codes => [0,2])
+
+  step 'Restore Original "r10k" Config'
+  on(master, "mv #{r10k_config_bak_path} #{r10k_config_path}")
+
+  clean_up_r10k(master, last_commit, git_environments_path)
+
+  step 'Remove Squid'
+  on(master, puppet("apply -e 'service {'squid' : ensure => stopped}'"))
+  on(master, remove_squid)
+end
+
+#Setup
+step 'Stub the forge'
+stub_forge_on(master)
+
+step 'Backup Current "r10k" Config'
+on(master, "mv #{r10k_config_path} #{r10k_config_bak_path}")
+
+step 'Update the "r10k" Config'
+create_remote_file(master, r10k_config_path, r10k_conf)
+
+step 'Download license file from int-resources'
+curl_on(master, 'http://int-resources.ops.puppetlabs.net/QA_resources/r10k/license.key -o /etc/puppetlabs/license.key')
+
+step 'Checkout "production" Branch'
+git_on(master, 'checkout production', git_environments_path)
+
+step 'Inject New "site.pp" to the "production" Environment'
+inject_site_pp(master, site_pp_path, site_pp)
+
+step 'Copy Puppetfile to "production" Environment with PE only module'
+create_remote_file(master, "#{git_environments_path}/Puppetfile", 'mod "ztr-peonly"')
+
+step 'Push Changes'
+git_add_commit_push(master, 'production', 'add Puppetfile', git_environments_path)
+
+step 'Install and configure squid proxy'
+on(master, install_squid)
+
+step 'turn off the firewall'
+on(master, puppet("apply -e 'service {'iptables' : ensure => stopped}'"))
+
+step 'start squid proxy'
+on(master, puppet("apply -e 'service {'squid' : ensure => running}'"))
+
+#Tests
+step 'Deploy "production" Environment via r10k'
+on(master, "#{r10k_fqp} deploy environment -p")
+
+step 'Read the squid logs'
+on(master, "cat #{squid_log}") do |result|
+  assert_match(squid_log_regex, result.stdout, 'Proxy logs did not indicate use of the proxy.')
+end
+
+agents.each do |agent|
+  step "Run Puppet Agent"
+  on(agent, puppet('agent', '--test', '--environment production'), :acceptable_exit_codes => [0,2]) do |result|
+    assert_no_match(/Error:/, result.stderr, 'Unexpected error was detected!')
+    assert_match(notify_message_regex, result.stdout, 'Expected message not found!')
+  end
+end

data/lib/r10k/errors.rb

@@ -39,5 +39,23 @@ module R10K
 
       @options = options
     end
+
+    protected
+
+    def structure_exception(name, exc)
+      struct = []
+      struct << "#{name}:"
+      if exc.respond_to?(:format)
+        struct << indent(exc.format)
+      else
+        struct << indent(exc.message)
+      end
+      struct.join("\n")
+    end
+
+    def indent(str, level = 4)
+      prefix = ' ' * level
+      str.gsub(/^/, prefix)
+    end
   end
 end

data/lib/r10k/forge/module_release.rb

@@ -95,11 +95,11 @@ module R10K
         file_lists = PuppetForge::Unpacker.unpack(download_path.to_s, target_dir.to_s, unpack_path.to_s)
         logger.debug2 "Valid files unpacked: #{file_lists[:valid]}"
         if !file_lists[:invalid].empty?
-          logger.warn "These files existed in the module's tar file, but are invalid filetypes and were not " +
-                      "unpacked: #{file_lists[:invalid]}"
+          logger.debug1 "These files existed in the module's tar file, but are invalid filetypes and were not " +
+                        "unpacked: #{file_lists[:invalid]}"
         end
         if !file_lists[:symlinks].empty?
-          raise R10K::Error, "Symlinks are unsupported and were not unpacked from the module tarball. " + 
+          raise R10K::Error, "Symlinks are unsupported and were not unpacked from the module tarball. " +
                              "#{@forge_release.slug} contained these ignored symlinks: #{file_lists[:symlinks]}"
         end
       end

data/lib/r10k/git.rb

@@ -1,3 +1,4 @@
+require 'uri'
 require 'r10k/features'
 require 'r10k/errors'
 require 'r10k/settings'
@@ -133,11 +134,63 @@ module R10K
     extend R10K::Settings::Mixin::ClassMethods
 
     def_setting_attr :private_key
+    def_setting_attr :proxy
     def_setting_attr :username
     def_setting_attr :repositories, {}
 
     def self.get_repo_settings(remote)
       self.settings[:repositories].find {|r| r[:remote] == remote }
     end
+
+    def self.get_proxy_for_remote(remote)
+      # We only support proxy for HTTP(S) transport
+      return nil unless remote =~ /^http(s)?/i
+
+      repo_settings = self.get_repo_settings(remote)
+
+      if repo_settings && repo_settings.has_key?(:proxy)
+        proxy = repo_settings[:proxy] unless repo_settings[:proxy].nil? || repo_settings[:proxy].empty?
+      else
+        proxy = self.settings[:proxy]
+      end
+
+      R10K::Git.log_proxy_for_remote(proxy, remote) if proxy
+
+      proxy
+    end
+
+    def self.log_proxy_for_remote(proxy, remote)
+      # Sanitize passwords out of the proxy URI for loggging.
+      proxy_uri = URI.parse(proxy)
+      proxy_str = "#{proxy_uri.scheme}://"
+      proxy_str << "#{proxy_uri.userinfo.gsub(/:(.*)$/, ':<FILTERED>')}@" if proxy_uri.userinfo
+      proxy_str << "#{proxy_uri.host}:#{proxy_uri.port}"
+
+      logger.debug { "Using HTTP proxy '#{proxy_str}' for '#{remote}'." }
+
+      nil
+    end
+
+    # Execute block with given proxy configured in ENV
+    def self.with_proxy(new_proxy)
+      unless new_proxy.nil?
+        old_proxy = Hash[
+          ['HTTPS_PROXY', 'HTTP_PROXY', 'https_proxy', 'http_proxy'].collect do |var|
+            old_value = ENV[var]
+            ENV[var] = new_proxy
+
+            [var, old_value]
+          end
+        ]
+      end
+
+      begin
+        yield
+      ensure
+        ENV.update(old_proxy) if old_proxy
+      end
+
+      nil
+    end
   end
 end

data/lib/r10k/git/rugged/bare_repository.rb

@@ -32,14 +32,15 @@ class R10K::Git::Rugged::BareRepository < R10K::Git::Rugged::BaseRepository
   # @return [void]
   def clone(remote)
     logger.debug1 { "Cloning '#{remote}' into #{@path}" }
-    @_rugged_repo = ::Rugged::Repository.init_at(@path.to_s, true)
-    with_repo do |repo|
+
+    @_rugged_repo = ::Rugged::Repository.init_at(@path.to_s, true).tap do |repo|
       config = repo.config
       config['remote.origin.url']    = remote
       config['remote.origin.fetch']  = '+refs/*:refs/*'
       config['remote.origin.mirror'] = 'true'
-      fetch
     end
+
+    fetch('origin')
   rescue Rugged::SshError, Rugged::NetworkError => e
     raise R10K::Git::GitError.new(e.message, :git_dir => git_dir, :backtrace => e.backtrace)
   end
@@ -47,13 +48,21 @@ class R10K::Git::Rugged::BareRepository < R10K::Git::Rugged::BaseRepository
   # Fetch refs and objects from the origin remote
   #
   # @return [void]
-  def fetch
+  def fetch(remote_name='origin')
     backup_branches = wipe_branches
-    logger.debug1 { "Fetching remote 'origin' at #{@path}" }
+    logger.debug1 { "Fetching remote '#{remote_name}' at #{@path}" }
     options = {:credentials => credentials}
     refspecs = ['+refs/*:refs/*']
-    results = with_repo { |repo| repo.fetch('origin', refspecs, options) }
-    report_transfer(results, 'origin')
+
+    remote = remotes[remote_name]
+    proxy = R10K::Git.get_proxy_for_remote(remote)
+    results = nil
+
+    R10K::Git.with_proxy(proxy) do
+      results = with_repo { |repo| repo.fetch(remote_name, refspecs, options) }
+    end
+
+    report_transfer(results, remote_name)
   rescue Rugged::SshError, Rugged::NetworkError => e
     restore_branches(backup_branches)
     raise R10K::Git::GitError.new(e.message, :git_dir => git_dir, :backtrace => e.backtrace)

data/lib/r10k/git/rugged/base_repository.rb

@@ -45,10 +45,24 @@ class R10K::Git::Rugged::BaseRepository
     end
   end
 
+  def remotes
+    remotes_hash = {}
+
+    if @_rugged_repo
+      @_rugged_repo.remotes.each do |remote|
+        remotes_hash[remote.name] = remote.url
+      end
+    end
+
+    remotes_hash
+  end
+
   private
 
-  def with_repo
-    yield @_rugged_repo if @_rugged_repo
+  def with_repo(opts={})
+    if @_rugged_repo
+      yield @_rugged_repo
+    end
   ensure
     @_rugged_repo.close if @_rugged_repo
   end

data/lib/r10k/git/rugged/credentials.rb

@@ -12,9 +12,18 @@ class R10K::Git::Rugged::Credentials
   # @param repository [R10K::Git::Rugged::BaseRepository]
   def initialize(repository)
     @repository = repository
+    @called = 0
   end
 
   def call(url, username_from_url, allowed_types)
+    @called += 1
+
+    # Break out of infinite HTTP auth retry loop introduced in libgit2/rugged 0.24.0, libssh
+    # auth seems to already abort after ~50 attempts.
+    if @called > 50
+      raise R10K::Git::GitError.new("Authentication failed for Git remote #{url.inspect}.")
+    end
+
     if allowed_types.include?(:ssh_key)
       get_ssh_key_credentials(url, username_from_url)
     elsif allowed_types.include?(:plaintext)