r10k 3.9.1 → 3.11.0

data/lib/r10k/action/deploy/module.rb

@@ -18,14 +18,18 @@ module R10K
         # @param opts [Hash] A hash of options defined in #allowed_initialized_opts
         #   and managed by the SetOps mixin within the Action::Base class.
         #   Corresponds to the CLI flags and options.
-        # @param argv [CRI::ArgumentList] A list-like collection of the remaining
-        #   arguments to the CLI invocation (after removing flags and options).
+        # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like
+        #   collection of the remaining arguments to the CLI invocation (after
+        #   removing flags and options).
         # @param settings [Hash] A hash of configuration loaded from the relevant
         #   config (r10k.yaml).
-        def initialize(opts, argv, settings)
+        #
+        # @note All arguments will be required in the next major version
+        def initialize(opts, argv, settings = {})
           super
 
           requested_env = @opts[:environment] ? [@opts[:environment].gsub(/\W/, '_')] : []
+          @modified_envs = []
 
           @settings = @settings.merge({
             overrides: {
@@ -34,6 +38,7 @@ module R10K
                 generate_types: @generate_types
               },
               modules: {
+                exclude_spec: settings.dig(:deploy, :exclude_spec),
                 requested_modules: @argv.map.to_a,
                 # force here is used to make it easier to reason about
                 force: !@no_force
@@ -66,6 +71,21 @@ module R10K
 
         def visit_deployment(deployment)
           yield
+        ensure
+          if (postcmd = @settings[:postrun])
+            if @modified_envs.any?
+              if postcmd.grep('$modifiedenvs').any?
+                envs_to_run = @modified_envs.join(' ')
+                logger.debug "Running postrun command for environments #{envs_to_run}."
+                postcmd = postcmd.map { |e| e.gsub('$modifiedenvs', envs_to_run) }
+              end
+              subproc = R10K::Util::Subprocess.new(postcmd)
+              subproc.logger = logger
+              subproc.execute
+            else
+              logger.debug "No environments were modified, not executing postrun command."
+            end
+          end
         end
 
         def visit_source(source)
@@ -82,10 +102,15 @@ module R10K
             environment.deploy
 
             requested_mods = @settings.dig(:overrides, :modules, :requested_modules) || []
-            generate_types = @settings.dig(:overrides, :environments, :generate_types)
-            if generate_types && !((environment.modules.map(&:name) & requested_mods).empty?)
-              logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
-              environment.generate_types!
+            # We actually synced a module in this env
+            if !((environment.modules.map(&:name) & requested_mods).empty?)
+              # Record modified environment for postrun command
+              @modified_envs << environment.dirname
+
+              if generate_types = @settings.dig(:overrides, :environments, :generate_types)
+                logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
+                environment.generate_types!
+              end
             end
           end
         end
@@ -93,12 +118,16 @@ module R10K
         def allowed_initialize_opts
           super.merge(environment: true,
                       cachedir: :self,
+                      'exclude-spec': :self,
                       'no-force': :self,
                       'generate-types': :self,
                       'puppet-path': :self,
                       'puppet-conf': :self,
                       'private-key': :self,
-                      'oauth-token': :self)
+                      'oauth-token': :self,
+                      'github-app-id': :self,
+                      'github-app-key': :self,
+                      'github-app-ttl': :self)
         end
       end
     end

data/lib/r10k/action/runner.rb

@@ -44,10 +44,13 @@ module R10K
 
         overrides = {}
         overrides[:cachedir] = @opts[:cachedir] if @opts.key?(:cachedir)
-        overrides[:deploy] = {} if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types')
-        overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
-        overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] unless @opts[:'puppet-conf'].nil?
-        overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
+        if @opts.key?(:'puppet-path') || @opts.key?(:'generate-types') || @opts.key?(:'exclude-spec') || @opts.key?(:'puppet-conf')
+          overrides[:deploy] = {}
+          overrides[:deploy][:puppet_path] = @opts[:'puppet-path'] if @opts.key?(:'puppet-path')
+          overrides[:deploy][:puppet_conf] = @opts[:'puppet-conf'] if @opts.key?(:'puppet-conf')
+          overrides[:deploy][:generate_types] = @opts[:'generate-types'] if @opts.key?(:'generate-types')
+          overrides[:deploy][:exclude_spec] = @opts[:'exclude-spec'] if @opts.key?(:'exclude-spec')
+        end
 
         with_overrides = config_settings.merge(overrides) do |key, oldval, newval|
           newval = oldval.merge(newval) if oldval.is_a? Hash
@@ -67,15 +70,20 @@ module R10K
         exit(8)
       end
 
+      # Set up authorization from license file if it wasn't
+      # already set via the config
       def setup_authorization
-        begin
-          license = R10K::Util::License.load
+        if PuppetForge::Connection.authorization.nil?
+          begin
+            license = R10K::Util::License.load
 
-          if license.respond_to?(:authorization_token)
-            PuppetForge::Connection.authorization = license.authorization_token
+            if license.respond_to?(:authorization_token)
+              logger.debug "Using token from license to connect to the Forge."
+              PuppetForge::Connection.authorization = license.authorization_token
+            end
+          rescue R10K::Error => e
+            logger.warn e.message
           end
-        rescue R10K::Error => e
-          logger.warn e.message
         end
       end
 
@@ -100,11 +108,26 @@ module R10K
       def add_credential_overrides(overrides)
         sshkey_path = @opts[:'private-key']
         token_path = @opts[:'oauth-token']
+        app_id = @opts[:'github-app-id']
+        app_private_key_path = @opts[:'github-app-key']
+        app_ttl = @opts[:'github-app-ttl']
 
         if sshkey_path && token_path
           raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy."
         end
 
+        if sshkey_path && (app_private_key_path || app_id)
+          raise R10K::Error, "Cannot specify both an SSH key and an SSL key or Github App id to use with this deploy."
+        end
+
+        if token_path && (app_private_key_path || app_id)
+          raise R10K::Error, "Cannot specify both an OAuth token and an SSL key or Github App id to use with this deploy."
+        end
+
+        if app_id && ! app_private_key_path || app_private_key_path && ! app_id
+          raise R10K::Error, "Must specify both id and SSL private key to use Github App for this deploy."
+        end
+
         if sshkey_path
           overrides[:git] ||= {}
           overrides[:git][:private_key] = sshkey_path
@@ -121,6 +144,18 @@ module R10K
               repo[:oauth_token] = token_path
             end
           end
+        elsif app_id
+          overrides[:git] ||= {}
+          overrides[:git][:github_app_id] = app_id
+          overrides[:git][:github_app_key] = app_private_key_path
+          overrides[:git][:github_app_ttl] = app_ttl
+          if repo_settings = overrides[:git][:repositories]
+            repo_settings.each do |repo|
+              repo[:github_app_id] = app_id
+              repo[:github_app_key] = app_private_key_path
+              repo[:github_app_ttl] = app_ttl
+            end
+          end
         end
 
         overrides

data/lib/r10k/cli/deploy.rb

@@ -24,6 +24,7 @@ module R10K::CLI
         option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required
         flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
         flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment'
+        flag nil, :'exclude-spec', 'Exclude the module\'s spec dir from deployment'
         option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd|
           unless File.executable? value
             $stderr.puts "The specified puppet executable #{value} is not executable."
@@ -34,6 +35,9 @@ module R10K::CLI
         option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required
         option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required
         option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-id', 'Github App id. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-key', 'Github App private key. Only valid with rugged provider', argument: :required
+        option nil, :'github-app-ttl', 'Github App token expiration, in seconds. Only valid with rugged provider', default: "120", argument: :optional
 
         run do |opts, args, cmd|
           puts cmd.help(:verbose => opts[:verbose])

data/lib/r10k/git.rb

@@ -135,6 +135,9 @@ module R10K
 
     def_setting_attr :private_key
     def_setting_attr :oauth_token
+    def_setting_attr :github_app_id
+    def_setting_attr :github_app_key
+    def_setting_attr :github_app_ttl
     def_setting_attr :proxy
     def_setting_attr :username
     def_setting_attr :repositories, {}

data/lib/r10k/git/cache.rb

@@ -111,6 +111,6 @@ class R10K::Git::Cache
 
   # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= @remote.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
   end
 end

data/lib/r10k/git/rugged/credentials.rb

@@ -1,6 +1,10 @@
 require 'r10k/git/rugged'
 require 'r10k/git/errors'
 require 'r10k/logging'
+require 'json'
+require 'jwt'
+require 'net/http'
+require 'openssl'
 
 # Generate credentials for secured remote connections.
 #
@@ -62,15 +66,29 @@ class R10K::Git::Rugged::Credentials
 
   def get_plaintext_credentials(url, username_from_url)
     per_repo_oauth_token = nil
+    per_repo_github_app_id = nil
+    per_repo_github_app_key = nil
+    per_repo_github_app_ttl = nil
+
     if per_repo_settings = R10K::Git.get_repo_settings(url)
       per_repo_oauth_token = per_repo_settings[:oauth_token]
+      per_repo_github_app_id = per_repo_settings[:github_app_id]
+      per_repo_github_app_key = per_repo_settings[:github_app_key]
+      per_repo_github_app_ttl = per_repo_settings[:github_app_ttl]
     end
 
+    app_id = per_repo_github_app_id || R10K::Git.settings[:github_app_id]
+    app_key = per_repo_github_app_key || R10K::Git.settings[:github_app_key]
+    app_ttl = per_repo_github_app_ttl || R10K::Git.settings[:github_app_ttl]
+
     if token_path = per_repo_oauth_token || R10K::Git.settings[:oauth_token]
       @oauth_token ||= extract_token(token_path, url)
 
       user = 'x-oauth-token'
       password = @oauth_token
+    elsif app_id && app_key && app_ttl
+      user = 'x-access-token'
+      password = github_app_token(app_id, app_key, app_ttl)
     else
       user = get_git_username(url, username_from_url)
       password = URI.parse(url).password || ''
@@ -125,4 +143,63 @@ class R10K::Git::Rugged::Credentials
 
     user
   end
+
+  def github_app_token(app_id, private_key, ttl)
+    raise R10K::Git::GitError, _('Github App id contains invalid characters.') unless app_id =~ /^\d+$/
+    raise R10K::Git::GitError, _('Github App token ttl contains invalid characters.') unless ttl =~ /^\d+$/
+    raise R10K::Git::GitError, _('Github App key is missing or unreadable') unless File.readable?(private_key)
+
+    begin
+      ssl_key = OpenSSL::PKey::RSA.new(File.read(private_key).strip)
+      unless ssl_key.private?
+        raise R10K::Git::GitError, _('Github App key is not a valid SSL private key')
+      end
+    rescue OpenSSL::PKey::RSAError
+      raise R10K::Git::GitError, _('Github App key is not a valid SSL key')
+    end
+
+    logger.debug2 _("Using Github App id %{app_id} with SSL key from %{key_path}") % { key_path: private_key, app_id: app_id }
+
+    jwt_issue_time = Time.now.to_i - 60
+    jwt_exp_time = (jwt_issue_time + 60) + ttl.to_i
+    payload = { iat: jwt_issue_time, exp: jwt_exp_time, iss: app_id }
+    jwt = JWT.encode(payload, ssl_key, "RS256")
+
+    get = URI.parse("https://api.github.com/app/installations")
+    get_request = Net::HTTP::Get.new(get)
+    get_request["Authorization"] = "Bearer #{jwt}"
+    get_request["Accept"] = "application/vnd.github.v3+json"
+    get_req_options = { use_ssl: get.scheme == "https", }
+    get_response = Net::HTTP.start(get.hostname, get.port, get_req_options) do |http|
+      http.request(get_request)
+    end
+
+    unless (get_response.class < Net::HTTPSuccess)
+      logger.debug2 _("Unexpected response code: #{get_response.code}\nResponse body: #{get_response.body}")
+      raise R10K::Git::GitError, _("Error using private key to get Github App access token from url")
+    end
+
+    access_tokens_url = JSON.parse(get_response.body)[0]['access_tokens_url']
+
+    post = URI.parse(access_tokens_url)
+    post_request = Net::HTTP::Post.new(post)
+    post_request["Authorization"] = "Bearer #{jwt}"
+    post_request["Accept"] = "application/vnd.github.v3+json"
+    post_req_options = { use_ssl: post.scheme == "https", }
+    post_response = Net::HTTP.start(post.hostname, post.port, post_req_options) do |http|
+      http.request(post_request)
+    end
+
+    unless (post_response.class < Net::HTTPSuccess)
+      logger.debug2 _("Unexpected response code: #{post_response.code}\nResponse body: #{post_response.body}")
+      raise R10K::Git::GitError, _("Error using private key to generate access token from #{access_token_url}")
+    end
+
+    token = JSON.parse(post_response.body)['token']
+
+    raise R10K::Git::GitError, _("Github App token contains invalid characters.") unless valid_token?(token)
+
+    logger.debug2 _("Github App token generated, expires at: %{expire}") % {expire: JSON.parse(post_response.body)['expires_at']}
+    token
+  end
 end

data/lib/r10k/git/stateful_repository.rb

@@ -93,6 +93,7 @@ class R10K::Git::StatefulRepository
   # @api private
   def sync_cache?(ref)
     return true if !@cache.exist?
+    return true if ref == 'HEAD'
     return true if !([:commit, :tag].include? @cache.ref_type(ref))
     return false
   end

data/lib/r10k/initializers.rb

@@ -56,6 +56,9 @@ module R10K
         with_setting(:proxy) { |value| R10K::Git.settings[:proxy] = value }
         with_setting(:repositories) { |value| R10K::Git.settings[:repositories] = value }
         with_setting(:oauth_token) { |value| R10K::Git.settings[:oauth_token] = value }
+        with_setting(:github_app_id) { |value| R10K::Git.settings[:github_app_id] = value }
+        with_setting(:github_app_key) { |value| R10K::Git.settings[:github_app_key] = value }
+        with_setting(:github_app_ttl) { |value| R10K::Git.settings[:github_app_ttl] = value }
       end
     end
 
@@ -63,6 +66,13 @@ module R10K
       def call
         with_setting(:baseurl) { |value| PuppetForge.host = value }
         with_setting(:proxy) { |value| PuppetForge::Connection.proxy = value }
+        with_setting(:authorization_token) { |value|
+          if @settings[:baseurl]
+            PuppetForge::Connection.authorization = value
+          else
+            raise R10K::Error, "Cannot specify a Forge authorization token without configuring a custom baseurl."
+          end
+        }
       end
     end
   end

data/lib/r10k/module/base.rb

@@ -35,6 +35,10 @@ class R10K::Module::Base
   #   @return [String] Where the module was sourced from. E.g., "Puppetfile"
   attr_accessor :origin
 
+  # @!attribute [rw] spec_deletable
+  #   @return [Boolean] set this to true if the spec dir can be safely removed, ie in the moduledir
+  attr_accessor :spec_deletable
+
   # There's been some churn over `author` vs `owner` and `full_name` over
   # `title`, so in the short run it's easier to support both and deprecate one
   # later.
@@ -52,6 +56,9 @@ class R10K::Module::Base
     @path = Pathname.new(File.join(@dirname, @name))
     @environment = environment
     @overrides = args.delete(:overrides) || {}
+    @spec_deletable = true
+    @exclude_spec = args.delete(:exclude_spec)
+    @exclude_spec = @overrides[:modules].delete(:exclude_spec) if @overrides.dig(:modules, :exclude_spec)
     @origin = 'external' # Expect Puppetfile or R10k::Environment to set this to a specific value
 
     @requested_modules = @overrides.dig(:modules, :requested_modules) || []
@@ -64,6 +71,36 @@ class R10K::Module::Base
     path.to_s
   end
 
+  # Delete the spec dir if @exclude_spec has been set to true and @spec_deletable is also true
+  def maybe_delete_spec_dir
+    if @exclude_spec
+      if @spec_deletable
+        delete_spec_dir
+      else
+        logger.info _("Spec dir for #{@title} will not be deleted because it is not in the moduledir")
+      end
+    end
+  end
+
+  # Actually remove the spec dir
+  def delete_spec_dir
+    spec_path = @path + 'spec'
+    if spec_path.symlink?
+      spec_path = spec_path.realpath
+    end
+    if spec_path.directory?
+      logger.debug2 _("Deleting spec data at #{spec_path}")
+      # Use the secure flag for the #rm_rf method to avoid security issues
+      # involving TOCTTOU(time of check to time of use); more details here:
+      # https://ruby-doc.org/stdlib-2.7.0/libdoc/fileutils/rdoc/FileUtils.html#method-c-rm_rf
+      # Additionally, #rm_rf also has problems in windows with with symlink targets
+      # also being deleted; this should be revisted if Windows becomes higher priority.
+      FileUtils.rm_rf(spec_path, secure: true)
+    else
+      logger.debug2 _("No spec dir detected at #{spec_path}, skipping deletion")
+    end
+  end
+
   # Synchronize this module with the indicated state.
   # @param [Hash] opts Deprecated
   def sync(opts={})

data/lib/r10k/module/forge.rb

@@ -50,7 +50,7 @@ class R10K::Module::Forge < R10K::Module::Base
       :version => :expected_version,
       :source  => ::R10K::Util::Setopts::Ignore,
       :type    => ::R10K::Util::Setopts::Ignore,
-    })
+    }, :raise_on_unhandled => false)
 
     @expected_version ||= current_version || :latest
 
@@ -68,6 +68,7 @@ class R10K::Module::Forge < R10K::Module::Base
       when :mismatched
         reinstall
       end
+      maybe_delete_spec_dir
     end
   end
 
@@ -83,7 +84,11 @@ class R10K::Module::Forge < R10K::Module::Base
   def expected_version
     if @expected_version == :latest
       begin
-        @expected_version = @v3_module.current_release.version
+        if @v3_module.current_release
+          @expected_version = @v3_module.current_release.version
+        else
+          raise PuppetForge::ReleaseNotFound, _("The module %{title} does not appear to have any published releases, cannot determine latest version.") % { title: @title }
+        end
       rescue Faraday::ResourceNotFound => e
         raise PuppetForge::ReleaseNotFound, _("The module %{title} does not exist on %{url}.") % {title: @title, url: PuppetForge::V3::Release.conn.url_prefix}, e.backtrace
       end

data/lib/r10k/module/git.rb

@@ -52,7 +52,7 @@ class R10K::Module::Git < R10K::Module::Base
       :git                     => :remote,
       :default_branch          => :default_ref,
       :default_branch_override => :default_override_ref,
-    })
+    }, :raise_on_unhandled => false)
 
     force = @overrides.dig(:modules, :force)
     @force = force == false ? false : true
@@ -86,6 +86,7 @@ class R10K::Module::Git < R10K::Module::Base
   def sync(opts={})
     force = opts[:force] || @force
     @repo.sync(version, force) if should_sync?
+    maybe_delete_spec_dir
   end
 
   def status