r10k 3.9.3 → 3.12.1

data/lib/r10k/environment/with_modules.rb

@@ -23,6 +23,7 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   def initialize(name, basedir, dirname, options = {})
     super
 
+    @all_modules = nil
     @managed_content = {}
     @modules = []
     @moduledir = case options[:moduledir]
@@ -43,10 +44,12 @@ class R10K::Environment::WithModules < R10K::Environment::Base
   #     - The r10k environment object
   #     - A Puppetfile in the environment's content
   def modules
-    return @modules if puppetfile.nil?
+    if @all_modules.nil?
+      puppetfile_modules = super()
+      @all_modules = @modules + puppetfile_modules
+    end
 
-    puppetfile.load unless puppetfile.loaded?
-    @modules + puppetfile.modules
+    @all_modules
   end
 
   def module_conflicts?(mod_b)
@@ -126,13 +129,6 @@ class R10K::Environment::WithModules < R10K::Environment::Base
 
   include R10K::Util::Purgeable
 
-  # Returns an array of the full paths that can be purged.
-  # @note This implements a required method for the Purgeable mixin
-  # @return [Array<String>]
-  def managed_directories
-    [@full_path]
-  end
-
   # Returns an array of the full paths of filenames that should exist. Files
   # inside managed_directories that are not listed in desired_contents will
   # be purged.

data/lib/r10k/git/cache.rb

@@ -111,6 +111,6 @@ class R10K::Git::Cache
 
   # Reformat the remote name into something that can be used as a directory
   def sanitized_dirname
-    @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-')
+    @sanitized_dirname ||= @remote.gsub(/(\w+:\/\/)(.*)(@)/, '\1').gsub(/[^@\w\.-]/, '-')
   end
 end

data/lib/r10k/git/rugged/credentials.rb

@@ -1,6 +1,10 @@
 require 'r10k/git/rugged'
 require 'r10k/git/errors'
 require 'r10k/logging'
+require 'json'
+require 'jwt'
+require 'net/http'
+require 'openssl'
 
 # Generate credentials for secured remote connections.
 #
@@ -62,15 +66,29 @@ class R10K::Git::Rugged::Credentials
 
   def get_plaintext_credentials(url, username_from_url)
     per_repo_oauth_token = nil
+    per_repo_github_app_id = nil
+    per_repo_github_app_key = nil
+    per_repo_github_app_ttl = nil
+
     if per_repo_settings = R10K::Git.get_repo_settings(url)
       per_repo_oauth_token = per_repo_settings[:oauth_token]
+      per_repo_github_app_id = per_repo_settings[:github_app_id]
+      per_repo_github_app_key = per_repo_settings[:github_app_key]
+      per_repo_github_app_ttl = per_repo_settings[:github_app_ttl]
     end
 
+    app_id = per_repo_github_app_id || R10K::Git.settings[:github_app_id]
+    app_key = per_repo_github_app_key || R10K::Git.settings[:github_app_key]
+    app_ttl = per_repo_github_app_ttl || R10K::Git.settings[:github_app_ttl]
+
     if token_path = per_repo_oauth_token || R10K::Git.settings[:oauth_token]
       @oauth_token ||= extract_token(token_path, url)
 
       user = 'x-oauth-token'
       password = @oauth_token
+    elsif app_id && app_key && app_ttl
+      user = 'x-access-token'
+      password = github_app_token(app_id, app_key, app_ttl)
     else
       user = get_git_username(url, username_from_url)
       password = URI.parse(url).password || ''
@@ -125,4 +143,63 @@ class R10K::Git::Rugged::Credentials
 
     user
   end
+
+  def github_app_token(app_id, private_key, ttl)
+    raise R10K::Git::GitError, _('Github App id contains invalid characters.') unless app_id =~ /^\d+$/
+    raise R10K::Git::GitError, _('Github App token ttl contains invalid characters.') unless ttl =~ /^\d+$/
+    raise R10K::Git::GitError, _('Github App key is missing or unreadable') unless File.readable?(private_key)
+
+    begin
+      ssl_key = OpenSSL::PKey::RSA.new(File.read(private_key).strip)
+      unless ssl_key.private?
+        raise R10K::Git::GitError, _('Github App key is not a valid SSL private key')
+      end
+    rescue OpenSSL::PKey::RSAError
+      raise R10K::Git::GitError, _('Github App key is not a valid SSL key')
+    end
+
+    logger.debug2 _("Using Github App id %{app_id} with SSL key from %{key_path}") % { key_path: private_key, app_id: app_id }
+
+    jwt_issue_time = Time.now.to_i - 60
+    jwt_exp_time = (jwt_issue_time + 60) + ttl.to_i
+    payload = { iat: jwt_issue_time, exp: jwt_exp_time, iss: app_id }
+    jwt = JWT.encode(payload, ssl_key, "RS256")
+
+    get = URI.parse("https://api.github.com/app/installations")
+    get_request = Net::HTTP::Get.new(get)
+    get_request["Authorization"] = "Bearer #{jwt}"
+    get_request["Accept"] = "application/vnd.github.v3+json"
+    get_req_options = { use_ssl: get.scheme == "https", }
+    get_response = Net::HTTP.start(get.hostname, get.port, get_req_options) do |http|
+      http.request(get_request)
+    end
+
+    unless (get_response.class < Net::HTTPSuccess)
+      logger.debug2 _("Unexpected response code: #{get_response.code}\nResponse body: #{get_response.body}")
+      raise R10K::Git::GitError, _("Error using private key to get Github App access token from url")
+    end
+
+    access_tokens_url = JSON.parse(get_response.body)[0]['access_tokens_url']
+
+    post = URI.parse(access_tokens_url)
+    post_request = Net::HTTP::Post.new(post)
+    post_request["Authorization"] = "Bearer #{jwt}"
+    post_request["Accept"] = "application/vnd.github.v3+json"
+    post_req_options = { use_ssl: post.scheme == "https", }
+    post_response = Net::HTTP.start(post.hostname, post.port, post_req_options) do |http|
+      http.request(post_request)
+    end
+
+    unless (post_response.class < Net::HTTPSuccess)
+      logger.debug2 _("Unexpected response code: #{post_response.code}\nResponse body: #{post_response.body}")
+      raise R10K::Git::GitError, _("Error using private key to generate access token from #{access_token_url}")
+    end
+
+    token = JSON.parse(post_response.body)['token']
+
+    raise R10K::Git::GitError, _("Github App token contains invalid characters.") unless valid_token?(token)
+
+    logger.debug2 _("Github App token generated, expires at: %{expire}") % {expire: JSON.parse(post_response.body)['expires_at']}
+    token
+  end
 end

data/lib/r10k/git/stateful_repository.rb

@@ -35,6 +35,7 @@ class R10K::Git::StatefulRepository
     @cache.resolve(ref)
   end
 
+  # Returns true if the sync actually updated the repo, false otherwise
   def sync(ref, force=true)
     @cache.sync if sync_cache?(ref)
 
@@ -46,6 +47,7 @@ class R10K::Git::StatefulRepository
 
     workdir_status = status(ref)
 
+    updated = true
     case workdir_status
     when :absent
       logger.debug(_("Cloning %{repo_path} and checking out %{ref}") % {repo_path: @repo.path, ref: ref })
@@ -64,15 +66,20 @@ class R10K::Git::StatefulRepository
         @repo.checkout(sha, {:force => force})
       else
         logger.warn(_("Skipping %{repo_path} due to local modifications") % {repo_path: @repo.path})
+        updated = false
       end
     else
       logger.debug(_("%{repo_path} is already at Git ref %{ref}") % {repo_path: @repo.path, ref: ref })
+      updated = false
     end
+    updated
   end
 
   def status(ref)
     if !@repo.exist?
       :absent
+    elsif !@cache.exist?
+      :mismatched
     elsif !@repo.git_dir.exist?
       :mismatched
     elsif !@repo.git_dir.directory?
@@ -93,6 +100,7 @@ class R10K::Git::StatefulRepository
   # @api private
   def sync_cache?(ref)
     return true if !@cache.exist?
+    return true if ref == 'HEAD'
     return true if !([:commit, :tag].include? @cache.ref_type(ref))
     return false
   end

data/lib/r10k/git.rb

@@ -135,6 +135,9 @@ module R10K
 
     def_setting_attr :private_key
     def_setting_attr :oauth_token
+    def_setting_attr :github_app_id
+    def_setting_attr :github_app_key
+    def_setting_attr :github_app_ttl
     def_setting_attr :proxy
     def_setting_attr :username
     def_setting_attr :repositories, {}

data/lib/r10k/initializers.rb

@@ -56,6 +56,9 @@ module R10K
         with_setting(:proxy) { |value| R10K::Git.settings[:proxy] = value }
         with_setting(:repositories) { |value| R10K::Git.settings[:repositories] = value }
         with_setting(:oauth_token) { |value| R10K::Git.settings[:oauth_token] = value }
+        with_setting(:github_app_id) { |value| R10K::Git.settings[:github_app_id] = value }
+        with_setting(:github_app_key) { |value| R10K::Git.settings[:github_app_key] = value }
+        with_setting(:github_app_ttl) { |value| R10K::Git.settings[:github_app_ttl] = value }
       end
     end
 
@@ -63,6 +66,7 @@ module R10K
       def call
         with_setting(:baseurl) { |value| PuppetForge.host = value }
         with_setting(:proxy) { |value| PuppetForge::Connection.proxy = value }
+        with_setting(:authorization_token) { |value| PuppetForge::Connection.authorization = value }
       end
     end
   end

data/lib/r10k/module/base.rb

@@ -1,9 +1,12 @@
 require 'r10k/module'
+require 'r10k/logging'
 require 'puppet_forge'
 
 # This class defines a common interface for module implementations.
 class R10K::Module::Base
 
+  include R10K::Logging
+
   # @!attribute [r] title
   #   @return [String] The forward slash separated owner and name of the module
   attr_reader :title
@@ -35,6 +38,10 @@ class R10K::Module::Base
   #   @return [String] Where the module was sourced from. E.g., "Puppetfile"
   attr_accessor :origin
 
+  # @!attribute [rw] spec_deletable
+  #   @return [Boolean] set this to true if the spec dir can be safely removed, ie in the moduledir
+  attr_accessor :spec_deletable
+
   # There's been some churn over `author` vs `owner` and `full_name` over
   # `title`, so in the short run it's easier to support both and deprecate one
   # later.
@@ -43,7 +50,7 @@ class R10K::Module::Base
 
   # @param title [String]
   # @param dirname [String]
-  # @param args [Array]
+  # @param args [Hash]
   def initialize(title, dirname, args, environment=nil)
     @title   = PuppetForge::V3.normalize_name(title)
     @dirname = dirname
@@ -52,6 +59,9 @@ class R10K::Module::Base
     @path = Pathname.new(File.join(@dirname, @name))
     @environment = environment
     @overrides = args.delete(:overrides) || {}
+    @spec_deletable = true
+    @exclude_spec = args.delete(:exclude_spec)
+    @exclude_spec = @overrides[:modules].delete(:exclude_spec) if @overrides.dig(:modules, :exclude_spec)
     @origin = 'external' # Expect Puppetfile or R10k::Environment to set this to a specific value
 
     @requested_modules = @overrides.dig(:modules, :requested_modules) || []
@@ -64,8 +74,39 @@ class R10K::Module::Base
     path.to_s
   end
 
+  # Delete the spec dir if @exclude_spec has been set to true and @spec_deletable is also true
+  def maybe_delete_spec_dir
+    if @exclude_spec
+      if @spec_deletable
+        delete_spec_dir
+      else
+        logger.info _("Spec dir for #{@title} will not be deleted because it is not in the moduledir")
+      end
+    end
+  end
+
+  # Actually remove the spec dir
+  def delete_spec_dir
+    spec_path = @path + 'spec'
+    if spec_path.symlink?
+      spec_path = spec_path.realpath
+    end
+    if spec_path.directory?
+      logger.debug2 _("Deleting spec data at #{spec_path}")
+      # Use the secure flag for the #rm_rf method to avoid security issues
+      # involving TOCTTOU(time of check to time of use); more details here:
+      # https://ruby-doc.org/stdlib-2.7.0/libdoc/fileutils/rdoc/FileUtils.html#method-c-rm_rf
+      # Additionally, #rm_rf also has problems in windows with with symlink targets
+      # also being deleted; this should be revisted if Windows becomes higher priority.
+      FileUtils.rm_rf(spec_path, secure: true)
+    else
+      logger.debug2 _("No spec dir detected at #{spec_path}, skipping deletion")
+    end
+  end
+
   # Synchronize this module with the indicated state.
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
     raise NotImplementedError
   end

data/lib/r10k/module/definition.rb

@@ -0,0 +1,64 @@
+require 'r10k/module'
+
+class R10K::Module::Definition < R10K::Module::Base
+
+  attr_reader :version
+
+  def initialize(name, dirname:, args:, implementation:, environment: nil)
+    @original_name  = name
+    @original_args  = args.dup
+    @implementation = implementation
+    @version        = implementation.statically_defined_version(name, args)
+
+    super(name, dirname, args, environment)
+  end
+
+  def to_implementation
+    mod = @implementation.new(@title, @dirname, @original_args, @environment)
+
+    mod.origin = origin
+    mod.spec_deletable = spec_deletable
+
+    mod
+  end
+
+  # syncing is a noop for module definitions
+  # Returns false to inidicate the module was not updated
+  def sync(args = {})
+    logger.debug1(_("Not updating module %{name}, assuming content unchanged") % {name: name})
+    false
+  end
+
+  def status
+    :insync
+  end
+
+  def properties
+    type = nil
+
+    if @args[:type]
+      type = @args[:type]
+    elsif @args[:ref] || @args[:commit] || @args[:branch] || @args[:tag]
+      type = 'git'
+    elsif @args[:svn]
+      # This logic is clear and included for completeness sake, though at
+      # this time module definitions do not support SVN versions.
+      type = 'svn'
+    else
+      type = 'forge'
+    end
+
+    {
+      expected: version,
+      # We can't get the value for `actual` here because that requires the
+      # implementation (and potentially expensive operations by the
+      # implementation). Some consumers will check this value, if it exists
+      # and if not, fall back to the expected version. That is the correct
+      # behavior when assuming modules are unchanged, and why `actual` is set
+      # to `nil` here.
+      actual: nil,
+      type: type
+    }
+  end
+end
+

data/lib/r10k/module/forge.rb

@@ -25,6 +25,10 @@ class R10K::Module::Forge < R10K::Module::Base
     expected_version == :latest || expected_version.nil? || PuppetForge::Util.version_valid?(expected_version)
   end
 
+  def self.statically_defined_version(name, args)
+    args[:version] if args[:version].is_a?(String)
+  end
+
   # @!attribute [r] metadata
   #   @api private
   #   @return [PuppetForge::Metadata]
@@ -35,8 +39,6 @@ class R10K::Module::Forge < R10K::Module::Base
   #   @return [PuppetForge::V3::Module] The Puppet Forge module metadata
   attr_reader :v3_module
 
-  include R10K::Logging
-
   include R10K::Util::Setopts
 
   def initialize(title, dirname, opts, environment=nil)
@@ -58,17 +60,24 @@ class R10K::Module::Forge < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
+    updated = false
     if should_sync?
       case status
       when :absent
         install
+        updated = true
       when :outdated
         upgrade
+        updated = true
       when :mismatched
         reinstall
+        updated = true
       end
+      maybe_delete_spec_dir
     end
+    updated
   end
 
   def properties
@@ -83,7 +92,11 @@ class R10K::Module::Forge < R10K::Module::Base
   def expected_version
     if @expected_version == :latest
       begin
-        @expected_version = @v3_module.current_release.version
+        if @v3_module.current_release
+          @expected_version = @v3_module.current_release.version
+        else
+          raise PuppetForge::ReleaseNotFound, _("The module %{title} does not appear to have any published releases, cannot determine latest version.") % { title: @title }
+        end
       rescue Faraday::ResourceNotFound => e
         raise PuppetForge::ReleaseNotFound, _("The module %{title} does not exist on %{url}.") % {title: @title, url: PuppetForge::V3::Release.conn.url_prefix}, e.backtrace
       end

data/lib/r10k/module/git.rb

@@ -13,6 +13,21 @@ class R10K::Module::Git < R10K::Module::Base
     false
   end
 
+  # Will be called if self.implement? above returns true. Will return
+  # the version info, if version is statically defined in the modules
+  # declaration.
+  def self.statically_defined_version(name, args)
+    if !args[:type] && (args[:ref] || args[:tag] || args[:commit])
+      if args[:ref] && args[:ref].to_s.match(/[0-9a-f]{40}/)
+        args[:ref]
+      else
+        args[:tag] || args[:commit]
+      end
+    elsif args[:type].to_s == 'git' && args[:version] && args[:version].to_s.match(/[0-9a-f]{40}/)
+      args[:version]
+    end
+  end
+
   # @!attribute [r] repo
   #   @api private
   #   @return [R10K::Git::StatefulRepository]
@@ -83,9 +98,16 @@ class R10K::Module::Git < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
     force = opts[:force] || @force
-    @repo.sync(version, force) if should_sync?
+    if should_sync?
+      updated = @repo.sync(version, force)
+    else
+      updated = false
+    end
+    maybe_delete_spec_dir
+    updated
   end
 
   def status

data/lib/r10k/module/local.rb

@@ -1,5 +1,4 @@
 require 'r10k/module'
-require 'r10k/logging'
 
 # A dummy module type that can be used to "protect" Puppet modules that exist
 # inside of the Puppetfile "moduledir" location. Local modules will not be
@@ -12,10 +11,12 @@ class R10K::Module::Local < R10K::Module::Base
     args.is_a?(Hash) && (args[:local] || args[:type].to_s == 'local')
   end
 
-  include R10K::Logging
+  def self.statically_defined_version(*)
+    "0.0.0"
+  end
 
   def version
-    "0.0.0"
+    self.class.statically_defined_version
   end
 
   def properties
@@ -31,7 +32,9 @@ class R10K::Module::Local < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] false, because local modules are always considered in-sync
   def sync(opts={})
     logger.debug1 _("Module %{title} is a local module, always indicating synced.") % {title: title}
+    false
   end
 end

data/lib/r10k/module/svn.rb

@@ -10,6 +10,10 @@ class R10K::Module::SVN < R10K::Module::Base
     args.has_key?(:svn) || args[:type].to_s == 'svn'
   end
 
+  def self.statically_defined_version(name, args)
+    nil
+  end
+
   # @!attribute [r] expected_revision
   #   @return [String] The SVN revision that the repo should have checked out
   attr_reader :expected_revision
@@ -70,17 +74,24 @@ class R10K::Module::SVN < R10K::Module::Base
   end
 
   # @param [Hash] opts Deprecated
+  # @return [Boolean] true if the module was updated, false otherwise
   def sync(opts={})
+    updated = false
     if should_sync?
       case status
       when :absent
         install
+        updated = true
       when :mismatched
         reinstall
+        updated = true
       when :outdated
         update
+        updated = true
       end
+      maybe_delete_spec_dir
     end
+    updated
   end
 
   def exist?

data/lib/r10k/module.rb

@@ -22,17 +22,35 @@ module R10K::Module
   #
   # @return [Object < R10K::Module] A member of the implementing subclass
   def self.new(name, basedir, args, environment=nil)
+    with_implementation(name, args) do |implementation|
+      implementation.new(name, basedir, args, environment)
+    end
+  end
+
+  # Takes the same signature as Module.new but returns an metadata module
+  def self.from_metadata(name, basedir, args, environment=nil)
+    with_implementation(name, args) do |implementation|
+      R10K::Module::Definition.new(name,
+                                   dirname: basedir,
+                                   args: args,
+                                   implementation: implementation,
+                                   environment: environment)
+    end
+  end
+
+  def self.with_implementation(name, args, &block)
     if implementation = @klasses.find { |klass| klass.implement?(name, args) }
-      obj = implementation.new(name, basedir, args, environment)
-      obj
+      block.call(implementation)
     else
       raise _("Module %{name} with args %{args} doesn't have an implementation. (Are you using the right arguments?)") % {name: name, args: args.inspect}
     end
   end
 
+
   require 'r10k/module/base'
   require 'r10k/module/git'
   require 'r10k/module/svn'
   require 'r10k/module/local'
   require 'r10k/module/forge'
+  require 'r10k/module/definition'
 end

data/lib/r10k/module_loader/puppetfile/dsl.rb

@@ -0,0 +1,42 @@
+module R10K
+  module ModuleLoader
+    class Puppetfile
+      class DSL
+        # A barebones implementation of the Puppetfile DSL
+        #
+        # @api private
+
+        def initialize(librarian, metadata_only: false)
+          @librarian     = librarian
+          @metadata_only = metadata_only
+        end
+
+        def mod(name, args = nil)
+          if args.is_a?(Hash)
+            opts = args
+          else
+            opts = { version: args }
+          end
+
+          if @metadata_only
+            @librarian.add_module_metadata(name, opts)
+          else
+            @librarian.add_module(name, opts)
+          end
+        end
+
+        def forge(location)
+          @librarian.set_forge(location)
+        end
+
+        def moduledir(location)
+          @librarian.set_moduledir(location)
+        end
+
+        def method_missing(method, *args)
+          raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method}
+        end
+      end
+    end
+  end
+end