akerl-r10k 3.14.2.1

data/lib/r10k/git/rugged/credentials.rb

@@ -0,0 +1,205 @@
+require 'r10k/git/rugged'
+require 'r10k/git/errors'
+require 'r10k/logging'
+require 'json'
+require 'jwt'
+require 'net/http'
+require 'openssl'
+
+# Generate credentials for secured remote connections.
+#
+# @api private
+class R10K::Git::Rugged::Credentials
+
+  include R10K::Logging
+
+  # @param repository [R10K::Git::Rugged::BaseRepository]
+  def initialize(repository)
+    @repository = repository
+    @called = 0
+  end
+
+  def call(url, username_from_url, allowed_types)
+    @called += 1
+
+    # Break out of infinite HTTP auth retry loop introduced in libgit2/rugged 0.24.0, libssh
+    # auth seems to already abort after ~50 attempts.
+    if @called > 50
+      raise R10K::Git::GitError.new(_("Authentication failed for Git remote %{url}.") % {url: url.inspect} )
+    end
+
+    if allowed_types.include?(:ssh_key)
+      get_ssh_key_credentials(url, username_from_url)
+    elsif allowed_types.include?(:plaintext)
+      get_plaintext_credentials(url, username_from_url)
+    else
+      get_default_credentials(url, username_from_url)
+    end
+  end
+
+  def get_ssh_key_credentials(url, username_from_url)
+    user = get_git_username(url, username_from_url)
+
+    per_repo_private_key = nil
+    if per_repo_settings = R10K::Git.get_repo_settings(url)
+      per_repo_private_key = per_repo_settings[:private_key]
+    end
+
+    global_private_key = R10K::Git.settings[:private_key]
+
+    if per_repo_private_key
+      private_key = per_repo_private_key
+      logger.debug2 _("Using per-repository private key %{key} for URL %{url}") % {key: private_key, url: url.inspect}
+    elsif global_private_key
+      private_key = global_private_key
+      logger.debug2 _("URL %{url} has no per-repository private key using '%{key}'." ) % {key: private_key, url: url.inspect}
+    else
+      raise R10K::Git::GitError.new(_("Git remote %{url} uses the SSH protocol but no private key was given") % {url: url.inspect}, :git_dir => @repository.path.to_s)
+    end
+
+    if !File.readable?(private_key)
+      raise R10K::Git::GitError.new(_("Unable to use SSH key auth for %{url}: private key %{private_key} is missing or unreadable") % {url: url.inspect, private_key: private_key.inspect}, :git_dir => @repository.path.to_s)
+    end
+
+    Rugged::Credentials::SshKey.new(:username => user, :privatekey => private_key)
+  end
+
+  def get_plaintext_credentials(url, username_from_url)
+    per_repo_oauth_token = nil
+    per_repo_github_app_id = nil
+    per_repo_github_app_key = nil
+    per_repo_github_app_ttl = nil
+
+    if per_repo_settings = R10K::Git.get_repo_settings(url)
+      per_repo_oauth_token = per_repo_settings[:oauth_token]
+      per_repo_github_app_id = per_repo_settings[:github_app_id]
+      per_repo_github_app_key = per_repo_settings[:github_app_key]
+      per_repo_github_app_ttl = per_repo_settings[:github_app_ttl]
+    end
+
+    app_id = per_repo_github_app_id || R10K::Git.settings[:github_app_id]
+    app_key = per_repo_github_app_key || R10K::Git.settings[:github_app_key]
+    app_ttl = per_repo_github_app_ttl || R10K::Git.settings[:github_app_ttl]
+
+    if token_path = per_repo_oauth_token || R10K::Git.settings[:oauth_token]
+      @oauth_token ||= extract_token(token_path, url)
+
+      user = 'x-oauth-token'
+      password = @oauth_token
+    elsif app_id && app_key && app_ttl
+      user = 'x-access-token'
+      password = github_app_token(app_id, app_key, app_ttl)
+    else
+      user = get_git_username(url, username_from_url)
+      password = URI.parse(url).password || ''
+    end
+    Rugged::Credentials::UserPassword.new(username: user, password: password)
+  end
+
+  def extract_token(token_path, url)
+    if token_path == '-'
+      token = $stdin.read.strip
+      logger.debug2 _("Using OAuth token from stdin for URL %{url}") % { url: url }
+    elsif File.readable?(token_path)
+      token = File.read(token_path).strip
+      logger.debug2 _("Using OAuth token from %{token_path} for URL %{url}") % { token_path: token_path, url: url }
+    else
+      raise R10K::Git::GitError, _("%{path} is missing or unreadable, cannot load OAuth token") % { path: token_path }
+    end
+
+    unless valid_token?(token)
+      raise R10K::Git::GitError, _("Supplied OAuth token contains invalid characters.")
+    end
+
+    token
+  end
+
+  # This regex is the only real requirement for OAuth token format,
+  # per https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/
+  # Bitbucket's tokens also can include an underscore, so that is added here.
+  def valid_token?(token)
+    return token =~ /^[\w\-\.~_\+\/]+$/
+  end
+
+  def get_default_credentials(url, username_from_url)
+    Rugged::Credentials::Default.new
+  end
+
+  def get_git_username(url, username_from_url)
+    git_user = R10K::Git.settings[:username]
+
+    user = nil
+
+    if !username_from_url.nil? && !username_from_url.empty?
+      user = username_from_url
+      logger.debug2 _("URL %{url} includes the username %{username}, using that user for authentication.") % {url: url.inspect, username: username_from_url}
+    elsif git_user
+      user = git_user
+      logger.debug2 _("URL %{url} did not specify a user, using %{user} from configuration") % {url: url.inspect, user: user.inspect}
+    else
+      user = Etc.getlogin
+      logger.debug2 _("URL %{url} did not specify a user, using current user %{user}") % {url: url.inspect, user: user.inspect}
+    end
+
+    user
+  end
+
+  def github_app_token(app_id, private_key, ttl)
+    raise R10K::Git::GitError, _('Github App id contains invalid characters.') unless app_id =~ /^\d+$/
+    raise R10K::Git::GitError, _('Github App token ttl contains invalid characters.') unless ttl =~ /^\d+$/
+    raise R10K::Git::GitError, _('Github App key is missing or unreadable') unless File.readable?(private_key)
+
+    begin
+      ssl_key = OpenSSL::PKey::RSA.new(File.read(private_key).strip)
+      unless ssl_key.private?
+        raise R10K::Git::GitError, _('Github App key is not a valid SSL private key')
+      end
+    rescue OpenSSL::PKey::RSAError
+      raise R10K::Git::GitError, _('Github App key is not a valid SSL key')
+    end
+
+    logger.debug2 _("Using Github App id %{app_id} with SSL key from %{key_path}") % { key_path: private_key, app_id: app_id }
+
+    jwt_issue_time = Time.now.to_i - 60
+    jwt_exp_time = (jwt_issue_time + 60) + ttl.to_i
+    payload = { iat: jwt_issue_time, exp: jwt_exp_time, iss: app_id }
+    jwt = JWT.encode(payload, ssl_key, "RS256")
+
+    get = URI.parse("https://api.github.com/app/installations")
+    get_request = Net::HTTP::Get.new(get)
+    get_request["Authorization"] = "Bearer #{jwt}"
+    get_request["Accept"] = "application/vnd.github.v3+json"
+    get_req_options = { use_ssl: get.scheme == "https", }
+    get_response = Net::HTTP.start(get.hostname, get.port, get_req_options) do |http|
+      http.request(get_request)
+    end
+
+    unless (get_response.class < Net::HTTPSuccess)
+      logger.debug2 _("Unexpected response code: #{get_response.code}\nResponse body: #{get_response.body}")
+      raise R10K::Git::GitError, _("Error using private key to get Github App access token from url")
+    end
+
+    access_tokens_url = JSON.parse(get_response.body)[0]['access_tokens_url']
+
+    post = URI.parse(access_tokens_url)
+    post_request = Net::HTTP::Post.new(post)
+    post_request["Authorization"] = "Bearer #{jwt}"
+    post_request["Accept"] = "application/vnd.github.v3+json"
+    post_req_options = { use_ssl: post.scheme == "https", }
+    post_response = Net::HTTP.start(post.hostname, post.port, post_req_options) do |http|
+      http.request(post_request)
+    end
+
+    unless (post_response.class < Net::HTTPSuccess)
+      logger.debug2 _("Unexpected response code: #{post_response.code}\nResponse body: #{post_response.body}")
+      raise R10K::Git::GitError, _("Error using private key to generate access token from #{access_token_url}")
+    end
+
+    token = JSON.parse(post_response.body)['token']
+
+    raise R10K::Git::GitError, _("Github App token contains invalid characters.") unless valid_token?(token)
+
+    logger.debug2 _("Github App token generated, expires at: %{expire}") % {expire: JSON.parse(post_response.body)['expires_at']}
+    token
+  end
+end

data/lib/r10k/git/rugged/thin_repository.rb

@@ -0,0 +1,96 @@
+require 'r10k/git'
+require 'r10k/git/rugged/working_repository'
+require 'r10k/git/rugged/cache'
+
+class R10K::Git::Rugged::ThinRepository < R10K::Git::Rugged::WorkingRepository
+  def initialize(basedir, dirname, cache_repo)
+    @cache_repo = cache_repo
+
+    super(basedir, dirname)
+  end
+
+  # Clone this git repository
+  #
+  # @param remote [String] The Git remote to clone
+  # @param opts [Hash]
+  #
+  # @options opts [String] :ref The git ref to check out on clone
+  #
+  # @return [void]
+  def clone(remote, opts = {})
+    logger.debug1 { "Cloning '#{remote}' into #{@path}" }
+    @cache_repo.sync
+
+    cache_objects_dir = @cache_repo.objects_dir.to_s
+
+    # {Rugged::Repository.clone_at} doesn't support :alternates, which
+    # completely breaks how thin repositories need to work. To circumvent
+    # this we manually create a Git repository, set up git remotes, and
+    # update 'objects/info/alternates' with the path. We don't actually
+    # fetch any objects because we don't need them, and we don't actually
+    # use any refs in this repository so we skip all those steps.
+    ::Rugged::Repository.init_at(@path.to_s, false)
+    @_rugged_repo = ::Rugged::Repository.new(@path.to_s, :alternates => [cache_objects_dir])
+    alternates << cache_objects_dir
+
+    with_repo do |repo|
+      config = repo.config
+      config['remote.origin.url']    = remote
+      config['remote.origin.fetch']  = '+refs/heads/*:refs/remotes/origin/*'
+      config['remote.cache.url']     = @cache_repo.git_dir.to_s
+      config['remote.cache.fetch']   = '+refs/heads/*:refs/remotes/cache/*'
+    end
+
+    checkout(opts.fetch(:ref, 'HEAD'))
+  end
+
+  def checkout(ref, opts = {})
+    super(@cache_repo.resolve(ref), opts)
+  end
+
+  # Fetch refs and objects from one of the Git remotes
+  #
+  # @param remote [String] The remote to fetch, defaults to 'cache'
+  # @return [void]
+  def fetch(remote = 'cache')
+    super(remote)
+  end
+
+  # @return [String] The cache remote URL
+  def cache
+    with_repo { |repo| repo.config['remote.cache.url'] }
+  end
+
+  def tracked_paths(ref="HEAD")
+    with_repo do |repo|
+      commit = repo.rev_parse(ref)
+
+      unless commit && commit.tree
+        raise R10K::Error.new("Unable to resolve '#{ref}' to a valid commit in repo #{@path}")
+      end
+
+      commit.tree.walk(:postorder).collect do |root, entry|
+        root.empty? ? entry[:name] : File.join(root, entry[:name])
+      end
+    end
+  end
+
+  def stage_files(files=['.'])
+    with_repo do |repo|
+      index = repo.index
+      files.each { |p| index.add( :path => p ) }
+    end
+  end
+
+  private
+
+  # Override the parent class repo setup so that we can make sure the alternates file is up to date
+  # before we create the Rugged::Repository object, which reads from the alternates file.
+  def setup_rugged_repo
+    entry_added = alternates.add?(@cache_repo.objects_dir.to_s)
+    if entry_added
+      logger.debug2 { _("Updated repo %{path} to include alternate object db path %{objects_dir}") % {path: @path, objects_dir: @cache_repo.objects_dir} }
+    end
+    super
+  end
+end

data/lib/r10k/git/rugged/working_repository.rb

@@ -0,0 +1,149 @@
+require 'r10k/git/rugged'
+require 'r10k/git/rugged/base_repository'
+require 'r10k/git/errors'
+
+class R10K::Git::Rugged::WorkingRepository < R10K::Git::Rugged::BaseRepository
+
+  # @return [Pathname] The path to the Git repository inside of this directory
+  def git_dir
+    @path + '.git'
+  end
+
+  # @param basedir [String] The base directory of the Git repository
+  # @param dirname [String] The directory name of the Git repository
+  def initialize(basedir, dirname)
+    @path = Pathname.new(File.join(basedir, dirname))
+  end
+
+  # Clone this git repository
+  #
+  # @param remote [String] The Git remote to clone
+  # @param opts [Hash]
+  #
+  # @options opts [String] :ref The git ref to check out on clone
+  # @options opts [String] :reference A Git repository to use as an alternate object database
+  #
+  # @return [void]
+  def clone(remote, opts = {})
+    logger.debug1 { _("Cloning '%{remote}' into %{path}") % {remote: remote, path: @path } }
+
+    # libgit2/rugged doesn't support cloning a repository and providing an
+    # alternate object database, making the handling of :alternates a noop.
+    # Unfortunately this means that this method can't really use alternates
+    # and running the clone will duplicate all objects in the specified
+    # repository. However alternate databases can be handled when an existing
+    # repository is loaded, so loading a cloned repo will correctly use
+    # alternate object database.
+    options = {:credentials => credentials}
+    options.merge!(:alternates => [File.join(opts[:reference], 'objects')]) if opts[:reference]
+
+    proxy = R10K::Git.get_proxy_for_remote(remote)
+
+    R10K::Git.with_proxy(proxy) do
+      @_rugged_repo = ::Rugged::Repository.clone_at(remote, @path.to_s, options)
+    end
+
+    if opts[:reference]
+      alternates << File.join(opts[:reference], 'objects')
+    end
+
+    if opts[:ref]
+      # todo:  always check out something; since we're fetching a repository we
+      # won't populate the working directory.
+      checkout(opts[:ref])
+    end
+  rescue Rugged::SshError, Rugged::NetworkError => e
+    raise R10K::Git::GitError.new(e.message, :git_dir => git_dir, :backtrace => e.backtrace)
+  end
+
+  # Check out the given Git ref
+  #
+  # @param ref [String] The git reference to check out
+  # @return [void]
+  def checkout(ref, opts = {})
+    sha = resolve(ref)
+
+    if sha
+      logger.debug2 { _("Checking out ref '%{ref}' (resolved to SHA '%{sha}') in repository %{path}") % {ref: ref, sha: sha, path: @path} }
+    else
+      raise R10K::Git::GitError.new("Unable to check out unresolvable ref '#{ref}'", git_dir: git_dir)
+    end
+
+    # :force defaults to true
+    force = !opts.has_key?(:force) || opts[:force]
+
+    with_repo do |repo|
+      # rugged/libgit2 will not update (at least) the execute bit a file if the SHA is already at
+      # the value being reset to, so this is now changed to an if ... else
+      if force
+        repo.reset(sha, :hard)
+      else
+        repo.checkout(sha)
+      end
+    end
+  end
+
+  def fetch(remote_name = 'origin')
+    logger.debug1 { _("Fetching remote '%{remote}' at %{path}") % {remote: remote_name, path: @path} }
+    options = {:credentials => credentials}
+    refspecs = ["+refs/heads/*:refs/remotes/#{remote_name}/*"]
+
+    remote = remotes[remote_name]
+    proxy = R10K::Git.get_proxy_for_remote(remote)
+    results = nil
+
+    R10K::Git.with_proxy(proxy) do
+      results = with_repo { |repo| repo.fetch(remote_name, refspecs, **options) }
+    end
+
+    report_transfer(results, remote)
+  rescue Rugged::SshError, Rugged::NetworkError => e
+    raise R10K::Git::GitError.new(e.message, :git_dir => git_dir, :backtrace => e.backtrace)
+  end
+
+  def exist?
+    @path.exist?
+  end
+
+  def head
+    resolve('HEAD')
+  end
+
+  def alternates
+    R10K::Git::Alternates.new(git_dir)
+  end
+
+  def origin
+    with_repo { |repo| repo.config['remote.origin.url'] }
+  end
+
+  def dirty?(exclude_spec=false)
+    with_repo do |repo|
+      if exclude_spec
+        diff = repo.diff_workdir('HEAD').filter { |d| ! d.delta.old_file[:path].start_with?('spec/') }
+      else
+        diff = repo.diff_workdir('HEAD').to_a
+      end
+
+      diff.each do |p|
+        logger.debug(_("Found local modifications in %{file_path}" % {file_path: File.join(@path, p.delta.old_file[:path])}))
+        logger.debug1(p.to_s)
+      end
+
+      return diff.size > 0
+    end
+  end
+
+  private
+
+  def with_repo
+    if @_rugged_repo.nil? && git_dir.exist?
+      setup_rugged_repo
+    end
+    super
+  end
+
+  def setup_rugged_repo
+    @_rugged_repo = ::Rugged::Repository.new(@path.to_s, :alternates => alternates.to_a)
+  end
+end

data/lib/r10k/git/rugged.rb

@@ -0,0 +1,17 @@
+require 'r10k/git'
+
+begin
+  require 'rugged'
+rescue LoadError
+end
+
+module R10K
+  module Git
+    module Rugged
+      require 'r10k/git/rugged/bare_repository'
+      require 'r10k/git/rugged/working_repository'
+      require 'r10k/git/rugged/cache'
+      require 'r10k/git/rugged/thin_repository'
+    end
+  end
+end

data/lib/r10k/git/shellgit/bare_repository.rb

@@ -0,0 +1,43 @@
+require 'r10k/git/shellgit'
+require 'r10k/git/shellgit/base_repository'
+
+# Create and manage Git bare repositories.
+class R10K::Git::ShellGit::BareRepository < R10K::Git::ShellGit::BaseRepository
+
+  # @param basedir [String] The base directory of the Git repository
+  # @param dirname [String] The directory name of the Git repository
+  def initialize(basedir, dirname)
+    @path = Pathname.new(File.join(basedir, dirname))
+  end
+
+  # @return [Pathname] The path to this Git repository
+  def git_dir
+    @path
+  end
+
+  # @return [Pathname] The path to the objects directory in this Git repository
+  def objects_dir
+    @path + "objects"
+  end
+
+  def clone(remote)
+    proxy = R10K::Git.get_proxy_for_remote(remote)
+
+    R10K::Git.with_proxy(proxy) do
+      git ['clone', '--mirror', remote, git_dir.to_s]
+    end
+  end
+
+  def fetch(remote_name='origin')
+    remote = remotes[remote_name]
+    proxy = R10K::Git.get_proxy_for_remote(remote)
+
+    R10K::Git.with_proxy(proxy) do
+      git ['fetch', remote_name, '--prune'], :git_dir => git_dir.to_s
+    end
+  end
+
+  def exist?
+    @path.exist?
+  end
+end

data/lib/r10k/git/shellgit/base_repository.rb

@@ -0,0 +1,137 @@
+require 'r10k/git/shellgit'
+require 'r10k/util/subprocess'
+require 'r10k/logging'
+
+class R10K::Git::ShellGit::BaseRepository
+
+  # @abstract
+  # @return [Pathname] The path to the Git directory
+  def git_dir
+    raise NotImplementedError
+  end
+
+  # Resolve the given Git ref to a commit
+  #
+  # @param pattern [String] The git ref to resolve
+  # @return [String, nil] The commit SHA if the ref could be resolved, nil otherwise.
+  def resolve(pattern)
+    result = git ['rev-parse', "#{pattern}^{commit}"], :git_dir => git_dir.to_s, :raise_on_fail => false
+    if result.success?
+      result.stdout
+    end
+  end
+
+  # For compatibility with R10K::Git::Ref
+  # @todo remove alias
+  alias rev_parse resolve
+
+  # @return [Array<String>] All local branches in this repository
+  def branches
+    for_each_ref('refs/heads')
+  end
+
+  def is_branch?(pattern)
+    result = git ['rev-parse', '-q', '--verify', "refs/heads/#{pattern}"], :git_dir => git_dir.to_s, :raise_on_fail => false
+
+    result.success?
+  end
+
+  # @return [Array<String>] All tags in this repository
+  def tags
+    for_each_ref('refs/tags')
+  end
+
+  def is_tag?(pattern)
+    result = git ['rev-parse', '-q', '--verify', "refs/tags/#{pattern}"], :git_dir => git_dir.to_s, :raise_on_fail => false
+
+    result.success?
+  end
+
+  # @return [Symbol] The type of the given ref, one of :branch, :tag, :commit, or :unknown
+  def ref_type(pattern)
+    @_ref_type_cache ||= {}
+
+    @_ref_type_cache[pattern] ||= begin
+      # Try to match and resolve SHA refs as quickly as possible.
+      if pattern =~ /^[0-9a-f]{5,40}$/i && resolve(pattern)
+        :commit
+      elsif is_tag? pattern
+        :tag
+      elsif is_branch? pattern
+        :branch
+      elsif resolve(pattern)
+        :commit
+      else
+        :unknown
+      end
+    end
+  end
+
+  # @return [Hash] Collection of remotes for this repo, keys are the remote name and values are the remote URL.
+  def remotes
+    result = git ['config', '--local', '--get-regexp', '^remote\..*\.url$'], :git_dir => git_dir.to_s, :raise_on_fail => false
+
+    if result.success?
+      Hash[
+        result.stdout.split("\n").collect do |remote|
+          matches = /^remote\.(.*)\.url (.*)$/.match(remote)
+
+          [matches[1], matches[2]]
+        end
+      ]
+    else
+      {}
+    end
+  end
+
+  include R10K::Logging
+
+  private
+
+  # @param pattern [String]
+  def for_each_ref(pattern)
+    matcher = %r[#{pattern}/(.*)$]
+    output = git ['for-each-ref', pattern, '--format', '%(refname)'], :git_dir => git_dir.to_s
+    output.stdout.scan(matcher).flatten
+  end
+
+  # Wrap git commands
+  #
+  # @param cmd [Array<String>] cmd The arguments for the git prompt
+  # @param opts [Hash] opts
+  #
+  # @option opts [String] :path
+  # @option opts [String] :git_dir
+  # @option opts [String] :work_tree
+  # @option opts [String] :raise_on_fail
+  #
+  # @raise [R10K::ExecutionFailure] If the executed command exited with a
+  #   nonzero exit code.
+  #
+  # @return [String] The git command output
+  def git(cmd, opts = {})
+    raise_on_fail = opts.fetch(:raise_on_fail, true)
+
+    argv = %w{git}
+
+    if opts[:path]
+      argv << "--git-dir"   << File.join(opts[:path], '.git')
+      argv << "--work-tree" << opts[:path]
+    else
+      if opts[:git_dir]
+        argv << "--git-dir" << opts[:git_dir]
+      end
+      if opts[:work_tree]
+        argv << "--work-tree" << opts[:work_tree]
+      end
+    end
+
+    argv.concat(cmd)
+
+    subproc = R10K::Util::Subprocess.new(argv)
+    subproc.raise_on_fail = raise_on_fail
+    subproc.logger = self.logger
+
+    subproc.execute
+  end
+end

data/lib/r10k/git/shellgit/cache.rb

@@ -0,0 +1,11 @@
+require 'r10k/git/shellgit'
+require 'r10k/git/cache'
+
+class R10K::Git::ShellGit::Cache < R10K::Git::Cache
+
+  @instance_cache = R10K::InstanceCache.new(self)
+
+  def self.bare_repository
+    R10K::Git::ShellGit::BareRepository
+  end
+end