puppetfile-resolver 0.5.0 → 0.6.2

data/lib/puppetfile-resolver/spec_searchers/git/gclone.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'tempfile'
+require 'English'
+require 'puppetfile-resolver/util'
+require 'puppetfile-resolver/spec_searchers/common'
+require 'puppetfile-resolver/spec_searchers/git_configuration'
+require 'puppetfile-resolver/util'
+require 'uri'
+
+module PuppetfileResolver
+  module SpecSearchers
+    module Git
+      module GClone
+        METADATA_FILE = 'metadata.json'
+
+        # @summary clones the remote url and reads the metadata file
+        # @returns [String] the content of the metadata file
+        def self.metadata(puppetfile_module, resolver_ui, config)
+          repo_url = puppetfile_module.remote
+
+          unless PuppetfileResolver::Util.git?
+            resolver_ui.debug { 'Git executible not found, unable to use git clone resolution' }
+
+            return nil
+          end
+          return nil if repo_url.nil?
+          return nil unless valid_http_url?(repo_url)
+
+          ref = puppetfile_module.ref ||
+                puppetfile_module.tag ||
+                puppetfile_module.commit ||
+                puppetfile_module.branch ||
+                'HEAD'
+
+          resolver_ui.debug { "Querying git repository #{repo_url}" }
+
+          clone_and_read_file(repo_url, ref, config)
+        end
+
+        # @summary clones the git url and reads the file at the given ref
+        #          a temp directory will be created and then destroyed during
+        #          the cloning and reading process
+        # @param ref [String] the git ref, branch, commit, tag
+        # @param file [String] the file you wish to read
+        # @returns [String] the content of the file
+        def self.clone_and_read_file(url, ref, config)
+          Dir.mktmpdir(nil, config.clone_dir) do |dir|
+            clone = ['git', 'clone', url, dir]
+            clone += ['--config', "http.proxy=#{config.proxy}", '--config', "https.proxy=#{config.proxy}"] if config.proxy
+
+            bare_clone = clone + ['--bare', '--depth=1']
+            bare_clone.push("--branch=#{ref}") unless ref == 'HEAD'
+
+            # Try to clone a bare repository. If that fails, fall back to a full clone.
+            # Cloning might fail because the repo does not exist or is otherwise
+            # inaccessible, but it can also fail because cloning a bare repository from
+            # a commit/SHA1 fails. Falling back to a full clone ensures that we support
+            # commits/SHA1s like Puppetfile does.
+            _stdout, _stderr, process = ::PuppetfileResolver::Util.run_command(bare_clone)
+
+            unless process.success?
+              _stdout, stderr, process = ::PuppetfileResolver::Util.run_command(clone)
+
+              unless process.success?
+                msg = if config.proxy
+                        "Cloning #{url} with proxy #{config.proxy} failed: #{stderr}"
+                      else
+                        "Cloning #{url} failed: #{stderr}"
+                      end
+                raise msg
+              end
+            end
+
+            Dir.chdir(dir) do
+              content, stderr, process = ::PuppetfileResolver::Util.run_command(['git', 'show', "#{ref}:#{METADATA_FILE}"])
+              raise "Could not find #{METADATA_FILE} for ref #{ref} at #{url}: #{stderr}" unless process.success?
+              return content
+            end
+          end
+        end
+
+        def self.valid_http_url?(url)
+          # uri does not work with git urls, return true
+          return true if url.start_with?('git@')
+
+          uri = URI.parse(url)
+          uri.is_a?(URI::HTTP) && !uri.host.nil?
+        rescue URI::InvalidURIError
+          false
+        end
+      end
+    end
+  end
+end

data/lib/puppetfile-resolver/spec_searchers/git/github.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'puppetfile-resolver/util'
+require 'puppetfile-resolver/spec_searchers/common'
+require 'puppetfile-resolver/spec_searchers/git_configuration'
+
+module PuppetfileResolver
+  module SpecSearchers
+    module Git
+      module GitHub
+        def self.metadata(puppetfile_module, resolver_ui, config)
+          repo_url = nil
+          if puppetfile_module.remote.start_with?('git@github.com:')
+            repo_url = puppetfile_module.remote.slice(15..-1)
+            repo_url = repo_url.slice(0..-5) if repo_url.end_with?('.git')
+          elsif puppetfile_module.remote.start_with?('https://github.com/')
+            repo_url = puppetfile_module.remote.slice(19..-1)
+            repo_url = repo_url.slice(0..-5) if repo_url.end_with?('.git')
+          end
+          return nil if repo_url.nil?
+
+          metadata_url = 'https://raw.githubusercontent.com/' + repo_url + '/'
+          if puppetfile_module.ref
+            metadata_url += puppetfile_module.ref + '/'
+          elsif puppetfile_module.tag
+            metadata_url += puppetfile_module.tag + '/'
+          else
+            # Default to master. Should it raise?
+            metadata_url += 'master/'
+          end
+          metadata_url += 'metadata.json'
+
+          resolver_ui.debug { "Querying GitHub with #{metadata_url}" }
+          err_msg = "Unable to find module at #{puppetfile_module.remote}"
+          err_msg += config.proxy ? " with proxy #{config.proxy}: " : ': '
+          response = nil
+
+          begin
+            response = ::PuppetfileResolver::Util.net_http_get(metadata_url, config.proxy)
+          rescue ::StandardError => e
+            raise err_msg + e.message
+          end
+
+          if response.code != '200'
+            resolver_ui.debug(err_msg + "Expected HTTP Code 200, but received #{response.code}")
+            return nil
+          end
+          response.body
+        end
+      end
+    end
+  end
+end

data/lib/puppetfile-resolver/spec_searchers/git/gitlab.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'puppetfile-resolver/util'
+require 'puppetfile-resolver/spec_searchers/common'
+require 'puppetfile-resolver/spec_searchers/git_configuration'
+
+module PuppetfileResolver
+  module SpecSearchers
+    module Git
+      module GitLab
+        def self.metadata(puppetfile_module, resolver_ui, config)
+          repo_url = nil
+          if puppetfile_module.remote.start_with?('git@gitlab.com:')
+            repo_url = puppetfile_module.remote.slice(15..-1)
+            repo_url = repo_url.slice(0..-5) if repo_url.end_with?('.git')
+          elsif puppetfile_module.remote.start_with?('https://gitlab.com/')
+            repo_url = puppetfile_module.remote.slice(19..-1)
+            repo_url = repo_url.slice(0..-5) if repo_url.end_with?('.git')
+          end
+          return nil if repo_url.nil?
+
+          # Example URL
+          # https://gitlab.com/simp/pupmod-simp-crypto_policy/-/raw/0.1.4/metadata.json
+          metadata_url = 'https://gitlab.com/' + repo_url + '/-/raw/'
+          if puppetfile_module.ref
+            metadata_url += puppetfile_module.ref + '/'
+          elsif puppetfile_module.tag
+            metadata_url += puppetfile_module.tag + '/'
+          else
+            # Default to master. Should it raise?
+            metadata_url += 'master/'
+          end
+          metadata_url += 'metadata.json'
+
+          resolver_ui.debug { "Querying GitLab with #{metadata_url}" }
+          err_msg = "Unable to find module at #{puppetfile_module.remote}"
+          err_msg += config.proxy ? " with proxy #{config.proxy}: " : ': '
+          response = nil
+
+          begin
+            response = ::PuppetfileResolver::Util.net_http_get(metadata_url, config.proxy)
+          rescue ::StandardError => e
+            raise err_msg + e.message
+          end
+
+          if response.code != '200'
+            resolver_ui.debug(err_msg + "Expected HTTP Code 200, but received #{response.code}")
+            return nil
+          end
+          response.body
+        end
+      end
+    end
+  end
+end

data/lib/puppetfile-resolver/spec_searchers/git.rb

@@ -3,6 +3,9 @@
 require 'puppetfile-resolver/util'
 require 'puppetfile-resolver/spec_searchers/common'
 require 'puppetfile-resolver/spec_searchers/git_configuration'
+require 'puppetfile-resolver/spec_searchers/git/github'
+require 'puppetfile-resolver/spec_searchers/git/gitlab'
+require 'puppetfile-resolver/spec_searchers/git/gclone'
 
 module PuppetfileResolver
   module SpecSearchers
@@ -12,54 +15,21 @@ module PuppetfileResolver
         # Has the information been cached?
         return cache.load(dep_id) if cache.exist?(dep_id)
 
-        # We _could_ git clone this, but it'll take too long. So for now, just
-        # try and resolve github based repositories by crafting a URL
+        # The git clone method takes around (1s) depending on repo size.  Not sure if the
+        # other methods take longer or shorter but I preserved the legacy code for now.
+        # Technically, the gclone class could replace the other classes and speed up
+        # this process here.
+        metadata = GitHub.metadata(puppetfile_module, resolver_ui, config) ||
+                   GitLab.metadata(puppetfile_module, resolver_ui, config) ||
+                   GClone.metadata(puppetfile_module, resolver_ui, config)
 
-        repo_url = nil
-        if puppetfile_module.remote.start_with?('git@github.com:')
-          repo_url = puppetfile_module.remote.slice(15..-1)
-          repo_url = repo_url.slice(0..-5) if repo_url.end_with?('.git')
-        end
-        if puppetfile_module.remote.start_with?('https://github.com/')
-          repo_url = puppetfile_module.remote.slice(19..-1)
-          repo_url = repo_url.slice(0..-5) if repo_url.end_with?('.git')
-        end
-
-        return [] if repo_url.nil?
-
-        metadata_url = 'https://raw.githubusercontent.com/' + repo_url + '/'
-        if puppetfile_module.ref
-          metadata_url += puppetfile_module.ref + '/'
-        elsif puppetfile_module.tag
-          metadata_url += puppetfile_module.tag + '/'
-        else
-          # Default to master.  Should it raise?
-          metadata_url += 'master/'
-        end
-        metadata_url += 'metadata.json'
-
-        require 'net/http'
-        require 'uri'
-
-        resolver_ui.debug { "Querying GitHub with #{metadata_url}" }
-        err_msg = "Unable to find module at #{puppetfile_module.remote}"
-        err_msg += config.proxy ? " with proxy #{config.proxy}: " : ': '
-        response = nil
-
-        begin
-          response = ::PuppetfileResolver::Util.net_http_get(metadata_url, config.proxy)
-        rescue ::StandardError => e
-          raise err_msg + e.message
-        end
-
-        if response.code != '200'
-          resolver_ui.debug(err_msg + "Expected HTTP Code 200, but received #{response.code}")
+        if metadata.nil? || metadata.empty?
+          # Cache that we couldn't find the metadata
           cache.save(dep_id, [])
           return []
         end
 
-        # TODO: symbolise_object should be in a Util namespace
-        metadata = ::PuppetfileResolver::Util.symbolise_object(::JSON.parse(response.body))
+        metadata = ::PuppetfileResolver::Util.symbolise_object(::JSON.parse(metadata))
         result = [Models::ModuleSpecification.new(
           name: metadata[:name],
           origin: :git,

data/lib/puppetfile-resolver/spec_searchers/git_configuration.rb

@@ -3,7 +3,7 @@
 module PuppetfileResolver
   module SpecSearchers
     class GitConfiguration
-      attr_accessor :proxy
+      attr_accessor :proxy, :clone_dir
     end
   end
 end

data/lib/puppetfile-resolver/util.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'open3'
+
 module PuppetfileResolver
   module Util
     def self.symbolise_object(object)
@@ -43,5 +45,22 @@ module PuppetfileResolver
       Net::HTTP.start(*start_args, http_options) { |http| return http.request(Net::HTTP::Get.new(uri)) }
       nil
     end
+
+    # @summary runs the command on the shell
+    # @param cmd [Array] an array of command and args
+    # @returns [Array] the result of running the comand and the process
+    # @example run_command(['git', '--version'])
+    def self.run_command(cmd)
+      Open3.capture3(*cmd)
+    end
+
+    # @summary checks if git is installed and on the path
+    # @returns [Boolean] true if git is found in the path
+    def self.git?
+      Open3.capture3('git', '--version')
+      true
+    rescue Errno::ENOENT
+      false
+    end
   end
 end

data/lib/puppetfile-resolver/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PuppetfileResolver
-  VERSION ||= '0.5.0'
+  VERSION ||= '0.6.2'
 end

data/spec/integration/kitchensink_spec.rb

@@ -3,10 +3,8 @@ require 'spec_helper'
 require 'puppetfile-resolver/resolver'
 require 'puppetfile-resolver/puppetfile'
 
-
 describe 'KitchenSink Tests' do
   it 'should resolve a complete Puppetfile' do
-
     content = <<-PUPFILE
     forge 'https://forge.puppet.com'
 
@@ -14,6 +12,10 @@ describe 'KitchenSink Tests' do
       :git => 'https://github.com/puppetlabs/puppetlabs-powershell',
       :tag => 'v4.0.0'
 
+    mod 'simpkv',
+      :git => 'https://gitlab.com/simp/pupmod-simp-simpkv.git',
+      :tag => '0.7.1'
+
     mod 'puppetlabs/stdlib', '6.3.0'
 
     # Local module path module
@@ -35,6 +37,9 @@ describe 'KitchenSink Tests' do
     expect(result.specifications).to include('powershell')
     expect(result.specifications['powershell'].version.to_s).to eq('4.0.0')
 
+    expect(result.specifications).to include('simpkv')
+    expect(result.specifications['simpkv'].version.to_s).to eq('0.7.1')
+
     expect(result.specifications).to include('stdlib')
     expect(result.specifications['stdlib'].version.to_s).to eq('6.3.0')
 

data/spec/unit/puppetfile-resolver/spec_searchers/git/gclone_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+require 'puppetfile-resolver/spec_searchers/git/gclone'
+require 'puppetfile-resolver/spec_searchers/git_configuration'
+require 'logger'
+require 'json'
+
+describe PuppetfileResolver::SpecSearchers::Git::GClone do
+  PuppetfileModule = Struct.new(:remote, :ref, :branch, :commit, :tag, keyword_init: true)
+  config = PuppetfileResolver::SpecSearchers::GitConfiguration.new
+
+  let(:url) do
+    'https://github.com/puppetlabs/puppetlabs-powershell'
+  end
+
+  let(:puppetfile_module) do
+    PuppetfileModule.new(remote: url)
+  end
+
+
+  context 'valid url' do
+    it 'reads metadata' do
+      content = subject.metadata(puppetfile_module, Logger.new(STDERR), config)
+      expect(JSON.parse(content)['name']).to eq('puppetlabs-powershell')
+    end
+
+    context 'with tag' do
+      let(:puppetfile_module) do
+        PuppetfileModule.new(remote: url, ref: '2.1.2')
+      end
+
+      it 'reads metadata' do
+        content = subject.metadata(puppetfile_module, Logger.new(STDERR), config)
+        expect(JSON.parse(content)['name']).to eq('puppetlabs-powershell')
+      end
+    end
+
+    context 'with commit' do
+      let(:puppetfile_module) do
+        PuppetfileModule.new(remote: url, ref: '9276de695798097e8471b877a18df27f764eecda')
+      end
+
+      it 'reads metadata' do
+        content = subject.metadata(puppetfile_module, Logger.new(STDERR), config)
+        expect(JSON.parse(content)['name']).to eq('puppetlabs-powershell')
+      end
+    end
+
+    context 'with invalid ref' do
+      let(:puppetfile_module) do
+        PuppetfileModule.new(remote: url, ref: '8f7d5ea3ef49dadc5e166d5d802d091abc4b02bc')
+      end
+
+      it 'errors gracefully' do
+        expect { subject.metadata(puppetfile_module, Logger.new(STDERR), config) }.to raise_error(
+          /Could not find metadata\.json for ref .* at .*/
+        )
+      end
+    end
+  end
+
+  context 'invalid url' do
+    let(:url) do
+      'https://user:password@github.com/puppetlabs/puppetlabs-powershellbad'
+    end
+
+    it 'throws exception' do
+      expect{subject.metadata(puppetfile_module, Logger.new(STDERR), config)}
+      .to raise_exception(RuntimeError)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppetfile-resolver
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.2
 platform: ruby
 authors:
 - Glenn Sarti
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-16 00:00:00.000000000 Z
+date: 2022-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: molinillo
@@ -85,6 +85,9 @@ files:
 - lib/puppetfile-resolver/spec_searchers/forge.rb
 - lib/puppetfile-resolver/spec_searchers/forge_configuration.rb
 - lib/puppetfile-resolver/spec_searchers/git.rb
+- lib/puppetfile-resolver/spec_searchers/git/gclone.rb
+- lib/puppetfile-resolver/spec_searchers/git/github.rb
+- lib/puppetfile-resolver/spec_searchers/git/gitlab.rb
 - lib/puppetfile-resolver/spec_searchers/git_configuration.rb
 - lib/puppetfile-resolver/spec_searchers/local.rb
 - lib/puppetfile-resolver/spec_searchers/local_configuration.rb
@@ -103,11 +106,12 @@ files:
 - spec/unit/puppetfile-resolver/puppetfile/document_spec.rb
 - spec/unit/puppetfile-resolver/puppetfile/parser/r10k_eval_spec.rb
 - spec/unit/puppetfile-resolver/resolver_spec.rb
+- spec/unit/puppetfile-resolver/spec_searchers/git/gclone_spec.rb
 homepage: https://glennsarti.github.io/puppetfile-resolver/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -122,18 +126,19 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: Dependency resolver for Puppetfiles
 test_files:
-- spec/fixtures/modulepath/test_module/metadata.json
-- spec/fixtures/modulepath/test_module/README.md
-- spec/fixtures/proxy.rb
-- spec/integration/deprecation_spec.rb
-- spec/integration/kitchensink_spec.rb
-- spec/integration/proxy_spec.rb
-- spec/spec_helper.rb
+- spec/unit/puppetfile-resolver/spec_searchers/git/gclone_spec.rb
 - spec/unit/puppetfile-resolver/puppetfile/document_spec.rb
 - spec/unit/puppetfile-resolver/puppetfile/parser/r10k_eval_spec.rb
 - spec/unit/puppetfile-resolver/resolver_spec.rb
+- spec/integration/proxy_spec.rb
+- spec/integration/kitchensink_spec.rb
+- spec/integration/deprecation_spec.rb
+- spec/spec_helper.rb
+- spec/fixtures/modulepath/test_module/metadata.json
+- spec/fixtures/modulepath/test_module/README.md
+- spec/fixtures/proxy.rb