pdk 1.4.1 → 1.5.0

data/lib/pdk/util/bundler.rb

@@ -1,4 +1,5 @@
 require 'bundler'
+require 'digest'
 require 'fileutils'
 require 'pdk/util'
 require 'pdk/cli/exec'
@@ -8,11 +9,14 @@ module PDK
     module Bundler
       class BundleHelper; end
 
-      def self.ensure_bundle!
+      def self.ensure_bundle!(gem_overrides = nil)
         bundle = BundleHelper.new
 
-        if already_bundled?(bundle.gemfile)
-          PDK.logger.debug(_('Bundle has already been installed. Skipping run.'))
+        # This will default ensure_bundle! to re-resolving everything to latest
+        gem_overrides ||= { puppet: nil, hiera: nil, facter: nil }
+
+        if already_bundled?(bundle.gemfile, gem_overrides)
+          PDK.logger.debug(_('Bundler managed gems already up to date.'))
           return
         end
 
@@ -22,64 +26,83 @@ module PDK
         end
 
         unless bundle.locked?
-          if PDK::Util.package_install?
-            # In packaged installs, try to use vendored Gemfile.lock as a starting point.
-            # The 'bundle install' below will pick up any new dependencies.
-            vendored_gemfile_lock = File.join(PDK::Util.package_cachedir, 'Gemfile.lock')
+          # Generate initial default Gemfile.lock, either from package cache or
+          # by invoking `bundle lock`
+          bundle.lock!
+        end
 
-            if File.exist?(vendored_gemfile_lock)
-              PDK.logger.debug(_("No Gemfile.lock found in module. Using vendored Gemfile.lock from '%{source}'.") % { source: vendored_gemfile_lock })
-              FileUtils.cp(vendored_gemfile_lock, File.join(PDK::Util.module_root, 'Gemfile.lock'))
-            end
-          else
-            # In non-packaged installs, just let bundler resolve deps as normal.
-            unless bundle.lock!
-              raise PDK::CLI::FatalError, _('Unable to resolve Gemfile dependencies.')
-            end
-          end
+        # Check if all dependencies will be available once we update the lockfile.
+        begin
+          original_lockfile = bundle.gemfile_lock
+          temp_lockfile = "#{original_lockfile}.tmp"
+
+          FileUtils.mv(original_lockfile, temp_lockfile)
+
+          all_deps_available = bundle.installed?(gem_overrides)
+        ensure
+          FileUtils.mv(temp_lockfile, original_lockfile, force: true)
         end
 
+        bundle.update_lock!(with: gem_overrides, local: all_deps_available)
+
+        # If there are missing dependencies after updating the lockfile, let `bundle install`
+        # go out and get them.
         unless bundle.installed?
-          unless bundle.install!
-            raise PDK::CLI::FatalError, _('Unable to install missing Gemfile dependencies.')
-          end
+          bundle.install!(gem_overrides)
         end
 
-        mark_as_bundled!(bundle.gemfile)
+        mark_as_bundled!(bundle.gemfile, gem_overrides)
       end
 
-      def self.already_bundled?(gemfile)
-        !(@bundled ||= {})[gemfile].nil?
+      def self.ensure_binstubs!(*gems)
+        bundle = BundleHelper.new
+
+        bundle.binstubs!(gems)
       end
 
-      def self.mark_as_bundled!(gemfile)
-        (@bundled ||= {})[gemfile] = true
+      def self.already_bundled?(gemfile, gem_overrides)
+        !(@bundled ||= {})[bundle_cache_key(gemfile, gem_overrides)].nil?
       end
 
-      def self.ensure_binstubs!(*gems)
-        bundle = BundleHelper.new
+      def self.mark_as_bundled!(gemfile, gem_overrides)
+        (@bundled ||= {})[bundle_cache_key(gemfile, gem_overrides)] = true
+      end
 
-        unless bundle.binstubs!(gems) # rubocop:disable Style/GuardClause
-          raise PDK::CLI::FatalError, _('Unable to install requested binstubs.')
-        end
+      def self.bundle_cache_key(gemfile, gem_overrides)
+        override_sig = (gem_overrides || {}).sort_by { |gem, _| gem.to_s }.to_s
+        Digest::MD5.hexdigest(gemfile.to_s + override_sig)
       end
+      private_class_method :bundle_cache_key
 
       class BundleHelper
+        def gemfile
+          @gemfile ||= PDK::Util.find_upwards('Gemfile')
+        end
+
+        def gemfile_lock
+          return nil if gemfile.nil?
+          @gemfile_lock ||= File.join(File.dirname(gemfile), 'Gemfile.lock')
+        end
+
         def gemfile?
           !gemfile.nil?
         end
 
         def locked?
-          !gemfile_lock.nil?
+          !gemfile_lock.nil? && File.file?(gemfile_lock)
         end
 
-        def installed?
+        def installed?(gem_overrides = {})
           PDK.logger.debug(_('Checking for missing Gemfile dependencies.'))
 
-          argv = ['check', "--gemfile=#{gemfile}"]
+          argv = ['check', "--gemfile=#{gemfile}", '--dry-run']
           argv << "--path=#{bundle_cachedir}" unless PDK::Util.package_install?
 
-          result = bundle_command(*argv).execute!
+          cmd = bundle_command(*argv).tap do |c|
+            c.update_environment(gemfile_env(gem_overrides)) unless gem_overrides.empty?
+          end
+
+          result = cmd.execute!
 
           unless result[:exit_code].zero?
             PDK.logger.debug(result.values_at(:stdout, :stderr).join("\n"))
@@ -89,67 +112,141 @@ module PDK
         end
 
         def lock!
-          command = bundle_command('lock').tap do |c|
-            c.add_spinner(_('Resolving Gemfile dependencies.'))
+          if PDK::Util.package_install?
+            # In packaged installs, use vendored Gemfile.lock as a starting point.
+            # Subsequent 'bundle install' will still pick up any new dependencies.
+            vendored_lockfiles = [
+              File.join(PDK::Util.package_cachedir, "Gemfile-#{PDK::Util::RubyVersion.active_ruby_version}.lock"),
+              File.join(PDK::Util.package_cachedir, 'Gemfile.lock'),
+            ]
+
+            vendored_gemfile_lock = vendored_lockfiles.find { |lockfile| File.exist?(lockfile) }
+
+            unless vendored_gemfile_lock
+              raise PDK::CLI::FatalError, _('Vendored Gemfile.lock (%{source}) not found.') % {
+                source: vendored_gemfile_lock,
+              }
+            end
+
+            PDK.logger.debug(_('Using vendored Gemfile.lock from %{source}.') % { source: vendored_gemfile_lock })
+            FileUtils.cp(vendored_gemfile_lock, File.join(PDK::Util.module_root, 'Gemfile.lock'))
+          else
+            argv = ['lock']
+
+            cmd = bundle_command(*argv).tap do |c|
+              c.add_spinner(_('Resolving default Gemfile dependencies.'))
+            end
+
+            result = cmd.execute!
+
+            unless result[:exit_code].zero?
+              PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
+              raise PDK::CLI::FatalError, _('Unable to resolve default Gemfile dependencies.')
+            end
+
+            # After initial lockfile generation, re-resolve json gem to built-in
+            # version to avoid unncessary native compilation attempts. For packaged
+            # installs this is done during the generation of the vendored Gemfile.lock
+            update_lock!(only: { json: nil }, local: true)
+          end
+
+          true
+        end
+
+        def update_lock!(options = {})
+          PDK.logger.debug(_('Updating Gemfile dependencies.'))
+
+          argv = ['lock', '--update']
+
+          overrides = nil
+
+          if options && options[:only]
+            update_gems = options[:only].keys.map(&:to_s)
+            argv << update_gems
+            argv.flatten!
+
+            overrides = options[:only]
+          elsif options && options[:with]
+            overrides = options[:with]
+          end
+
+          argv << '--local' if options && options[:local]
+          argv << '--conservative' if options && options[:conservative]
+
+          cmd = bundle_command(*argv).tap do |c|
+            c.update_environment(gemfile_env(overrides)) if overrides
           end
 
-          result = command.execute!
+          result = cmd.execute!
 
           unless result[:exit_code].zero?
             PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
+            raise PDK::CLI::FatalError, _('Unable to resolve Gemfile dependencies.')
           end
 
-          result[:exit_code].zero?
+          true
         end
 
-        def install!
+        def install!(gem_overrides = {})
           argv = ['install', "--gemfile=#{gemfile}", '-j4']
           argv << "--path=#{bundle_cachedir}" unless PDK::Util.package_install?
 
-          command = bundle_command(*argv).tap do |c|
+          cmd = bundle_command(*argv).tap do |c|
             c.add_spinner(_('Installing missing Gemfile dependencies.'))
+            c.update_environment(gemfile_env(gem_overrides)) unless gem_overrides.empty?
           end
 
-          result = command.execute!
+          result = cmd.execute!
 
           unless result[:exit_code].zero?
             PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
+            raise PDK::CLI::FatalError, _('Unable to install missing Gemfile dependencies.')
           end
 
-          result[:exit_code].zero?
+          true
         end
 
         def binstubs!(gems)
           binstub_dir = File.join(File.dirname(gemfile), 'bin')
           return true if gems.all? { |gem| File.file?(File.join(binstub_dir, gem)) }
 
-          command = bundle_command('binstubs', *gems, '--force')
-
-          result = command.execute!
+          cmd = bundle_command('binstubs', *gems, '--force')
+          result = cmd.execute!
 
           unless result[:exit_code].zero?
             PDK.logger.fatal(_("Failed to generate binstubs for '%{gems}':\n%{output}") % { gems: gems.join(' '), output: result.values_at(:stdout, :stderr).join("\n") })
+            raise PDK::CLI::FatalError, _('Unable to install requested binstubs.')
           end
 
-          result[:exit_code].zero?
+          true
         end
 
-        def gemfile
-          @gemfile ||= PDK::Util.find_upwards('Gemfile')
+        def self.gemfile_env(gem_overrides)
+          gemfile_env = {}
+
+          return gemfile_env unless gem_overrides.respond_to?(:each)
+
+          gem_overrides.each do |gem, version|
+            gemfile_env['PUPPET_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'puppet' && !version.nil?
+            gemfile_env['FACTER_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'facter' && !version.nil?
+            gemfile_env['HIERA_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'hiera' && !version.nil?
+          end
+
+          gemfile_env
         end
 
         private
 
+        def gemfile_env(gem_overrides)
+          self.class.gemfile_env(gem_overrides)
+        end
+
         def bundle_command(*args)
           PDK::CLI::Exec::Command.new(PDK::CLI::Exec.bundle_bin, *args).tap do |c|
             c.context = :module
           end
         end
 
-        def gemfile_lock
-          @gemfile_lock ||= PDK::Util.find_upwards('Gemfile.lock')
-        end
-
         def bundle_cachedir
           @bundle_cachedir ||= PDK::Util.package_install? ? PDK::Util.package_cachedir : File.join(PDK::Util.cachedir)
         end

data/lib/pdk/util/git.rb

@@ -32,10 +32,27 @@ module PDK
         PDK::CLI::Exec.execute(git_bin, *args)
       end
 
-      def self.repo_exists?(repo, ref = nil)
-        args = ['ls-remote', '--exit-code', repo, ref].compact
+      def self.git_with_env(env, *args)
+        PDK::CLI::Exec.ensure_bin_present!(git_bin, 'git')
+
+        PDK::CLI::Exec.execute_with_env(env, git_bin, *args)
+      end
+
+      def self.repo?(maybe_repo)
+        return bare_repo?(maybe_repo) if File.directory?(maybe_repo)
+
+        remote_repo?(maybe_repo)
+      end
+
+      def self.bare_repo?(maybe_repo)
+        env = { 'GIT_DIR' => maybe_repo }
+        rev_parse = git_with_env(env, 'rev-parse', '--is-bare-repository')
+
+        rev_parse[:exit_code].zero? && rev_parse[:stdout].strip == 'true'
+      end
 
-        git(*args)[:exit_code].zero?
+      def self.remote_repo?(maybe_repo)
+        git('ls-remote', '--exit-code', maybe_repo)[:exit_code].zero?
       end
 
       def self.ls_remote(repo, ref)

data/lib/pdk/util/puppet_version.rb

@@ -0,0 +1,182 @@
+require 'pdk/util'
+
+module PDK
+  module Util
+    class PuppetVersion
+      class << self
+        extend Forwardable
+
+        def_delegators :instance, :find_gem_for, :from_pe_version, :from_module_metadata, :latest_available
+
+        attr_writer :instance
+
+        def instance
+          @instance ||= new
+        end
+      end
+
+      PE_VERSIONS_URL = 'https://forgeapi.puppet.com/private/versions/pe'.freeze
+
+      def latest_available
+        latest = find_gem(Gem::Requirement.create('>= 0'))
+
+        if latest.nil?
+          raise ArgumentError, _('Unable to find a Puppet gem in current Ruby environment or from Rubygems.org.')
+        end
+
+        latest
+      end
+
+      def find_gem_for(version_str)
+        version = parse_specified_version(version_str)
+
+        # Look for a gem matching exactly the version passed in.
+        if version.segments.length == 3
+          exact_match_gem = find_gem(Gem::Requirement.create(version))
+          return exact_match_gem unless exact_match_gem.nil?
+        end
+
+        # Construct a pessimistic version constraint to find the latest
+        # available gem matching the level of specificity of version_str.
+        requirement_string = version.approximate_recommendation
+        requirement_string += '.0' unless version.segments.length == 1
+        latest_requirement = Gem::Requirement.create(requirement_string)
+
+        latest_available_gem = find_gem(latest_requirement)
+
+        if latest_available_gem.nil?
+          raise ArgumentError, _('Unable to find a Puppet gem matching %{requirement}.') % {
+            requirement: latest_requirement,
+          }
+        end
+
+        # Only issue this warning if they requested an exact version that isn't available.
+        if version.segments.length == 3
+          PDK.logger.warn(_('Puppet %{requested_version} is not available, activating %{found_version} instead.') % {
+            requested_version: version_str,
+            found_version:     latest_available_gem[:gem_version].version,
+          })
+        end
+
+        latest_available_gem
+      end
+
+      def from_pe_version(version_str)
+        version = parse_specified_version(version_str)
+
+        gem_version = pe_version_map.find do |version_map|
+          version_map[:requirement].satisfied_by?(version)
+        end
+
+        if gem_version.nil?
+          raise ArgumentError, _('Unable to map Puppet Enterprise version %{pe_version} to a Puppet version.') % {
+            pe_version: version_str,
+          }
+        end
+
+        PDK.logger.info _('Puppet Enterprise %{pe_version} maps to Puppet %{puppet_version}.') % {
+          pe_version:     version_str,
+          puppet_version: gem_version[:gem_version],
+        }
+
+        find_gem_for(gem_version[:gem_version])
+      end
+
+      def from_module_metadata(metadata = nil)
+        metadata ||= PDK::Module::Metadata.from_file(PDK::Util.find_upwards('metadata.json'))
+        metadata.validate_puppet_version_requirement!
+        metadata_requirement = metadata.puppet_requirement
+
+        # Split combined requirements like ">= 4.7.0 < 6.0.0" into their
+        # component requirements [">= 4.7.0", "< 6.0.0"]
+        pattern = %r{#{Gem::Requirement::PATTERN_RAW}}
+        requirement_strings = metadata_requirement['version_requirement'].scan(pattern).map do |req|
+          req.compact.join(' ')
+        end
+
+        gem_requirement = Gem::Requirement.create(requirement_strings)
+        find_gem(gem_requirement)
+      end
+
+      private
+
+      def parse_specified_version(version_str)
+        Gem::Version.new(version_str)
+      rescue ArgumentError
+        raise ArgumentError, _('%{version} is not a valid version number.') % {
+          version: version_str,
+        }
+      end
+
+      def pe_version_map
+        @pe_version_map ||= fetch_pe_version_map.map { |version_map|
+          maps = version_map['versions'].map do |pe_release|
+            requirements = ["= #{pe_release['version']}"]
+
+            # Some PE release have a .0 Z release, which causes problems when
+            # the user specifies "X.Y" expecting to get the latest Z and
+            # instead getting the oldest.
+            requirements << "!= #{pe_release['version'].gsub(%r{\.\d+\Z}, '')}" if pe_release['version'].end_with?('.0')
+            {
+              requirement: Gem::Requirement.create(requirements),
+              gem_version: pe_release['puppet'],
+            }
+          end
+
+          maps << {
+            requirement: requirement_from_forge_range(version_map['release']),
+            gem_version: version_map['versions'].find { |r| r['version'] == version_map['latest'] }['puppet'],
+          }
+        }.flatten
+      end
+
+      def fetch_pe_version_map
+        map = PDK::Util::VendoredFile.new('pe_versions.json', PE_VERSIONS_URL).read
+
+        JSON.parse(map)
+      rescue PDK::Util::VendoredFile::DownloadError => e
+        raise PDK::CLI::FatalError, e.message
+      rescue JSON::ParserError
+        raise PDK::CLI::FatalError, _('Failed to parse Puppet Enterprise version map file.')
+      end
+
+      def requirement_from_forge_range(range_str)
+        Gem::Requirement.create("~> #{range_str.gsub(%r{\.x\Z}, '.0')}")
+      end
+
+      def rubygems_puppet_versions
+        return @rubygems_puppet_versions unless @rubygems_puppet_versions.nil?
+
+        fetcher = Gem::SpecFetcher.fetcher
+        puppet_tuples = fetcher.detect(:released) do |spec_tuple|
+          spec_tuple.name == 'puppet' && Gem::Platform.match(spec_tuple.platform)
+        end
+        puppet_versions = puppet_tuples.map { |name, _| name.version }.uniq
+        @rubygems_puppet_versions = puppet_versions.sort { |a, b| b <=> a }
+      end
+
+      def find_gem(requirement)
+        if PDK::Util.package_install?
+          find_in_package_cache(requirement)
+        else
+          find_in_rubygems(requirement)
+        end
+      end
+
+      def find_in_rubygems(requirement)
+        version = rubygems_puppet_versions.find { |r| requirement.satisfied_by?(r) }
+        version.nil? ? nil : { gem_version: version, ruby_version: PDK::Util::RubyVersion.default_ruby_version }
+      end
+
+      def find_in_package_cache(requirement)
+        PDK::Util::RubyVersion.versions.each do |ruby_version, _|
+          PDK::Util::RubyVersion.use(ruby_version)
+          version = PDK::Util::RubyVersion.available_puppet_versions.find { |r| requirement.satisfied_by?(r) }
+          return { gem_version: version, ruby_version: ruby_version } unless version.nil?
+        end
+
+        nil
+      end
+    end
+  end
+end