pdk-akerl 1.8.0.1

data/lib/pdk/util/bundler.rb

@@ -0,0 +1,253 @@
+require 'bundler'
+require 'digest'
+require 'fileutils'
+require 'pdk/util'
+require 'pdk/cli/exec'
+
+module PDK
+  module Util
+    module Bundler
+      class BundleHelper; end
+
+      def self.ensure_bundle!(gem_overrides = nil)
+        bundle = BundleHelper.new
+
+        # This will default ensure_bundle! to re-resolving everything to latest
+        gem_overrides ||= { puppet: nil, hiera: nil, facter: nil }
+
+        if already_bundled?(bundle.gemfile, gem_overrides)
+          PDK.logger.debug(_('Bundler managed gems already up to date.'))
+          return
+        end
+
+        unless bundle.gemfile?
+          PDK.logger.debug(_("No Gemfile found in '%{cwd}'. Skipping bundler management.") % { cwd: Dir.pwd })
+          return
+        end
+
+        unless bundle.locked?
+          # Generate initial default Gemfile.lock, either from package cache or
+          # by invoking `bundle lock`
+          bundle.lock!
+        end
+
+        # Check if all dependencies will be available once we update the lockfile.
+        begin
+          original_lockfile = bundle.gemfile_lock
+          temp_lockfile = "#{original_lockfile}.tmp"
+
+          FileUtils.mv(original_lockfile, temp_lockfile)
+
+          all_deps_available = bundle.installed?(gem_overrides)
+        ensure
+          FileUtils.mv(temp_lockfile, original_lockfile, force: true)
+        end
+
+        bundle.update_lock!(with: gem_overrides, local: all_deps_available)
+
+        # If there are missing dependencies after updating the lockfile, let `bundle install`
+        # go out and get them. If the specified puppet gem version points to a remote location
+        # or local filepath, then run bundle install as well.
+        if !bundle.installed? || (gem_overrides[:puppet] && gem_overrides[:puppet].start_with?('file://', 'git://', 'https://'))
+          bundle.install!(gem_overrides)
+        end
+
+        mark_as_bundled!(bundle.gemfile, gem_overrides)
+      end
+
+      def self.ensure_binstubs!(*gems)
+        bundle = BundleHelper.new
+
+        bundle.binstubs!(gems)
+      end
+
+      def self.already_bundled?(gemfile, gem_overrides)
+        !(@bundled ||= {})[bundle_cache_key(gemfile, gem_overrides)].nil?
+      end
+
+      def self.mark_as_bundled!(gemfile, gem_overrides)
+        (@bundled ||= {})[bundle_cache_key(gemfile, gem_overrides)] = true
+      end
+
+      def self.bundle_cache_key(gemfile, gem_overrides)
+        override_sig = (gem_overrides || {}).sort_by { |gem, _| gem.to_s }.to_s
+        Digest::MD5.hexdigest(gemfile.to_s + override_sig)
+      end
+      private_class_method :bundle_cache_key
+
+      class BundleHelper
+        def gemfile
+          @gemfile ||= PDK::Util.find_upwards('Gemfile')
+        end
+
+        def gemfile_lock
+          return nil if gemfile.nil?
+          @gemfile_lock ||= File.join(File.dirname(gemfile), 'Gemfile.lock')
+        end
+
+        def gemfile?
+          !gemfile.nil?
+        end
+
+        def locked?
+          !gemfile_lock.nil? && File.file?(gemfile_lock)
+        end
+
+        def installed?(gem_overrides = {})
+          PDK.logger.debug(_('Checking for missing Gemfile dependencies.'))
+
+          argv = ['check', "--gemfile=#{gemfile}", '--dry-run']
+
+          cmd = bundle_command(*argv).tap do |c|
+            c.update_environment(gemfile_env(gem_overrides)) unless gem_overrides.empty?
+          end
+
+          result = cmd.execute!
+
+          unless result[:exit_code].zero?
+            PDK.logger.debug(result.values_at(:stdout, :stderr).join("\n"))
+          end
+
+          result[:exit_code].zero?
+        end
+
+        def lock!
+          if PDK::Util.package_install?
+            # In packaged installs, use vendored Gemfile.lock as a starting point.
+            # Subsequent 'bundle install' will still pick up any new dependencies.
+            vendored_lockfiles = [
+              File.join(PDK::Util.package_cachedir, "Gemfile-#{PDK::Util::RubyVersion.active_ruby_version}.lock"),
+              File.join(PDK::Util.package_cachedir, 'Gemfile.lock'),
+            ]
+
+            vendored_gemfile_lock = vendored_lockfiles.find { |lockfile| File.exist?(lockfile) }
+
+            unless vendored_gemfile_lock
+              raise PDK::CLI::FatalError, _('Vendored Gemfile.lock (%{source}) not found.') % {
+                source: vendored_gemfile_lock,
+              }
+            end
+
+            PDK.logger.debug(_('Using vendored Gemfile.lock from %{source}.') % { source: vendored_gemfile_lock })
+            FileUtils.cp(vendored_gemfile_lock, File.join(PDK::Util.module_root, 'Gemfile.lock'))
+          else
+            argv = ['lock']
+
+            cmd = bundle_command(*argv).tap do |c|
+              c.add_spinner(_('Resolving default Gemfile dependencies.'))
+            end
+
+            result = cmd.execute!
+
+            unless result[:exit_code].zero?
+              PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
+              raise PDK::CLI::FatalError, _('Unable to resolve default Gemfile dependencies.')
+            end
+
+            # After initial lockfile generation, re-resolve json gem to built-in
+            # version to avoid unncessary native compilation attempts. For packaged
+            # installs this is done during the generation of the vendored Gemfile.lock
+            update_lock!(only: { json: nil }, local: true)
+          end
+
+          true
+        end
+
+        def update_lock!(options = {})
+          PDK.logger.debug(_('Updating Gemfile dependencies.'))
+
+          argv = ['lock', "--lockfile=#{gemfile_lock}", '--update']
+
+          overrides = nil
+
+          if options && options[:only]
+            update_gems = options[:only].keys.map(&:to_s)
+            argv << update_gems
+            argv.flatten!
+
+            overrides = options[:only]
+          elsif options && options[:with]
+            overrides = options[:with]
+          end
+
+          argv << '--local' if options && options[:local]
+          argv << '--conservative' if options && options[:conservative]
+
+          cmd = bundle_command(*argv).tap do |c|
+            c.update_environment('BUNDLE_GEMFILE' => gemfile)
+            c.update_environment(gemfile_env(overrides)) if overrides
+          end
+
+          result = cmd.execute!
+
+          unless result[:exit_code].zero?
+            PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
+            raise PDK::CLI::FatalError, _('Unable to resolve Gemfile dependencies.')
+          end
+
+          true
+        end
+
+        def install!(gem_overrides = {})
+          argv = ['install', "--gemfile=#{gemfile}"]
+          argv << '-j4' unless Gem.win_platform? && Gem::Version.new(PDK::Util::RubyVersion.active_ruby_version) < Gem::Version.new('2.3.5')
+
+          cmd = bundle_command(*argv).tap do |c|
+            c.add_spinner(_('Installing missing Gemfile dependencies.'))
+            c.update_environment(gemfile_env(gem_overrides)) unless gem_overrides.empty?
+          end
+
+          result = cmd.execute!
+
+          unless result[:exit_code].zero?
+            PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
+            raise PDK::CLI::FatalError, _('Unable to install missing Gemfile dependencies.')
+          end
+
+          true
+        end
+
+        def binstubs!(gems)
+          binstub_dir = File.join(File.dirname(gemfile), 'bin')
+          return true if gems.all? { |gem| File.file?(File.join(binstub_dir, gem)) }
+
+          cmd = bundle_command('binstubs', *gems, '--force')
+          result = cmd.execute!
+
+          unless result[:exit_code].zero?
+            PDK.logger.fatal(_("Failed to generate binstubs for '%{gems}':\n%{output}") % { gems: gems.join(' '), output: result.values_at(:stdout, :stderr).join("\n") })
+            raise PDK::CLI::FatalError, _('Unable to install requested binstubs.')
+          end
+
+          true
+        end
+
+        def self.gemfile_env(gem_overrides)
+          gemfile_env = {}
+
+          return gemfile_env unless gem_overrides.respond_to?(:each)
+
+          gem_overrides.each do |gem, version|
+            gemfile_env['PUPPET_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'puppet' && !version.nil?
+            gemfile_env['FACTER_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'facter' && !version.nil?
+            gemfile_env['HIERA_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'hiera' && !version.nil?
+          end
+
+          gemfile_env
+        end
+
+        private
+
+        def gemfile_env(gem_overrides)
+          self.class.gemfile_env(gem_overrides)
+        end
+
+        def bundle_command(*args)
+          PDK::CLI::Exec::Command.new(PDK::CLI::Exec.bundle_bin, *args).tap do |c|
+            c.context = :module
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pdk/util/filesystem.rb

@@ -0,0 +1,12 @@
+module PDK
+  module Util
+    module Filesystem
+      def write_file(path, content)
+        raise ArgumentError unless path.is_a?(String) || path.respond_to?(:to_path)
+
+        File.open(path, 'wb') { |f| f.write(content.encode(universal_newline: true)) }
+      end
+      module_function :write_file
+    end
+  end
+end

data/lib/pdk/util/git.rb

@@ -0,0 +1,74 @@
+module PDK
+  module Util
+    module Git
+      def self.git_bindir
+        @git_dir ||= File.join('private', 'git', Gem.win_platform? ? 'cmd' : 'bin')
+      end
+
+      def self.git_paths
+        @paths ||= begin
+          paths = [File.join(PDK::Util.pdk_package_basedir, git_bindir)]
+
+          if Gem.win_platform?
+            paths << File.join(PDK::Util.pdk_package_basedir, 'private', 'git', 'mingw64', 'bin')
+            paths << File.join(PDK::Util.pdk_package_basedir, 'private', 'git', 'mingw64', 'libexec', 'git-core')
+            paths << File.join(PDK::Util.pdk_package_basedir, 'private', 'git', 'usr', 'bin')
+          end
+
+          paths
+        end
+      end
+
+      def self.git_bin
+        git_bin = Gem.win_platform? ? 'git.exe' : 'git'
+        vendored_bin_path = File.join(git_bindir, git_bin)
+
+        PDK::CLI::Exec.try_vendored_bin(vendored_bin_path, git_bin)
+      end
+
+      def self.git(*args)
+        PDK::CLI::Exec.ensure_bin_present!(git_bin, 'git')
+
+        PDK::CLI::Exec.execute(git_bin, *args)
+      end
+
+      def self.git_with_env(env, *args)
+        PDK::CLI::Exec.ensure_bin_present!(git_bin, 'git')
+
+        PDK::CLI::Exec.execute_with_env(env, git_bin, *args)
+      end
+
+      def self.repo?(maybe_repo)
+        return bare_repo?(maybe_repo) if File.directory?(maybe_repo)
+
+        remote_repo?(maybe_repo)
+      end
+
+      def self.bare_repo?(maybe_repo)
+        env = { 'GIT_DIR' => maybe_repo }
+        rev_parse = git_with_env(env, 'rev-parse', '--is-bare-repository')
+
+        rev_parse[:exit_code].zero? && rev_parse[:stdout].strip == 'true'
+      end
+
+      def self.remote_repo?(maybe_repo)
+        git('ls-remote', '--exit-code', maybe_repo)[:exit_code].zero?
+      end
+
+      def self.ls_remote(repo, ref)
+        output = git('ls-remote', '--refs', repo, ref)
+
+        unless output[:exit_code].zero?
+          PDK.logger.error output[:stdout]
+          PDK.logger.error output[:stderr]
+          raise PDK::CLI::ExitWithError, _('Unable to access the template repository "%{repository}"') % {
+            repository: repo,
+          }
+        end
+
+        matching_refs = output[:stdout].split("\n").map { |r| r.split("\t") }
+        matching_refs.find { |_sha, remote_ref| remote_ref == ref }.first
+      end
+    end
+  end
+end

data/lib/pdk/util/puppet_version.rb

@@ -0,0 +1,242 @@
+require 'pdk/util'
+require 'pdk/util/git'
+
+module PDK
+  module Util
+    class PuppetVersion
+      class << self
+        extend Forwardable
+
+        def_delegators :instance, :puppet_dev_env, :puppet_dev_path, :fetch_puppet_dev, :find_gem_for, :from_pe_version, :from_module_metadata, :latest_available
+
+        attr_writer :instance
+
+        def instance
+          @instance ||= new
+        end
+      end
+
+      PE_VERSIONS_URL = 'https://forgeapi.puppet.com/private/versions/pe'.freeze
+      DEFAULT_PUPPET_DEV_URL = 'https://github.com/puppetlabs/puppet'.freeze
+      DEFAULT_PUPPET_DEV_BRANCH = 'master'.freeze
+
+      def puppet_dev_env
+        {
+          gem_version: 'file://%{path}' % { path: puppet_dev_path },
+          ruby_version: PDK::Util::RubyVersion.latest_ruby_version,
+        }
+      end
+
+      def puppet_dev_path
+        File.join(PDK::Util.cachedir, 'src', 'puppet')
+      end
+
+      def latest_available
+        latest = find_gem(Gem::Requirement.create('>= 0'))
+
+        if latest.nil?
+          raise ArgumentError, _('Unable to find a Puppet gem in current Ruby environment or from Rubygems.org.')
+        end
+
+        latest
+      end
+
+      def fetch_puppet_dev
+        # Check if the source is cloned and is a readable git repo
+        unless PDK::Util::Git.remote_repo? puppet_dev_path
+          # Check if the path has something in it already. Delete it and prepare for clone if so.
+          if File.exist? puppet_dev_path
+            File.delete(puppet_dev_path) if File.file? puppet_dev_path
+            FileUtils.rm_rf(puppet_dev_path) if File.directory? puppet_dev_path
+          end
+
+          FileUtils.mkdir_p puppet_dev_path
+          clone_result = PDK::Util::Git.git('clone', DEFAULT_PUPPET_DEV_URL, puppet_dev_path)
+          return if clone_result[:exit_code].zero?
+
+          PDK.logger.error clone_result[:stdout]
+          PDK.logger.error clone_result[:stderr]
+          raise PDK::CLI::FatalError, _("Unable to clone git repository at '%{repo}'.") % { repo: DEFAULT_PUPPET_DEV_URL }
+        end
+
+        # Fetch Updates from remote repository
+        fetch_result = PDK::Util::Git.git('-C', puppet_dev_path, 'fetch', 'origin')
+
+        unless fetch_result[:exit_code].zero?
+          PDK.logger.error fetch_result[:stdout]
+          PDK.logger.error fetch_result[:stderr]
+          raise PDK::CLI::FatalError, _("Unable to fetch from git remote at '%{repo}'.") % { repo: DEFAULT_PUPPET_DEV_URL }
+        end
+
+        # Reset local repo to latest
+        reset_result = PDK::Util::Git.git('-C', puppet_dev_path, 'reset', '--hard', 'origin/master')
+        return if reset_result[:exit_code].zero?
+
+        PDK.logger.error reset_result[:stdout]
+        PDK.logger.error reset_result[:stderr]
+        raise PDK::CLI::FatalError, _("Unable to update git repository at '%{cachedir}'.") % { repo: puppet_dev_path }
+      end
+
+      def find_gem_for(version_str)
+        version = parse_specified_version(version_str)
+
+        # Look for a gem matching exactly the version passed in.
+        if version.segments.length == 3
+          exact_match_gem = find_gem(Gem::Requirement.create(version))
+          return exact_match_gem unless exact_match_gem.nil?
+        end
+
+        # Construct a pessimistic version constraint to find the latest
+        # available gem matching the level of specificity of version_str.
+        requirement_string = version.approximate_recommendation
+        requirement_string += '.0' unless version.segments.length == 1
+        latest_requirement = Gem::Requirement.create(requirement_string)
+
+        latest_available_gem = find_gem(latest_requirement)
+
+        if latest_available_gem.nil?
+          raise ArgumentError, _('Unable to find a Puppet gem matching %{requirement}.') % {
+            requirement: latest_requirement,
+          }
+        end
+
+        # Only issue this warning if they requested an exact version that isn't available.
+        if version.segments.length == 3
+          PDK.logger.warn(_('Puppet %{requested_version} is not available, activating %{found_version} instead.') % {
+            requested_version: version_str,
+            found_version:     latest_available_gem[:gem_version].version,
+          })
+        end
+
+        latest_available_gem
+      end
+
+      def from_pe_version(version_str)
+        version = parse_specified_version(version_str)
+
+        gem_version = pe_version_map.find do |version_map|
+          version_map[:requirement].satisfied_by?(version)
+        end
+
+        if gem_version.nil?
+          raise ArgumentError, _('Unable to map Puppet Enterprise version %{pe_version} to a Puppet version.') % {
+            pe_version: version_str,
+          }
+        end
+
+        PDK.logger.info _('Puppet Enterprise %{pe_version} maps to Puppet %{puppet_version}.') % {
+          pe_version:     version_str,
+          puppet_version: gem_version[:gem_version],
+        }
+
+        find_gem_for(gem_version[:gem_version])
+      end
+
+      def from_module_metadata(metadata = nil)
+        if metadata.nil?
+          metadata_file = PDK::Util.find_upwards('metadata.json')
+
+          unless metadata_file
+            PDK.logger.warn _('Unable to determine Puppet version for module: no metadata.json present in module.')
+            return nil
+          end
+
+          metadata = PDK::Module::Metadata.from_file(metadata_file)
+        end
+
+        metadata.validate_puppet_version_requirement!
+        metadata_requirement = metadata.puppet_requirement
+
+        # Split combined requirements like ">= 4.7.0 < 6.0.0" into their
+        # component requirements [">= 4.7.0", "< 6.0.0"]
+        pattern = %r{#{Gem::Requirement::PATTERN_RAW}}
+        requirement_strings = metadata_requirement['version_requirement'].scan(pattern).map do |req|
+          req.compact.join(' ')
+        end
+
+        gem_requirement = Gem::Requirement.create(requirement_strings)
+        find_gem(gem_requirement)
+      end
+
+      private
+
+      def parse_specified_version(version_str)
+        Gem::Version.new(version_str)
+      rescue ArgumentError
+        raise ArgumentError, _('%{version} is not a valid version number.') % {
+          version: version_str,
+        }
+      end
+
+      def pe_version_map
+        @pe_version_map ||= fetch_pe_version_map.map { |version_map|
+          maps = version_map['versions'].map do |pe_release|
+            requirements = ["= #{pe_release['version']}"]
+
+            # Some PE release have a .0 Z release, which causes problems when
+            # the user specifies "X.Y" expecting to get the latest Z and
+            # instead getting the oldest.
+            requirements << "!= #{pe_release['version'].gsub(%r{\.\d+\Z}, '')}" if pe_release['version'].end_with?('.0')
+            {
+              requirement: Gem::Requirement.create(requirements),
+              gem_version: pe_release['puppet'],
+            }
+          end
+
+          maps << {
+            requirement: requirement_from_forge_range(version_map['release']),
+            gem_version: version_map['versions'].find { |r| r['version'] == version_map['latest'] }['puppet'],
+          }
+        }.flatten
+      end
+
+      def fetch_pe_version_map
+        map = PDK::Util::VendoredFile.new('pe_versions.json', PE_VERSIONS_URL).read
+
+        JSON.parse(map)
+      rescue PDK::Util::VendoredFile::DownloadError => e
+        raise PDK::CLI::FatalError, e.message
+      rescue JSON::ParserError
+        raise PDK::CLI::FatalError, _('Failed to parse Puppet Enterprise version map file.')
+      end
+
+      def requirement_from_forge_range(range_str)
+        Gem::Requirement.create("~> #{range_str.gsub(%r{\.x\Z}, '.0')}")
+      end
+
+      def rubygems_puppet_versions
+        return @rubygems_puppet_versions unless @rubygems_puppet_versions.nil?
+
+        fetcher = Gem::SpecFetcher.fetcher
+        puppet_tuples = fetcher.detect(:released) do |spec_tuple|
+          spec_tuple.name == 'puppet' && Gem::Platform.match(spec_tuple.platform)
+        end
+        puppet_versions = puppet_tuples.map { |name, _| name.version }.uniq
+        @rubygems_puppet_versions = puppet_versions.sort { |a, b| b <=> a }
+      end
+
+      def find_gem(requirement)
+        if PDK::Util.package_install?
+          find_in_package_cache(requirement)
+        else
+          find_in_rubygems(requirement)
+        end
+      end
+
+      def find_in_rubygems(requirement)
+        version = rubygems_puppet_versions.find { |r| requirement.satisfied_by?(r) }
+        version.nil? ? nil : { gem_version: version, ruby_version: PDK::Util::RubyVersion.default_ruby_version }
+      end
+
+      def find_in_package_cache(requirement)
+        PDK::Util::RubyVersion.versions.each do |ruby_version, _|
+          PDK::Util::RubyVersion.use(ruby_version)
+          version = PDK::Util::RubyVersion.available_puppet_versions.find { |r| requirement.satisfied_by?(r) }
+          return { gem_version: version, ruby_version: ruby_version } unless version.nil?
+        end
+
+        nil
+      end
+    end
+  end
+end