pdk-akerl 1.8.0.1

data/lib/pdk/util/ruby_version.rb

@@ -0,0 +1,147 @@
+require 'pdk/util'
+
+module PDK
+  module Util
+    class RubyVersion
+      class << self
+        extend Forwardable
+
+        def_delegators :instance, :gem_path, :gem_paths_raw, :gem_home, :available_puppet_versions, :bin_path
+
+        attr_reader :instance
+
+        def instance(version = nil)
+          use(version) unless version.nil?
+
+          if @instance.nil?
+            @instance = {}
+            @instance.default_proc = proc do |hash, key|
+              hash[key] = new(key)
+            end
+          end
+          @instance[active_ruby_version]
+        end
+
+        def active_ruby_version
+          @active_ruby_version || default_ruby_version
+        end
+
+        def use(version)
+          if versions.key?(version)
+            @active_ruby_version = version
+          else
+            raise ArgumentError, _('Unknown Ruby version "%{ruby_version}"') % {
+              ruby_version: version,
+            }
+          end
+        end
+
+        def scan_for_packaged_rubies
+          ruby_basedir = File.join(PDK::Util.pdk_package_basedir, 'private', 'ruby', '*')
+          Dir[ruby_basedir].sort.map { |ruby_dir|
+            version = File.basename(ruby_dir)
+            [version, version.split('.').take(2).concat(['0']).join('.')]
+          }.reverse.to_h
+        end
+
+        def default_ruby_version
+          # For now, the packaged versions will be using default of 2.4.4.
+          return '2.4.4' if PDK::Util.package_install?
+
+          # TODO: may not be a safe assumption that highest available version should be default
+          latest_ruby_version
+        end
+
+        def latest_ruby_version
+          versions.keys.sort { |a, b| Gem::Version.new(b) <=> Gem::Version.new(a) }.first
+        end
+
+        def versions
+          @versions ||= if PDK::Util.package_install?
+                          scan_for_packaged_rubies
+                        else
+                          { RbConfig::CONFIG['RUBY_PROGRAM_VERSION'] => RbConfig::CONFIG['ruby_version'] }
+                        end
+        end
+      end
+
+      attr_reader :ruby_version
+
+      def initialize(ruby_version = nil)
+        @ruby_version = ruby_version || default_ruby_version
+      end
+
+      def bin_path
+        if PDK::Util.package_install?
+          File.join(PDK::Util.pdk_package_basedir, 'private', 'ruby', ruby_version, 'bin')
+        else
+          RbConfig::CONFIG['bindir']
+        end
+      end
+
+      def gem_paths_raw
+        if PDK::Util.package_install?
+          # Subprocesses use their own set of gems which are managed by pdk or
+          # installed with the package. We also include the separate gem path
+          # where our packaged multi-puppet installations live.
+          [
+            File.join(PDK::Util.pdk_package_basedir, 'private', 'ruby', ruby_version, 'lib', 'ruby', 'gems', versions[ruby_version]),
+            File.join(PDK::Util.package_cachedir, 'ruby', versions[ruby_version]),
+            File.join(PDK::Util.pdk_package_basedir, 'private', 'puppet', 'ruby', versions[ruby_version]),
+          ]
+        else
+          # This allows the subprocess to find the 'bundler' gem, which isn't
+          # in GEM_HOME for gem installs.
+          # TODO: There must be a better way to do this than shelling out to
+          # gem... Perhaps can be replaced with "Gem.path"?
+          [File.absolute_path(File.join(`gem which bundler`, '..', '..', '..', '..'))]
+        end
+      end
+
+      def gem_path
+        gem_paths_raw.join(File::PATH_SEPARATOR)
+      end
+
+      def gem_home
+        # `bundle install --path` ignores all "system" installed gems and
+        # causes unnecessary package installs. `bundle install` (without
+        # --path) installs into GEM_HOME, which by default is non-user
+        # writeable.
+        # To still use the pre-installed packages, but allow folks to install
+        # additional gems, we set GEM_HOME to the user's cachedir and put all
+        # other cache locations onto GEM_PATH.
+        # See https://stackoverflow.com/a/11277228 for background
+        File.join(PDK::Util.cachedir, 'ruby', versions[ruby_version])
+      end
+
+      def available_puppet_versions
+        return @available_puppet_versions unless @available_puppet_versions.nil?
+
+        puppet_spec_files = Dir[File.join(gem_home, 'specifications', '**', 'puppet*.gemspec')]
+
+        gem_path.split(File::PATH_SEPARATOR).each do |path|
+          puppet_spec_files += Dir[File.join(path, 'specifications', '**', 'puppet*.gemspec')]
+        end
+
+        puppet_specs = []
+
+        puppet_spec_files.each do |specfile|
+          spec = Gem::Specification.load(specfile)
+          puppet_specs << spec if spec.name == 'puppet'
+        end
+
+        @available_puppet_versions = puppet_specs.map(&:version).sort { |a, b| b <=> a }
+      end
+
+      private
+
+      def default_ruby_version
+        self.class.default_ruby_version
+      end
+
+      def versions
+        self.class.versions
+      end
+    end
+  end
+end

data/lib/pdk/util/vendored_file.rb

@@ -0,0 +1,88 @@
+require 'pdk/util'
+require 'net/https'
+require 'openssl'
+require 'fileutils'
+require 'pdk/util/filesystem'
+
+module PDK
+  module Util
+    class VendoredFile
+      class DownloadError < StandardError; end
+
+      HTTP_ERRORS = [
+        EOFError,
+        Errno::ECONNRESET,
+        Errno::EINVAL,
+        Errno::ECONNREFUSED,
+        Net::HTTPBadResponse,
+        Net::HTTPHeaderSyntaxError,
+        Net::ProtocolError,
+        Timeout::Error,
+      ].freeze
+
+      attr_reader :file_name
+      attr_reader :url
+
+      include PDK::Util::Filesystem
+
+      def initialize(file_name, url)
+        @file_name = file_name
+        @url = url
+      end
+
+      def read
+        return File.read(package_vendored_path) if PDK::Util.package_install?
+        return File.read(gem_vendored_path) if File.file?(gem_vendored_path)
+
+        content = download_file
+
+        # TODO: should only write if it's valid JSON
+        # TODO: need a way to invalidate if out of date
+        FileUtils.mkdir_p(File.dirname(gem_vendored_path))
+        write_file(gem_vendored_path, content)
+        content
+      end
+
+      private
+
+      def download_file
+        PDK.logger.debug _('%{file_name} was not found in the cache, downloading it from %{url}.') % {
+          file_name: file_name,
+          url:       url,
+        }
+
+        uri = URI.parse(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        # TODO: Get rid of this, possible workaround:
+        # https://github.com/glennsarti/dev-tools/blob/master/RubyCerts.ps1
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE if Gem.win_platform?
+        request = Net::HTTP::Get.new(uri.request_uri)
+        response = http.request(request)
+
+        unless response.code == '200'
+          raise DownloadError, _('Unable to download %{url}. %{code}: %{message}.') % {
+            url:     url,
+            code:    response.code,
+            message: response.message,
+          }
+        end
+
+        response.body
+      rescue *HTTP_ERRORS => e
+        raise DownloadError, _('Unable to download %{url}. Check internet connectivity and try again. %{error}') % {
+          url: url,
+          error: e,
+        }
+      end
+
+      def package_vendored_path
+        @package_vendored_path ||= File.join(PDK::Util.package_cachedir, file_name)
+      end
+
+      def gem_vendored_path
+        @gem_vendored_path ||= File.join(PDK::Util.cachedir, PDK::VERSION, file_name)
+      end
+    end
+  end
+end

data/lib/pdk/util/version.rb

@@ -0,0 +1,42 @@
+require 'pdk/version'
+require 'pdk/cli/exec'
+require 'pdk/util/git'
+require 'pdk/logger'
+
+module PDK
+  module Util
+    module Version
+      def self.version_string
+        "#{PDK::VERSION} #{pdk_ref}".strip.freeze
+      end
+
+      def self.pdk_ref
+        ref = "#{pkg_sha} #{git_ref}".strip
+        ref.empty? ? nil : "(#{ref})"
+      end
+
+      def self.pkg_sha
+        if version_file && File.exist?(version_file)
+          ver = File.read(version_file)
+          sha = ver.strip.split('.')[5] unless ver.nil?
+        end
+
+        sha
+      end
+
+      def self.git_ref
+        source_git_dir = File.join(File.expand_path('../../..', File.dirname(__FILE__)), '.git')
+
+        return nil unless File.directory?(source_git_dir)
+
+        ref_result = PDK::Util::Git.git('--git-dir', source_git_dir, 'describe', '--all', '--long')
+        return ref_result[:stdout].strip if ref_result[:exit_code].zero?
+      end
+
+      def self.version_file
+        # FIXME: this gets called a LOT and doesn't currently get cached
+        PDK::Util.find_upwards('PDK_VERSION', File.dirname(__FILE__))
+      end
+    end
+  end
+end

data/lib/pdk/util/windows.rb

@@ -0,0 +1,13 @@
+module PDK
+  module Util
+    module Windows
+      module File; end
+
+      if Gem.win_platform?
+        require 'pdk/util/windows/api_types'
+        require 'pdk/util/windows/string'
+        require 'pdk/util/windows/file'
+      end
+    end
+  end
+end

data/lib/pdk/util/windows/api_types.rb

@@ -0,0 +1,57 @@
+require 'ffi'
+require 'pdk/util/windows/string'
+
+module PDK::Util::Windows::APITypes
+  module ::FFI
+    WIN32_FALSE = 0
+
+    # standard Win32 error codes
+    ERROR_SUCCESS = 0
+  end
+
+  class ::FFI::Pointer
+    def self.from_string_to_wide_string(str, &_block)
+      str = PDK::Util::Windows::String.wide_string(str)
+      FFI::MemoryPointer.new(:byte, str.bytesize) do |ptr|
+        # uchar here is synonymous with byte
+        ptr.put_array_of_uchar(0, str.bytes.to_a)
+
+        yield ptr
+      end
+
+      # ptr has already had free called, so nothing to return
+      nil
+    end
+
+    def read_wide_string(char_length, dst_encoding = Encoding::UTF_8, encode_options = {})
+      # char_length is number of wide chars (typically excluding NULLs), *not* bytes
+      str = get_bytes(0, char_length * 2).force_encoding('UTF-16LE')
+      str.encode(dst_encoding, str.encoding, encode_options)
+    rescue StandardError => e
+      PDK.logger.debug _('Unable to convert value %{string} to encoding %{encoding} due to %{error}') % {
+        string:   str.dump,
+        encoding: dst_encoding,
+        error:    e.inspect,
+      }
+      raise
+    end
+  end
+
+  # FFI Types
+  # https://github.com/ffi/ffi/wiki/Types
+
+  # Windows - Common Data Types
+  # https://msdn.microsoft.com/en-us/library/cc230309.aspx
+
+  # Windows Data Types
+  # https://msdn.microsoft.com/en-us/library/windows/desktop/aa383751(v=vs.85).aspx
+
+  FFI.typedef :uint32, :dword
+  # buffer_inout is similar to pointer (platform specific), but optimized for buffers
+  FFI.typedef :buffer_inout, :lpwstr
+  # buffer_in is similar to pointer (platform specific), but optimized for CONST read only buffers
+  FFI.typedef :buffer_in, :lpcwstr
+  # 8 bits per byte
+  FFI.typedef :uchar, :byte
+  FFI.typedef :uint16, :wchar
+end

data/lib/pdk/util/windows/file.rb

@@ -0,0 +1,36 @@
+require 'pdk/util/windows'
+
+module PDK::Util::Windows::File
+  require 'ffi'
+  extend FFI::Library
+  extend PDK::Util::Windows::String
+
+  def get_long_pathname(path)
+    converted = ''
+    FFI::Pointer.from_string_to_wide_string(path) do |path_ptr|
+      # includes terminating NULL
+      buffer_size = GetLongPathNameW(path_ptr, FFI::Pointer::NULL, 0)
+      FFI::MemoryPointer.new(:wchar, buffer_size) do |converted_ptr|
+        if GetLongPathNameW(path_ptr, converted_ptr, buffer_size) == FFI::WIN32_FALSE
+          raise _('Failed to call GetLongPathName')
+        end
+
+        converted = converted_ptr.read_wide_string(buffer_size - 1)
+      end
+    end
+
+    converted
+  end
+  module_function :get_long_pathname
+
+  ffi_convention :stdcall
+
+  # https://msdn.microsoft.com/en-us/library/windows/desktop/aa364980(v=vs.85).aspx
+  # DWORD WINAPI GetLongPathName(
+  #   _In_  LPCTSTR lpszShortPath,
+  #   _Out_ LPTSTR  lpszLongPath,
+  #   _In_  DWORD   cchBuffer
+  # );
+  ffi_lib :kernel32
+  attach_function :GetLongPathNameW, [:lpcwstr, :lpwstr, :dword], :dword
+end

data/lib/pdk/util/windows/string.rb

@@ -0,0 +1,16 @@
+require 'pdk/util/windows'
+
+module PDK::Util::Windows::String
+  def wide_string(str)
+    # if given a nil string, assume caller wants to pass a nil pointer to win32
+    return nil if str.nil?
+    # ruby (< 2.1) does not respect multibyte terminators, so it is possible
+    # for a string to contain a single trailing null byte, followed by garbage
+    # causing buffer overruns.
+    #
+    # See http://svn.ruby-lang.org/cgi-bin/viewvc.cgi?revision=41920&view=revision
+    newstr = str + "\0".encode(str.encoding)
+    newstr.encode!('UTF-16LE')
+  end
+  module_function :wide_string
+end

data/lib/pdk/validate.rb

@@ -0,0 +1,14 @@
+require 'pdk/validate/metadata_validator'
+require 'pdk/validate/puppet_validator'
+require 'pdk/validate/ruby_validator'
+require 'pdk/validate/tasks_validator'
+
+module PDK
+  module Validate
+    def self.validators
+      @validators ||= [MetadataValidator, PuppetValidator, RubyValidator, TasksValidator].freeze
+    end
+
+    class ParseOutputError < StandardError; end
+  end
+end

data/lib/pdk/validate/base_validator.rb

@@ -0,0 +1,209 @@
+require 'pdk'
+require 'pdk/cli/exec'
+require 'pdk/module'
+
+module PDK
+  module Validate
+    class BaseValidator
+      # Controls how many times the validator is invoked.
+      #
+      #   :once -       The validator will be invoked once and passed all the
+      #                 targets.
+      #   :per_target - The validator will be invoked for each target
+      #                 separately.
+      INVOKE_STYLE = :once
+
+      # Controls how the validator behaves if not passed any targets.
+      #
+      #   true  - PDK will not pass the globbed targets to the validator command
+      #           and it will instead rely on the underlying tool to find its
+      #           own default targets.
+      #   false - PDK will pass the globbed targets to the validator command.
+      ALLOW_EMPTY_TARGETS = false
+
+      def self.cmd_path
+        File.join(PDK::Util.module_root, 'bin', cmd)
+      end
+
+      # Parses the target strings provided from the CLI
+      #
+      # @param options [Hash] A Hash containing the input options from the CLI.
+      #
+      # @return targets [Array] An Array of Strings containing target file paths
+      #                         for the validator to validate.
+      # @return skipped [Array] An Array of Strings containing targets
+      #                         that are skipped due to target not containing
+      #                         any files that can be validated by the validator.
+      # @return invalid [Array] An Array of Strings containing targets that do
+      #                         not exist, and will not be run by validator.
+      def self.parse_targets(options)
+        # If no targets are specified, then we will run validations from the
+        # base module directory.
+
+        targets = options.fetch(:targets, []).empty? ? [PDK::Util.module_root] : options[:targets]
+
+        targets.map! { |r| r.gsub(File::ALT_SEPARATOR, File::SEPARATOR) } if File::ALT_SEPARATOR
+        skipped = []
+        invalid = []
+        matched = targets.map { |target|
+          if respond_to?(:pattern)
+            if File.directory?(target)
+              target_root = PDK::Util.module_root
+              pattern_glob = Array(pattern).map { |p| Dir.glob(File.join(target_root, p)) }
+
+              target_list = pattern_glob.flatten.map do |file|
+                if File.fnmatch(File.join(File.expand_path(target), '*'), file)
+                  Pathname.new(file).relative_path_from(Pathname.new(PDK::Util.module_root)).to_s
+                end
+              end
+
+              ignore_list = ignore_pathspec
+              target_list = target_list.reject { |file| ignore_list.match(file) }
+
+              skipped << target if target_list.flatten.empty?
+              target_list
+            elsif File.file?(target)
+              if Array(pattern).include? target
+                target
+              elsif Array(pattern).any? { |p| File.fnmatch(File.expand_path(p), File.expand_path(target)) }
+                target
+              else
+                skipped << target
+                next
+              end
+            else
+              invalid << target
+              next
+            end
+          else
+            target
+          end
+        }.compact.flatten
+        [matched, skipped, invalid]
+      end
+
+      def self.ignore_pathspec
+        ignore_pathspec = PDK::Module.default_ignored_pathspec.dup
+
+        if respond_to?(:pattern_ignore)
+          Array(pattern_ignore).each do |pattern|
+            ignore_pathspec.add(pattern)
+          end
+        end
+
+        ignore_pathspec
+      end
+
+      def self.parse_options(_options, targets)
+        targets
+      end
+
+      def self.spinner_text(_targets = nil)
+        _('Invoking %{cmd}') % { cmd: cmd }
+      end
+
+      def self.process_skipped(report, skipped = [])
+        skipped.each do |skipped_target|
+          PDK.logger.debug(_('%{validator}: Skipped \'%{target}\'. Target does not contain any files to validate (%{pattern}).') % { validator: name, target: skipped_target, pattern: pattern })
+          report.add_event(
+            file:     skipped_target,
+            source:   name,
+            message:  _('Target does not contain any files to validate (%{pattern}).') % { pattern: pattern },
+            severity: :info,
+            state:    :skipped,
+          )
+        end
+      end
+
+      def self.process_invalid(report, invalid = [])
+        invalid.each do |invalid_target|
+          PDK.logger.debug(_('%{validator}: Skipped \'%{target}\'. Target file not found.') % { validator: name, target: invalid_target })
+          report.add_event(
+            file:     invalid_target,
+            source:   name,
+            message:  _('File does not exist.'),
+            severity: :error,
+            state:    :error,
+          )
+        end
+      end
+
+      def self.allow_empty_targets?
+        self::ALLOW_EMPTY_TARGETS == true
+      end
+
+      def self.invoke(report, options = {})
+        targets, skipped, invalid = parse_targets(options)
+
+        process_skipped(report, skipped)
+        process_invalid(report, invalid)
+
+        return 0 if targets.empty?
+
+        PDK::Util::Bundler.ensure_binstubs!(cmd)
+
+        # If invoking :per_target, split the targets array into an array of
+        # single element arrays (one per target). If invoking :once, wrap the
+        # targets array in another array. This is so we can loop through the
+        # invokes with the same logic, regardless of which invoke style is
+        # needed.
+        #
+        if self::INVOKE_STYLE == :per_target
+          targets = targets.combination(1).to_a
+        else
+          targets = targets.each_slice(1000).to_a
+          options[:split_exec] = PDK::CLI::ExecGroup.new(spinner_text(targets), parallel: false)
+        end
+
+        if options.fetch(:targets, []).empty? && allow_empty_targets?
+          targets = [[]]
+        end
+
+        exit_codes = []
+
+        targets.each do |invokation_targets|
+          cmd_argv = parse_options(options, invokation_targets).unshift(cmd_path).compact
+          cmd_argv.unshift(File.join(PDK::Util::RubyVersion.bin_path, 'ruby.exe'), '-W0') if Gem.win_platform?
+
+          command = PDK::CLI::Exec::Command.new(*cmd_argv).tap do |c|
+            c.context = :module
+            c.environment = { 'PUPPET_GEM_VERSION' => options[:puppet] } if options[:puppet]
+            unless options[:split_exec]
+              exec_group = options[:exec_group]
+              if exec_group
+                sub_spinner = exec_group.add_spinner(spinner_text(invokation_targets))
+                c.register_spinner(sub_spinner)
+              else
+                c.add_spinner(spinner_text(invokation_targets))
+              end
+            end
+          end
+
+          if options[:split_exec]
+            options[:split_exec].register do
+              result = command.execute!
+
+              begin
+                parse_output(report, result, invokation_targets.compact)
+              rescue PDK::Validate::ParseOutputError => e
+                $stderr.puts e.message
+              end
+              result[:exit_code]
+            end
+          else
+            result = command.execute!
+            exit_codes << result[:exit_code]
+
+            begin
+              parse_output(report, result, invokation_targets.compact)
+            rescue PDK::Validate::ParseOutputError => e
+              $stderr.puts e.message
+            end
+          end
+        end
+
+        options.key?(:split_exec) ? options[:split_exec].exit_code : exit_codes.max
+      end
+    end
+  end
+end