pdk 1.9.0 → 3.2.0

data/lib/pdk/util/env.rb

@@ -0,0 +1,48 @@
+require 'pdk'
+require 'forwardable'
+
+module PDK
+  module Util
+    class Env
+      class WindowsEnv
+        extend Forwardable
+
+        # Note, these delegators may not have case insensitive keys
+        def_delegators :env_hash, :fetch, :select, :reject
+
+        def []=(key, value)
+          PDK::Util::Windows::Process.set_environment_variable(key, value)
+        end
+
+        def key?(key)
+          !env_hash.keys.find { |item| key.casecmp(item).zero? }.nil?
+        end
+
+        def [](key)
+          env_hash.each do |item, value|
+            next unless key.casecmp(item).zero?
+
+            return value
+          end
+          nil
+        end
+
+        private
+
+        def env_hash
+          PDK::Util::Windows::Process.environment_hash
+        end
+      end
+
+      class << self
+        extend Forwardable
+
+        def_delegators :implementation, :key?, :[], :[]=, :fetch, :select, :reject
+
+        def implementation
+          @implementation ||= Gem.win_platform? ? WindowsEnv.new : ENV
+        end
+      end
+    end
+  end
+end

data/lib/pdk/util/filesystem.rb

@@ -1,12 +1,149 @@
+require 'pdk'
+autoload :FileUtils, 'fileutils'
+
 module PDK
   module Util
     module Filesystem
       def write_file(path, content)
-        raise ArgumentError unless path.is_a?(String) || path.respond_to?(:to_path)
+        raise ArgumentError, 'content must be a String' unless content.is_a?(String)
+        raise ArgumentError, 'path must be a String or Pathname' unless path.is_a?(String) || path.respond_to?(:to_path)
+
+        # Harmonize newlines across platforms.
+        content = content.encode(universal_newline: true)
+
+        # Make sure all written files have a trailing newline.
+        content += "\n" unless content[-1] == "\n"
 
-        File.open(path, 'wb') { |f| f.write(content.encode(universal_newline: true)) }
+        File.binwrite(path, content)
       end
       module_function :write_file
+
+      def read_file(file, nil_on_error: false, open_args: 'r')
+        File.read(file, open_args: Array(open_args))
+      rescue StandardError => e
+        raise e unless nil_on_error
+
+        nil
+      end
+      module_function :read_file
+
+      def make_executable(file)
+        FileUtils.chmod('a+x', file)
+      end
+      module_function :make_executable
+
+      # :nocov:
+      # These methods just wrap core Ruby functionality and
+      # can be ignored for code coverage
+      def directory?(*args)
+        File.directory?(*args)
+      end
+      module_function :directory?
+
+      def mkdir_p(*args, **kwargs)
+        FileUtils.mkdir_p(*args, **kwargs)
+      end
+      module_function :mkdir_p
+
+      def file?(*args)
+        File.file?(*args)
+      end
+      module_function :file?
+
+      def expand_path(*args)
+        File.expand_path(*args)
+      end
+      module_function :expand_path
+
+      def glob(*args)
+        Dir.glob(*args)
+      end
+      module_function :glob
+
+      def fnmatch(*args)
+        File.fnmatch(*args)
+      end
+      module_function :fnmatch
+
+      def fnmatch?(*args)
+        File.fnmatch?(*args)
+      end
+      module_function :fnmatch?
+
+      def readable?(*args)
+        File.readable?(*args)
+      end
+      module_function :readable?
+
+      def exist?(*args)
+        File.exist?(*args)
+      end
+      module_function :exist?
+
+      def rm(*args, **kwargs)
+        FileUtils.rm(*args, **kwargs)
+      end
+      module_function :rm
+
+      def rm_f(*args, **kwargs)
+        FileUtils.rm_f(*args, **kwargs)
+      end
+      module_function :rm_f
+
+      def rm_rf(*args, **kwargs)
+        FileUtils.rm_rf(*args, **kwargs)
+      end
+      module_function :rm_rf
+
+      def remove_entry_secure(*args)
+        FileUtils.remove_entry_secure(*args)
+      end
+      module_function :remove_entry_secure
+
+      def zero?(*args)
+        File.empty?(*args)
+      end
+      module_function :zero?
+
+      def stat(*args)
+        File.stat(*args)
+      end
+      module_function :stat
+
+      def symlink?(*args)
+        File.symlink?(*args)
+      end
+      module_function :symlink?
+
+      def cp(*args, **kwargs)
+        FileUtils.cp(*args, **kwargs)
+      end
+      module_function :cp
+
+      def mv(*args, **kwargs)
+        FileUtils.mv(*args, **kwargs)
+      rescue Errno::ENOENT
+        # PDK-1169 - FileUtils.mv raises Errno::ENOENT when moving files inside
+        #            VMWare shared folders on Windows. So we need to catch this
+        #            case, check if the file exists to see if the exception is
+        #            legit and "move" the file with cp & rm.
+        src, dest, opts = args
+        raise unless File.exist?(src)
+
+        FileUtils.cp(src, dest, preserve: true)
+        if (opts ||= {})[:secure]
+          FileUtils.remove_entry_secure(src, opts[:force])
+        else
+          FileUtils.remove_entry(src, opts[:force])
+        end
+      end
+      module_function :mv
+
+      def executable?(*args)
+        File.executable?(*args)
+      end
+      module_function :executable?
+      # :nocov:
     end
   end
 end

data/lib/pdk/util/git.rb

@@ -1,6 +1,23 @@
+require 'pdk'
+
 module PDK
   module Util
+    class GitError < StandardError
+      attr_reader :stdout, :stderr, :exit_code, :args
+
+      def initialze(args, result)
+        @args = args
+        @stdout = result[:stdout]
+        @stderr = result[:stderr]
+        @exit_code = result[:exit_code]
+
+        super(format('Git command failed: git %{args}', args: args.join(' ')))
+      end
+    end
+
     module Git
+      GIT_QUERY_CACHE_TTL = 10
+
       def self.git_bindir
         @git_dir ||= File.join('private', 'git', Gem.win_platform? ? 'cmd' : 'bin')
       end
@@ -20,6 +37,8 @@ module PDK
       end
 
       def self.git_bin
+        require 'pdk/cli/exec'
+
         git_bin = Gem.win_platform? ? 'git.exe' : 'git'
         vendored_bin_path = File.join(git_bindir, git_bin)
 
@@ -27,21 +46,33 @@ module PDK
       end
 
       def self.git(*args)
+        require 'pdk/cli/exec'
+
         PDK::CLI::Exec.ensure_bin_present!(git_bin, 'git')
 
         PDK::CLI::Exec.execute(git_bin, *args)
       end
 
       def self.git_with_env(env, *args)
+        require 'pdk/cli/exec'
+
         PDK::CLI::Exec.ensure_bin_present!(git_bin, 'git')
 
         PDK::CLI::Exec.execute_with_env(env, git_bin, *args)
       end
 
       def self.repo?(maybe_repo)
-        return bare_repo?(maybe_repo) if File.directory?(maybe_repo)
-
-        remote_repo?(maybe_repo)
+        result = cached_git_query(maybe_repo, :repo?)
+        return result unless result.nil?
+
+        result = if PDK::Util::Filesystem.directory?(maybe_repo)
+                   # Use boolean shortcircuiting here. The mostly likely type of git repo
+                   # is a "normal" repo with a working tree. Bare repos do not have work tree
+                   work_tree?(maybe_repo) || bare_repo?(maybe_repo)
+                 else
+                   remote_repo?(maybe_repo)
+                 end
+        cache_query_result(maybe_repo, :repo?, result)
       end
 
       def self.bare_repo?(maybe_repo)
@@ -55,20 +86,89 @@ module PDK
         git('ls-remote', '--exit-code', maybe_repo)[:exit_code].zero?
       end
 
+      def self.work_tree?(path)
+        return false unless PDK::Util::Filesystem.directory?(path)
+
+        result = cached_git_query(path, :work_tree?)
+        return result unless result.nil?
+
+        Dir.chdir(path) do
+          rev_parse = git('rev-parse', '--is-inside-work-tree')
+          cache_query_result(path, :work_tree?, rev_parse[:exit_code].zero? && rev_parse[:stdout].strip == 'true')
+        end
+      end
+
+      def self.work_dir_clean?(repo)
+        raise PDK::CLI::ExitWithError, format('Unable to locate git work dir "%{workdir}"', workdir: repo) unless PDK::Util::Filesystem.directory?(repo)
+        raise PDK::CLI::ExitWithError, format('Unable to locate git dir "%{gitdir}"', gitdir: repo) unless PDK::Util::Filesystem.directory?(File.join(repo, '.git'))
+
+        git('--work-tree', repo, '--git-dir', File.join(repo, '.git'), 'status', '--untracked-files=no', '--porcelain', repo)[:stdout].empty?
+      end
+
       def self.ls_remote(repo, ref)
+        repo = "file://#{repo}" if PDK::Util::Filesystem.directory?(repo)
+
         output = git('ls-remote', '--refs', repo, ref)
 
         unless output[:exit_code].zero?
           PDK.logger.error output[:stdout]
           PDK.logger.error output[:stderr]
-          raise PDK::CLI::ExitWithError, _('Unable to access the template repository "%{repository}"') % {
-            repository: repo,
-          }
+          raise PDK::CLI::ExitWithError, format('Unable to access the template repository "%{repository}"', repository: repo)
+        end
+
+        matching_refs = output[:stdout].split(/\r?\n/).map { |r| r.split("\t") }
+        matching_ref = matching_refs.find { |_sha, remote_ref| ["refs/tags/#{ref}", "refs/remotes/origin/#{ref}", "refs/heads/#{ref}"].include?(remote_ref) }
+        raise PDK::CLI::ExitWithError, format('Unable to find a branch or tag named "%{ref}" in %{repo}', ref: ref, repo: repo) if matching_ref.nil?
+
+        matching_ref.first
+      end
+
+      def self.describe(path, ref = nil)
+        args = ['--git-dir', path, 'describe', '--all', '--long', '--always', ref].compact
+        result = git(*args)
+        raise PDK::Util::GitError, args, result unless result[:exit_code].zero?
+
+        result[:stdout].strip
+      end
+
+      def self.tag?(git_remote, tag_name)
+        git('ls-remote', '--tags', '--exit-code', git_remote, tag_name)[:exit_code].zero?
+      end
+
+      # Clears any cached information for git queries
+      # Should only be used during testing
+      # @api private
+      def self.clear_cached_information
+        @git_repo_expire_cache = nil
+        @git_repo_cache = nil
+      end
+
+      def self.cached_git_query(repo, query)
+        # TODO: Not thread safe
+        if @git_repo_expire_cache.nil? || Time.now > @git_repo_expire_cache
+          @git_repo_expire_cache = Time.now + GIT_QUERY_CACHE_TTL # Expire the cache every GIT_QUERY_CACHE_TTL seconds
+          @git_repo_cache = {}
         end
+        return nil if @git_repo_cache[repo].nil?
 
-        matching_refs = output[:stdout].split("\n").map { |r| r.split("\t") }
-        matching_refs.find { |_sha, remote_ref| remote_ref == ref }.first
+        @git_repo_cache[repo][query]
+      end
+      private_class_method :cached_git_query
+
+      def self.cache_query_result(repo, query, result)
+        # TODO: Not thread safe?
+        if @git_repo_expire_cache.nil?
+          @git_repo_expire_cache = Time.now + GIT_QUERY_CACHE_TTL
+          @git_repo_cache = {}
+        end
+        if @git_repo_cache[repo].nil?
+          @git_repo_cache[repo] = { query => result }
+        else
+          @git_repo_cache[repo][query] = result
+        end
+        result
       end
+      private_class_method :cache_query_result
     end
   end
 end

data/lib/pdk/util/json_finder.rb

@@ -0,0 +1,86 @@
+module PDK
+  module Util
+    # Processes a given string, looking for JSON objects and parsing them.
+    #
+    # @example A string with a JSON object and some junk characters
+    #   PDK::Util::JSONFinder.new('foo{"bar":1}').objects
+    #   => [{ 'bar' => 1 }]
+    #
+    # @example A string with mulitple JSON objects
+    #   PDK::Util::JSONFinder.new('foo{"bar":1}baz{"gronk":2}').objects
+    #   => [{ 'bar' => 1 }, { 'gronk' => 2 }]
+    class JSONFinder
+      # Creates a new instance of PDK::Util::JSONFinder.
+      #
+      # @param string [String] the string to find JSON objects inside of.
+      #
+      # @return [PDK::Util::JSONFinder] a new PDK::Util::JSONFinder object.
+      def initialize(string)
+        require 'strscan'
+
+        @scanner = StringScanner.new(string)
+      end
+
+      # Returns the parsed JSON objects from the string.
+      #
+      # @return [Array[Hash]] the parsed JSON objects present in the string.
+      def objects
+        return @objects unless @objects.nil?
+
+        require 'json'
+
+        until @scanner.eos?
+          @scanner.getch until @scanner.peek(1) == '{' || @scanner.eos?
+
+          (@objects ||= []) << begin
+            JSON.parse(read_object(true) || '')
+          rescue JSON::ParserError
+            nil
+          end
+        end
+
+        return [] if @objects.nil?
+
+        @objects = @objects.compact
+      end
+
+      private
+
+      # Recursively process the string to extract a complete JSON object.
+      #
+      # @param new_object [Boolean] Set to true if processing a new object to
+      #   capture the opening brace. Set to false if being called recursively
+      #   where the opening brace has already been captured.
+      #
+      # @return [String] The matched substring containing a JSON object.
+      def read_object(new_object = false)
+        matched_text = new_object ? @scanner.getch : ''
+
+        until @scanner.eos?
+          text = @scanner.scan_until(/(?:(?<!\\)"|\{|\})/)
+          unless text
+            @scanner.terminate
+            return nil
+          end
+          matched_text += text
+
+          case @scanner.matched
+          when '}'
+            break
+          when '"'
+            text = @scanner.scan_until(/(?<!\\)"/)
+            unless text
+              @scanner.terminate
+              return nil
+            end
+            matched_text += text
+          else
+            matched_text += read_object
+          end
+        end
+
+        matched_text
+      end
+    end
+  end
+end

data/lib/pdk/util/puppet_strings.rb

@@ -0,0 +1,125 @@
+require 'pdk'
+
+module PDK
+  module Util
+    module PuppetStrings
+      class NoObjectError < StandardError; end
+      class RunError < StandardError; end
+      class NoGeneratorError < StandardError; end
+
+      # Runs Puppet for the purposes of generating puppet-strings output.
+      #
+      # @param *args [String] additional parameters to pass to puppet.
+      #
+      # @return [Hash{Symbol=>Object}] the result of the command execution.
+      def self.puppet(*args)
+        require 'pdk/cli/exec/command'
+
+        PDK::Util::Bundler.ensure_binstubs!('puppet')
+
+        argv = [File.join(PDK::Util.module_root, 'bin', 'puppet')] + args
+        argv.unshift(File.join(PDK::Util::RubyVersion.bin_path, 'ruby.exe')) if Gem.win_platform?
+
+        command = PDK::CLI::Exec::Command.new(*argv).tap do |c|
+          c.context = :module
+          c.add_spinner('Examining module contents')
+        end
+
+        command.execute!
+      end
+
+      # Generates and parses the JSON output from puppet-strings.
+      #
+      # @returns [Hash{String=>Object}] the parsed puppet-strings output.
+      #
+      # @raises [PDK::Util::PuppetStrings::RunError] if the puppet-strings
+      #   command exits non-zero.
+      # @raises [PDK::Util::PuppetStrings::RunError] if the puppet-strings
+      #   command outputs invalid JSON.
+      def self.generate_hash
+        result = puppet('strings', 'generate', '--format', 'json')
+
+        raise RunError, result[:stderr] unless result[:exit_code].zero?
+
+        JSON.parse(result[:stdout])
+      rescue JSON::ParserError => e
+        PDK.logger.debug(e)
+        raise RunError, 'Unable to parse puppet-strings output'
+      end
+
+      # Searches the puppet-strings result to find the definition of the named
+      # object.
+      #
+      # @param name [String] the name of the object definition to search for.
+      #   If the object name is not prepended with the module name,
+      #   "#{module_name}::#{object_name}" will also be search for.
+      #
+      # @returns [Array[PDK::Generate::PuppetObject, Hash{String=>Object}]] the
+      #   appropriate generator class for the object as the first element and
+      #   the puppet-strings description hash for the definition.
+      #
+      # @raises [PDK::Util::PuppetStrings::NoObjectError] if the named object
+      #   can not be found in the puppet-strings output.
+      # @raises [PDK::Util::PuppetStrings::NoGeneratorError] if the named
+      #   object does not have a corresponding PDK generator class.
+      def self.find_object(name)
+        module_name = PDK::Util.module_metadata['name'].rpartition('-').last
+
+        object_names = [name]
+        object_names << "#{module_name}::#{name}" unless name.start_with?("#{module_name}::")
+
+        known_objects = generate_hash
+        object_type = known_objects.keys.find do |obj_type|
+          known_objects[obj_type].any? { |obj| object_names.include?(obj['name']) }
+        end
+
+        raise NoObjectError if object_type.nil?
+
+        generator = find_generator(object_type)
+
+        raise NoGeneratorError, object_type if generator.nil?
+
+        [generator, known_objects[object_type].find { |obj| object_names.include?(obj['name']) }]
+      end
+
+      # Generate a list of all objects that PDK has a generator for.
+      #
+      # @returns [Array[Array[PDK::Generate::PuppetObject,
+      #   Array[Hash{String=>Object}]]]] an associative array where the first
+      #   element of each pair is the generator class and the second element of
+      #   each pair is an array of description hashes from puppet-strings.
+      def self.all_objects
+        require 'pdk/generate'
+
+        generators = PDK::Generate.generators.select do |gen|
+          gen.const_defined?(:PUPPET_STRINGS_TYPE) && !gen::PUPPET_STRINGS_TYPE.nil?
+        end
+
+        known_objects = generate_hash
+
+        generators.map { |gen| [gen, known_objects[gen::PUPPET_STRINGS_TYPE]] }.reject do |_, obj|
+          obj.nil? || obj.empty?
+        end
+      end
+
+      # Evaluates the mapping of puppet-strings object type to PDK generator
+      # class.
+      #
+      # @param type [String] the object type as returned from puppet-strings.
+      #
+      # @returns [PDK:Generate::PuppetObject,nil] a child of
+      #   PDK::Generate::PuppetObject or nil a suitable generator is not found.
+      #
+      # @example
+      #   PDK::Util::PuppetStrings.find_generator('puppet_classes')
+      #   => PDK::Generate::PuppetClass
+      def self.find_generator(type)
+        require 'pdk/generate'
+
+        PDK::Generate.generators.find do |gen|
+          gen.const_defined?(:PUPPET_STRINGS_TYPE) && type == gen::PUPPET_STRINGS_TYPE
+        end
+      end
+    end
+  end
+end