puppet 6.0.0 → 6.0.1

data/lib/puppet/module/task.rb

@@ -71,19 +71,22 @@ class Puppet::Module
 
       { "name" => name, "path" =>  path }
     end
+    private_class_method :get_file_details
 
-    # Find task's required lib files and retrieve paths
-    # for both 'files' and 'implementation:files' metadata keys
-    def self.find_files(files, mod)
-      env = mod.environment.respond_to?(:name) ? mod.environment.name : 'production'
+    # Find task's required lib files and retrieve paths for both 'files' and 'implementation:files' metadata keys
+    def self.find_extra_files(metadata, envname = nil)
+      return [] if metadata.nil?
 
-      file_list = files.flat_map do |file|
+      files = metadata.fetch('files', []) +
+        metadata.fetch('implementations', []).flat_map { |impl| impl.fetch('files', []) }
+
+      files.uniq.flat_map do |file|
         module_name, mount, endpath = file.split("/", 3)
         # If there's a mount directory with no trailing slash this will be nil
         # We want it to be empty to construct a path
         endpath ||= ''
 
-        pup_module = Puppet::Module.find(module_name, env)
+        pup_module = Puppet::Module.find(module_name, envname)
         if pup_module.nil?
           msg = _("Could not find module %{module_name} containing task file %{filename}" %
                   {module_name: module_name, filename: endpath})
@@ -114,22 +117,18 @@ class Puppet::Module
             raise InvalidMetadata.new(msg, 'puppet.tasks/invalid-metadata')
           end
           dir_files = Dir.glob("#{path}**/*").select { |f| File.file?(f) }
-          files = dir_files.map { |f| get_file_details(f, pup_module) }
+          dir_files.map { |f| get_file_details(f, pup_module) }
         else
           if last_char
             msg = _("Files specified in task metadata cannot include a trailing slash: %{file}" % { file: file } )
             raise InvalidMetadata.new(msg, 'puppet.task/invalid-metadata')
           end
-          files = get_file_details(path, pup_module)
+          get_file_details(path, pup_module)
         end
-
-        files
       end
-      return file_list
     end
+    private_class_method :find_extra_files
 
-    # Copied from TaskInstantiator so we can use the Error classes here
-    # TODO: harmonize on one implementation
     # Executables list should contain the full path of all possible implementation files
     def self.find_implementations(name, directory, metadata, executables)
       basename = name.split('::')[1] || 'init'
@@ -148,7 +147,7 @@ class Puppet::Module
             msg = _("Task metadata for task %{name} specifies missing implementation %{implementation}" % { name: name, implementation: impl['name'] })
             raise InvalidTask.new(msg, 'puppet.tasks/missing-implementation', { missing: [impl['name']] } )
           end
-          { "name" => impl['name'], "requirements" => impl.fetch('requirements', []), "path" => path }
+          { "name" => impl['name'], "path" => path }
         end
         return implementations
       end
@@ -166,7 +165,13 @@ class Puppet::Module
         raise InvalidTask.new(msg, 'puppet.tasks/multiple-implementations')
       end
 
-      [{ "name" => File.basename(implementations.first), "path" => implementations.first, "requirements" => [] }]
+      [{ "name" => File.basename(implementations.first), "path" => implementations.first }]
+    end
+    private_class_method :find_implementations
+
+    def self.find_files(name, directory, metadata, executables, envname = nil)
+      # PXP agent relies on 'impls' (which is the task file) being first if there is no metadata
+      find_implementations(name, directory, metadata, executables) + find_extra_files(metadata, envname)
     end
 
     def self.is_tasks_metadata_filename?(name)
@@ -206,7 +211,7 @@ class Puppet::Module
       @module_executables = module_executables || []
     end
 
-    def read_metadata(file)
+    def self.read_metadata(file)
       Puppet::Util::Json.load(Puppet::FileSystem.read(file, :encoding => 'utf-8')) if file
     rescue SystemCallError, IOError => err
       msg = _("Error reading metadata: %{message}" % {message: err.message})
@@ -216,34 +221,15 @@ class Puppet::Module
     end
 
     def metadata
-      @metadata ||= read_metadata(@metadata_file)
-    end
-
-    def implementations
-      @implementations ||= self.class.find_implementations(@name, @module.tasks_directory, metadata, @module_executables)
+      @metadata ||= self.class.read_metadata(@metadata_file)
     end
 
     def files
-      md = metadata
-      outer_files = []
-      impl_lib_files = []
-      lib_files = []
-
-      unless md.nil?
-        outer_files = md['files'] if md.key?('files')
-        # There's definitely a more elegant way to do this...
-        if md.key?('implementations')
-          md['implementations'].each { |impl| impl_lib_files << impl['files'] if impl.key?('files') }
-        end
-        lib_files = self.class.find_files((impl_lib_files.flatten.uniq + outer_files).uniq, @module)
-      end
-      task_file = implementations.map {|imp| { 'name' => imp['name'], 'path' => imp['path'] } }
-      # PXP agent relies on 'impls' (which is the task file) being first if there is no metadata
-      task_file + lib_files
+      @files ||= self.class.find_files(@name, @module.tasks_directory, metadata, @module_executables, environment_name)
     end
 
     def validate
-      implementations
+      files
       true
     end
 
@@ -252,6 +238,11 @@ class Puppet::Module
       self.module == other.module
     end
 
+    def environment_name
+      @module.environment.respond_to?(:name) ? @module.environment.name : 'production'
+    end
+    private :environment_name
+
     def self.new_with_files(pup_module, name, task_files, module_executables)
       metadata_file = task_files.find { |f| is_tasks_metadata_filename?(f) }
       Puppet::Module::Task.new(pup_module, name, module_executables, metadata_file)

data/lib/puppet/parser/catalog_compiler.rb

@@ -29,4 +29,28 @@ class Puppet::Parser::CatalogCompiler < Puppet::Parser::Compiler
     raise Puppet::Error, message, detail.backtrace
   end
 
+  # Evaluates all added constructs, and validates the resulting catalog.
+  # This can be called whenever a series of evaluation of puppet code strings
+  # have reached a stable state (essentially that there are no relationships to
+  # non-existing resources).
+  #
+  # Raises an error if validation fails.
+  #
+  def compile_additions
+    evaluate_additions
+    validate
+  end
+
+  # Evaluates added constructs that are lazily evaluated until all of them have been evaluated.
+  # 
+  def evaluate_additions
+    evaluate_generators
+    finish
+  end
+
+  # Validates the current state of the catalog.
+  # Does not cause evaluation of lazy constructs.
+  def validate
+    validate_catalog(CatalogValidator::FINAL)
+  end
 end

data/lib/puppet/parser/compiler.rb

@@ -577,6 +577,7 @@ class Puppet::Parser::Compiler
       end
     end
   end
+  protected :evaluate_generators
 
   # Find and evaluate our main object, if possible.
   def evaluate_main
@@ -641,6 +642,7 @@ class Puppet::Parser::Compiler
 
     add_resource_metaparams
   end
+  protected :finish
 
   def add_resource_metaparams
     unless main = catalog.resource(:class, :main)
@@ -780,8 +782,8 @@ class Puppet::Parser::Compiler
     settings_resource = Puppet::Parser::Resource.new('class', SETTINGS, :scope => @topscope)
 
     @catalog.add_resource(settings_resource)
-
     settings_type.evaluate_code(settings_resource)
+    settings_resource.instance_variable_set(:@evaluated, true) # Prevents settings from being reevaluated
 
     scope = @topscope.class_scope(settings_type)
     scope.merge_settings(environment.name)

data/lib/puppet/pops/evaluator/deferred_resolver.rb

@@ -96,6 +96,9 @@ class DeferredResolver
     elsif x.is_a?(Puppet::Pops::Types::PSensitiveType::Sensitive)
       # rewrap in a new Sensitive after resolving any nested deferred values
       Puppet::Pops::Types::PSensitiveType::Sensitive.new(resolve(x.unwrap))
+    elsif x.is_a?(Puppet::Pops::Types::PBinaryType::Binary)
+      # use the ASCII-8BIT string that it wraps
+      x.binary_buffer
     else
       x
     end

data/lib/puppet/pops/evaluator/runtime3_converter.rb

@@ -105,8 +105,8 @@ class Runtime3Converter
   end
 
   def convert_Symbol(o, scope, undef_value)
-    o == :undef ? undef_value : o
-    #o == :undef && !@inner ? undef_value : o
+    return o unless o == :undef
+    !@inner ? undef_value : nil
   end
 
   def convert_PAnyType(o, scope, undef_value)

data/lib/puppet/pops/loader/ruby_legacy_function_instantiator.rb

@@ -20,17 +20,25 @@ class Puppet::Pops::Loader::RubyLegacyFunctionInstantiator
     loader_for_function = loader.private_loader
     here = get_binding(loader_for_function)
 
-    # This will to do the 3x loading and define the "function_<name>" and "real_function_<name>" methods
-    # in the anonymous module used to hold function definitions.
-    #
-    func_info = eval(ruby_code_string, here, source_ref, 1)
+    # Avoid reloading the function if already loaded via one of the APIs that trigger 3x function loading
+    # Check if function is already loaded the 3x way (and obviously not the 4x way since we would not be here in the
+    # first place.
+    environment = Puppet.lookup(:current_environment)
+    func_info = Puppet::Parser::Functions.environment_module(environment).get_function_info(typed_name.name.to_sym)
+    if func_info.nil?
+      # This will to do the 3x loading and define the "function_<name>" and "real_function_<name>" methods
+      # in the anonymous module used to hold function definitions.
+      #
+      func_info = eval(ruby_code_string, here, source_ref, 1)
 
-    unless func_info.is_a?(Hash)
-      raise ArgumentError, _("The code loaded from %{source_ref} did not produce the expected 3x function info Hash when evaluated. Got '%{klass}'") % { source_ref: source_ref, klass: created.class }
-    end
-    unless func_info[:name] == "function_#{typed_name.name()}"
-      raise ArgumentError, _("The code loaded from %{source_ref} produced mis-matched name, expected 'function_%{type_name}', got %{created_name}") % { 
-        source_ref: source_ref, type_name: typed_name.name, created_name: func_info[:name] }
+      # Validate what was loaded
+      unless func_info.is_a?(Hash)
+        raise ArgumentError, _("The code loaded from %{source_ref} did not produce the expected 3x function info Hash when evaluated. Got '%{klass}'") % { source_ref: source_ref, klass: created.class }
+      end
+      unless func_info[:name] == "function_#{typed_name.name()}"
+        raise ArgumentError, _("The code loaded from %{source_ref} produced mis-matched name, expected 'function_%{type_name}', got %{created_name}") % { 
+          source_ref: source_ref, type_name: typed_name.name, created_name: func_info[:name] }
+      end
     end
 
     created = Puppet::Functions::Function3x.create_function(typed_name.name(), func_info, loader_for_function)

data/lib/puppet/pops/loader/task_instantiator.rb

@@ -1,58 +1,9 @@
 # The TypeDefinitionInstantiator instantiates a type alias or a type definition
 #
+require 'puppet/module/task'
 module Puppet::Pops
 module Loader
 class TaskInstantiator
-  def self.load_metadata(loader, metadata)
-    if metadata.nil?
-      EMPTY_HASH
-    else
-      json_text = loader.get_contents(metadata)
-      begin
-        Puppet::Util::Json.load(json_text).freeze || EMPTY_HASH
-      rescue Puppet::Util::Json::ParseError => ex
-        raise Puppet::ParseError.new(ex.message, metadata)
-      end
-    end
-  end
-
-  def self.validate_implementations(typed_name, directory, metadata, executables)
-    name = typed_name.name
-    basename = typed_name.name_parts[1] || 'init'
-    # If 'implementations' is defined, it needs to mention at least one
-    # implementation, and everything it mentions must exist.
-    if metadata.key?('implementations')
-      if metadata['implementations'].is_a?(Array)
-        metadata['implementations'].map do |impl|
-          path = executables.find { |real_impl| File.basename(real_impl) == impl['name'] }
-          if path
-            { "name" => impl['name'], "requirements" => impl.fetch('requirements', []), "path" => path }
-          else
-            raise ArgumentError, _("Task metadata for task %{name} specifies missing implementation %{implementation}") %
-              { name: name, implementation: impl['name'] }
-          end
-        end
-      else
-        # If 'implementations' is the wrong type, we just pass it through and
-        # let the task type definition reject it.
-        metadata['implementations']
-      end
-    # If implementations isn't defined, then we use executables matching the
-    # task name, and only one may exist.
-    else
-      implementations = executables.select { |impl| File.basename(impl, '.*') == basename }
-      if implementations.empty?
-        raise ArgumentError, _('No source besides task metadata was found in directory %{directory} for task %{name}') %
-          { name: name, directory: directory }
-      elsif implementations.length > 1
-        raise ArgumentError, _("Multiple executables were found in directory %{directory} for task %{name}; define 'implementations' in metadata to differentiate between them") %
-          { name: name, directory: implementations[0] }
-      end
-
-      [{ "name" => File.basename(implementations.first), "path" => implementations.first, "requirements" => [] }]
-    end
-  end
-
   def self.create(loader, typed_name, source_refs)
     name = typed_name.name
     basename = typed_name.name_parts[1] || 'init'
@@ -60,31 +11,16 @@ class TaskInstantiator
     metadata_files, executables = source_refs.partition { |source_ref| source_ref.end_with?('.json') }
     metadata_file = metadata_files.find { |source_ref| File.basename(source_ref, '.json') == basename }
 
-    metadata = load_metadata(loader, metadata_file)
+    metadata = Puppet::Module::Task.read_metadata(metadata_file) || {}
 
-    implementation_metadata = validate_implementations(typed_name, dirname, metadata, executables)
+    files = Puppet::Module::Task.find_files(name, dirname, metadata, executables)
 
-    arguments = {
-      'name' => name,
-      'implementations' => implementation_metadata
-    }
+    task = { 'name' => name, 'metadata' => metadata, 'files' => files }
 
     begin
-      metadata.each_pair do |key, value|
-        if %w[parameters output].include?(key)
-          ps = {}
-          value.each_pair do |k, v|
-            pd = v.dup
-            t = v['type']
-            pd['type'] = t.nil? ? Types::TypeFactory.data : Types::TypeParser.singleton.parse(t)
-            ps[k] = pd
-          end
-          value = ps
-        end
-        arguments[key] = value unless arguments.key?(key)
-      end
+      task['parameters'] = convert_types(metadata['parameters'])
 
-      Types::TypeFactory.task.from_hash(arguments)
+      Types::TypeFactory.task.from_hash(task)
     rescue Types::TypeAssertionError => ex
       # Not strictly a parser error but from the users perspective, the file content didn't parse properly. The
       # ParserError also conveys file info (even though line is unknown)
@@ -92,6 +28,13 @@ class TaskInstantiator
       raise Puppet::ParseError.new(msg, metadata_file)
     end
   end
+
+  def self.convert_types(args)
+    args.each_with_object({}) do |(k, v), hsh|
+      hsh[k] = v['type'].nil? ? Types::TypeFactory.data : Types::TypeParser.singleton.parse(v['type'])
+    end if args
+  end
+  private_class_method :convert_types
 end
 end
 end

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -98,8 +98,7 @@ module HeredocSupport
 
         # Use a new lexer instance configured with a sub-locator to enable correct positioning
         sublexer = self.class.new()
-        locator = Locator::SubLocator.sub_locator(str,
-          locator.file, heredoc_line, heredoc_offset, leading.length())
+        locator = Locator::SubLocator.new(locator, heredoc_line, heredoc_offset, leading.length())
 
         # Emit a token that provides the grammar with location information about the lines on which the heredoc
         # content is based.

data/lib/puppet/pops/parser/lexer2.rb

@@ -189,7 +189,7 @@ class Lexer2
       ',' => lambda {  emit(TOKEN_COMMA, @scanner.pos) },
       '[' => lambda do
         before = @scanner.pos
-        if (before == 0 || @scanner.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
+        if (before == 0 || locator.string[locator.char_offset(before)-1,1] =~ /[[:blank:]\r\n]+/)
           emit(TOKEN_LISTSTART, before)
         else
           emit(TOKEN_LBRACK, before)

data/lib/puppet/pops/pcore.rb

@@ -39,39 +39,16 @@ module Pcore
             # Fully qualified name of the task
             name => { type => Pattern[/\\A[a-z][a-z0-9_]*(?:::[a-z][a-z0-9_]*)*\\z/] },
 
-            # List of implementations with requirements
-            implementations => { type => Array[Struct[name => String, path => String, Optional[requirements] => Array[String]], 1] },
-
-            # Task description
-            description => { type => Optional[String], value => undef },
-
-            # Puppet Task version
-            puppet_task_version => { type => Integer, value => 1 },
-
-            # Type, description, and sensitive property of each parameter
-            parameters => {
-              type => Optional[Hash[
-                Pattern[/\\A[a-z][a-z0-9_]*\\z/],
-                Struct[
-                  Optional[description] => String,
-                  Optional[sensitive] => Boolean,
-                  type => Type]]],
-              value => undef
-            },
-
-             # Type, description, and sensitive property of each output
-            output => {
-              type => Optional[Hash[
-                Pattern[/\\A[a-z][a-z0-9_]*\\z/],
-                Struct[
-                  Optional[description] => String,
-                  Optional[sensitive] => Boolean,
-                  type => Type]]],
-              value => undef
-            },
-
-            supports_noop => { type => Boolean, value => false },
-            input_method => { type => Optional[String] },
+            # List of file references referenced by metadata and their paths on disk.
+            # If there are no implementations listed in metadata, the first file is always
+            # the task executable.
+            files => { type => Array[Struct[name => String, path => String]] },
+
+            # Task metadata
+            metadata => { type => Hash[String, Any] },
+
+            # Map parameter names to their parsed data type
+            parameters => { type => Optional[Hash[Pattern[/\\A[a-z][a-z0-9_]*\\z/], Type]], value => undef },
           }
         }
       PUPPET

data/lib/puppet/pops/serialization.rb

@@ -19,6 +19,7 @@ module Serialization
 
   # Type key used for symbols
   PCORE_TYPE_SENSITIVE = 'Sensitive'.freeze
+  PCORE_TYPE_BINARY = 'Binary'.freeze
 
   # Type key used for symbols
   PCORE_TYPE_SYMBOL = 'Symbol'.freeze

data/lib/puppet/pops/serialization/to_data_converter.rb

@@ -70,7 +70,15 @@ module Serialization
 
     def to_data(value)
       if value.nil? || Types::PScalarDataType::DEFAULT.instance?(value)
-        value
+        if @rich_data && value.is_a?(String) && value.encoding == Encoding::ASCII_8BIT
+          # Transform the binary string to rich Binary
+          {
+            PCORE_TYPE_KEY => PCORE_TYPE_BINARY,
+            PCORE_VALUE_KEY => Puppet::Pops::Types::PBinaryType::Binary.from_binary_string(value).to_s
+          }
+        else
+          value
+        end
       elsif :default == value
         if @rich_data
           { PCORE_TYPE_KEY => PCORE_TYPE_DEFAULT }