puppet 5.5.22 → 6.0.0

data/lib/puppet/pops/loader/ruby_legacy_function_instantiator.rb

@@ -0,0 +1,54 @@
+# The RubyLegacyFunctionInstantiator instantiates a Puppet::Functions::Function given the ruby source
+# that calls Puppet::Functions.create_function.
+#
+class Puppet::Pops::Loader::RubyLegacyFunctionInstantiator
+  # Produces an instance of the Function class with the given typed_name, or fails with an error if the
+  # given ruby source does not produce this instance when evaluated.
+  #
+  # @param loader [Puppet::Pops::Loader::Loader] The loader the function is associated with
+  # @param typed_name [Puppet::Pops::Loader::TypedName] the type / name of the function to load
+  # @param source_ref [URI, String] a reference to the source / origin of the ruby code to evaluate
+  # @param ruby_code_string [String] ruby code in a string
+  #
+  # @return [Puppet::Pops::Functions.Function] - an instantiated function with global scope closure associated with the given loader
+  #
+  def self.create(loader, typed_name, source_ref, ruby_code_string)
+    unless ruby_code_string.is_a?(String) && ruby_code_string =~ /Puppet\:\:Parser\:\:Functions.*newfunction/m
+      raise ArgumentError, _("The code loaded from %{source_ref} does not seem to be a Puppet 3x API function - no 'newfunction' call.") % { source_ref: source_ref }
+    end
+    # make the private loader available in a binding to allow it to be passed on
+    loader_for_function = loader.private_loader
+    here = get_binding(loader_for_function)
+
+    # This will to do the 3x loading and define the "function_<name>" and "real_function_<name>" methods
+    # in the anonymous module used to hold function definitions.
+    #
+    func_info = eval(ruby_code_string, here, source_ref, 1)
+
+    unless func_info.is_a?(Hash)
+      raise ArgumentError, _("The code loaded from %{source_ref} did not produce the expected 3x function info Hash when evaluated. Got '%{klass}'") % { source_ref: source_ref, klass: created.class }
+    end
+    unless func_info[:name] == "function_#{typed_name.name()}"
+      raise ArgumentError, _("The code loaded from %{source_ref} produced mis-matched name, expected 'function_%{type_name}', got %{created_name}") % { 
+        source_ref: source_ref, type_name: typed_name.name, created_name: func_info[:name] }
+    end
+
+    created = Puppet::Functions::Function3x.create_function(typed_name.name(), func_info, loader_for_function)
+
+    # create the function instance - it needs closure (scope), and loader (i.e. where it should start searching for things
+    # when calling functions etc.
+    # It should be bound to global scope
+
+    # Sets closure scope to nil, to let it be picked up at runtime from Puppet.lookup(:global_scope)
+    # If function definition used the loader from the binding to create a new loader, that loader wins
+    created.new(nil, loader_for_function)
+  end
+
+  # Produces a binding where the given loader is bound as a local variable (loader_injected_arg). This variable can be used in loaded
+  # ruby code - e.g. to call Puppet::Function.create_loaded_function(:name, loader,...)
+  #
+  def self.get_binding(loader_injected_arg)
+    binding
+  end
+  private_class_method :get_binding
+end

data/lib/puppet/pops/loader/static_loader.rb

@@ -6,57 +6,21 @@ module Loader
 class StaticLoader < Loader
 
   BUILTIN_TYPE_NAMES = %w{
-      Augeas
       Component
-      Computer
-      Cron
       Exec
       File
       Filebucket
       Group
-      Host
-      Interface
-      K5login
-      Macauthorization
-      Mailalias
-      Maillist
-      Mcx
-      Mount
-      Nagios_command
-      Nagios_contact
-      Nagios_contactgroup
-      Nagios_host
-      Nagios_hostdependency
-      Nagios_hostescalation
-      Nagios_hostgroup
-      Nagios_hostextinfo
-      Nagios_service
-      Nagios_servicedependency
-      Nagios_serviceescalation
-      Nagios_serviceextinfo
-      Nagios_servicegroup
-      Nagios_timeperiod
       Node
       Notify
       Package
       Resources
-      Router
       Schedule
-      Scheduled_task
-      Selboolean
-      Selmodule
       Service
-      Ssh_authorized_key
-      Sshkey
       Stage
       Tidy
       User
-      Vlan
       Whit
-      Yumrepo
-      Zfs
-      Zone
-      Zpool
     }.freeze
 
   BUILTIN_TYPE_NAMES_LC = Set.new(BUILTIN_TYPE_NAMES.map { |n| n.downcase }).freeze

data/lib/puppet/pops/loader/task_instantiator.rb

@@ -3,66 +3,94 @@
 module Puppet::Pops
 module Loader
 class TaskInstantiator
-  def self.create(loader, typed_name, source_refs)
-    name = typed_name.name
-    metadata = nil
-    task_source = nil
-    source_refs.each do |source_ref|
-      if source_ref.end_with?('.json')
-        metadata = source_ref
-      elsif task_source.nil?
-        task_source = source_ref
-      else
-        raise ArgumentError, _('Only one file can exists besides the .json file for task %{name} in directory %{directory}') %
-          { name: name, directory: File.dirname(source_refs[0]) }
-      end
-    end
-
-    if task_source.nil?
-      raise ArgumentError, _('No source besides task metadata was found in directory %{directory} for task %{name}') %
-        { name: name, directory: File.dirname(source_refs[0]) }
-    end
-    create_task(loader, name, task_source, metadata)
-  end
-
-  def self.create_task(loader, name, task_source, metadata)
+  def self.load_metadata(loader, metadata)
     if metadata.nil?
-      create_task_from_hash(name, task_source, EMPTY_HASH)
+      EMPTY_HASH
     else
       json_text = loader.get_contents(metadata)
       begin
-        create_task_from_hash(name, task_source, Puppet::Util::Json.load(json_text) || EMPTY_HASH)
+        Puppet::Util::Json.load(json_text).freeze || EMPTY_HASH
       rescue Puppet::Util::Json::ParseError => ex
         raise Puppet::ParseError.new(ex.message, metadata)
-      rescue Types::TypeAssertionError => ex
-        # Not strictly a parser error but from the users perspective, the file content didn't parse properly. The
-        # ParserError also conveys file info (even though line is unknown)
-        msg = _('Failed to load metadata for task %{name}: %{reason}') % { :name => name, :reason => ex.message }
-        raise Puppet::ParseError.new(msg, metadata)
       end
     end
   end
 
-  def self.create_task_from_hash(name, task_source, hash)
+  def self.validate_implementations(typed_name, directory, metadata, executables)
+    name = typed_name.name
+    basename = typed_name.name_parts[1] || 'init'
+    # If 'implementations' is defined, it needs to mention at least one
+    # implementation, and everything it mentions must exist.
+    if metadata.key?('implementations')
+      if metadata['implementations'].is_a?(Array)
+        metadata['implementations'].map do |impl|
+          path = executables.find { |real_impl| File.basename(real_impl) == impl['name'] }
+          if path
+            { "name" => impl['name'], "requirements" => impl.fetch('requirements', []), "path" => path }
+          else
+            raise ArgumentError, _("Task metadata for task %{name} specifies missing implementation %{implementation}") %
+              { name: name, implementation: impl['name'] }
+          end
+        end
+      else
+        # If 'implementations' is the wrong type, we just pass it through and
+        # let the task type definition reject it.
+        metadata['implementations']
+      end
+    # If implementations isn't defined, then we use executables matching the
+    # task name, and only one may exist.
+    else
+      implementations = executables.select { |impl| File.basename(impl, '.*') == basename }
+      if implementations.empty?
+        raise ArgumentError, _('No source besides task metadata was found in directory %{directory} for task %{name}') %
+          { name: name, directory: directory }
+      elsif implementations.length > 1
+        raise ArgumentError, _("Multiple executables were found in directory %{directory} for task %{name}; define 'implementations' in metadata to differentiate between them") %
+          { name: name, directory: implementations[0] }
+      end
+
+      [{ "name" => File.basename(implementations.first), "path" => implementations.first, "requirements" => [] }]
+    end
+  end
+
+  def self.create(loader, typed_name, source_refs)
+    name = typed_name.name
+    basename = typed_name.name_parts[1] || 'init'
+    dirname = File.dirname(source_refs[0])
+    metadata_files, executables = source_refs.partition { |source_ref| source_ref.end_with?('.json') }
+    metadata_file = metadata_files.find { |source_ref| File.basename(source_ref, '.json') == basename }
+
+    metadata = load_metadata(loader, metadata_file)
+
+    implementation_metadata = validate_implementations(typed_name, dirname, metadata, executables)
+
     arguments = {
       'name' => name,
-      'executable' => task_source
+      'implementations' => implementation_metadata
     }
-    hash.each_pair do |key, value|
-      if 'parameters' == key || 'output' == key
-        ps = {}
-        value.each_pair do |k, v|
-          pd = v.dup
-          t = v['type']
-          pd['type'] = t.nil? ? Types::TypeFactory.data : Types::TypeParser.singleton.parse(t)
-          ps[k] = pd
+
+    begin
+      metadata.each_pair do |key, value|
+        if %w[parameters output].include?(key)
+          ps = {}
+          value.each_pair do |k, v|
+            pd = v.dup
+            t = v['type']
+            pd['type'] = t.nil? ? Types::TypeFactory.data : Types::TypeParser.singleton.parse(t)
+            ps[k] = pd
+          end
+          value = ps
         end
-        value = ps
+        arguments[key] = value unless arguments.key?(key)
       end
-      arguments[key] = value
-    end
 
-    Types::TypeFactory.task.from_hash(arguments)
+      Types::TypeFactory.task.from_hash(arguments)
+    rescue Types::TypeAssertionError => ex
+      # Not strictly a parser error but from the users perspective, the file content didn't parse properly. The
+      # ParserError also conveys file info (even though line is unknown)
+      msg = _('Failed to load metadata for task %{name}: %{reason}') % { name: name, reason: ex.message }
+      raise Puppet::ParseError.new(msg, metadata_file)
+    end
   end
 end
 end

data/lib/puppet/pops/loaders.rb

@@ -13,6 +13,7 @@ class Loaders
 
   attr_reader :static_loader
   attr_reader :puppet_system_loader
+  attr_reader :puppet_cache_loader
   attr_reader :public_environment_loader
   attr_reader :private_environment_loader
   attr_reader :environment
@@ -43,35 +44,34 @@ class Loaders
     #
     @puppet_system_loader = create_puppet_system_loader()
 
-    # 2. Environment loader - i.e. what is bound across the environment, may change for each setup
+    # 2. Cache loader(optional) - i.e. what puppet stores on disk via pluginsync; gate behind the for_agent flag.
+    # 3. Environment loader - i.e. what is bound across the environment, may change for each setup
     #    TODO: loaders need to work when also running in an agent doing catalog application. There is no
     #    concept of environment the same way as when running as a master (except when doing apply).
     #    The creation mechanisms should probably differ between the two.
-    #
     @private_environment_loader = if for_agent
-      add_loader_by_name(Loader::SimpleEnvironmentLoader.new(@puppet_system_loader, 'agent environment'))
+      @puppet_cache_loader = create_puppet_cache_loader()
+      create_environment_loader(environment, @puppet_cache_loader)
     else
-      create_environment_loader(environment)
+      create_environment_loader(environment, @puppet_system_loader)
     end
 
     Pcore.init_env(@private_environment_loader)
 
-    # 3. module loaders are set up from the create_environment_loader, they register themselves
+    # 4. module loaders are set up from the create_environment_loader, they register themselves
   end
 
   # Called after loader has been added to Puppet Context as :loaders so that dynamic types can
   # be pre-loaded with a fully configured loader system
   def pre_load
     @puppet_system_loader.load(:type, 'error')
-
-    # Will move to Bolt
-    @puppet_system_loader.load(:type, 'executionresult')
   end
 
   # Clears the cached static and puppet_system loaders (to enable testing)
   #
   def self.clear
     @@static_loader = nil
+    Puppet::Pops::Types::TypeFactory.clear
     Model.class_variable_set(:@@pcore_ast_initialized, false)
     Model.register_pcore_types
   end
@@ -283,7 +283,7 @@ class Loaders
         nil
       elsif File.directory?(file)
         raise Puppet::Error, "manifest of environment '#{@environment.name}' appoints directory '#{file}'. It must be a file"
-      elsif File.exist?(file)
+      elsif File.exists?(file)
         parser.parse_file(file)
       else
         raise Puppet::Error, "manifest of environment '#{@environment.name}' appoints '#{file}'. It does not exist"
@@ -356,7 +356,11 @@ class Loaders
     Loader::ModuleLoaders.system_loader_from(static_loader, self)
   end
 
-  def create_environment_loader(environment)
+  def create_puppet_cache_loader()
+    Loader::ModuleLoaders.cached_loader_from(puppet_system_loader, self)
+  end
+
+  def create_environment_loader(environment, parent_loader)
     # This defines where to start parsing/evaluating - the "initial import" (to use 3x terminology)
     # Is either a reference to a single .pp file, or a directory of manifests. If the environment becomes
     # a module and can hold functions, types etc. then these are available across all other modules without
@@ -377,11 +381,11 @@ class Loaders
     env_path = env_conf.nil? || !env_conf.is_a?(Puppet::Settings::EnvironmentConf) ? nil : env_conf.path_to_env
 
     if Puppet[:tasks]
-      loader = Loader::ModuleLoaders.environment_loader_from(puppet_system_loader, self, env_path)
+      loader = Loader::ModuleLoaders.environment_loader_from(parent_loader, self, env_path)
     else
       # Create the 3.x resource type loader
       static_loader.runtime_3_init
-      @runtime3_type_loader = add_loader_by_name(Loader::Runtime3TypeLoader.new(puppet_system_loader, self, environment, env_conf.nil? ? nil : env_path))
+      @runtime3_type_loader = add_loader_by_name(Loader::Runtime3TypeLoader.new(parent_loader, self, environment, env_conf.nil? ? nil : env_path))
 
       if env_path.nil?
         # Not a real directory environment, cannot work as a module TODO: Drop when legacy env are dropped?
@@ -505,8 +509,8 @@ class Loaders
         nil
       else
         module_data.private_loader =
-          if module_data.restrict_to_dependencies? && !Puppet[:tasks]
-            create_loader_with_only_dependencies_visible(module_data)
+          if module_data.restrict_to_dependencies?
+            create_loader_with_dependencies_first(module_data)
           else
             create_loader_with_all_modules_visible(module_data)
           end
@@ -516,29 +520,13 @@ class Loaders
     private
 
     def create_loader_with_all_modules_visible(from_module_data)
-      Puppet.debug{"ModuleLoader: module '#{from_module_data.name}' has unknown dependencies - it will have all other modules visible"}
-
       @loaders.add_loader_by_name(Loader::DependencyLoader.new(from_module_data.public_loader, "#{from_module_data.name} private", all_module_loaders()))
     end
 
-    def create_loader_with_only_dependencies_visible(from_module_data)
-      if from_module_data.unmet_dependencies?
-        if Puppet[:strict] != :off
-          msg = "ModuleLoader: module '#{from_module_data.name}' has unresolved dependencies" \
-              " - it will only see those that are resolved." \
-              " Use 'puppet module list --tree' to see information about modules"
-          case Puppet[:strict]
-          when :error
-              raise LoaderError.new(msg)
-          when :warning
-            Puppet.warn_once(:unresolved_module_dependencies,
-                             "unresolved_dependencies_for_module_#{from_module_data.name}",
-                             msg)
-          end
-        end
-      end
+    def create_loader_with_dependencies_first(from_module_data)
       dependency_loaders = from_module_data.dependency_names.collect { |name| @index[name].public_loader }
-      @loaders.add_loader_by_name(Loader::DependencyLoader.new(from_module_data.public_loader, "#{from_module_data.name} private", dependency_loaders))
+      visible_loaders = dependency_loaders + (all_module_loaders() - dependency_loaders)
+      @loaders.add_loader_by_name(Loader::DependencyLoader.new(from_module_data.public_loader, "#{from_module_data.name} private", visible_loaders))
     end
   end
 end

data/lib/puppet/pops/lookup/hiera_config.rb

@@ -132,7 +132,7 @@ class HieraConfig
       if config_path.exist?
         env_context = EnvironmentContext.adapt(lookup_invocation.scope.compiler.environment)
         loaded_config = env_context.cached_file_data(config_path) do |content|
-          parsed = YAML.load(content, config_path)
+          parsed = Puppet::Util::Yaml.safe_load(content, [Symbol], config_path)
 
           # For backward compatibility, we must treat an empty file, or a yaml that doesn't
           # produce a Hash as Hiera version 3 default.
@@ -339,7 +339,6 @@ class HieraConfig
   def self.not_implemented(impl, method_name)
     raise NotImplementedError, "The class #{impl.class.name} should have implemented the method #{method_name}()"
   end
-  private_class_method :not_implemented
 end
 
 # @api private

data/lib/puppet/pops/lookup/sub_lookup.rb

@@ -30,7 +30,7 @@ module SubLookup
       raise yield('Syntax error') unless segments.size * 2 == count + 1
       segments.map! do |segment|
         segment.strip!
-        if segment.start_with?('"', "'")
+        if segment.start_with?('"') || segment.start_with?("'")
           segment[1..-2]
         elsif segment =~ /^(:?[+-]?[0-9]+)$/
           segment.to_i

data/lib/puppet/pops/merge_strategy.rb

@@ -173,14 +173,12 @@ module Puppet::Pops
 
     protected
 
-    class << self
-      # Returns the type used to validate the options hash
-      #
-      # @return [Types::PStructType] the puppet type
-      #
-      def options_t
-        @options_t ||=Types::TypeParser.singleton.parse("Struct[{strategy=>Optional[Pattern[/#{key}/]]}]")
-      end
+    # Returns the type used to validate the options hash
+    #
+    # @return [Types::PStructType] the puppet type
+    #
+    def self.options_t
+      @options_t ||=Types::TypeParser.singleton.parse("Struct[{strategy=>Optional[Pattern[/#{key}/]]}]")
     end
 
     # Returns the type used to validate the options hash
@@ -385,20 +383,18 @@ module Puppet::Pops
 
     protected
 
-    class << self
-      # Returns a type that allows all deep_merge options except 'preserve_unmergeables' since we force
-      # the setting of that option to false
-      #
-      # @return [Types::PAnyType] the puppet type used when validating the options hash
-      def options_t
-        @options_t ||= Types::TypeParser.singleton.parse('Struct[{'\
-                                                         "strategy=>Optional[Pattern[#{key}]],"\
-                                                         'knockout_prefix=>Optional[String],'\
-                                                         'merge_debug=>Optional[Boolean],'\
-                                                         'merge_hash_arrays=>Optional[Boolean],'\
-                                                         'sort_merged_arrays=>Optional[Boolean],'\
-                                                         '}]')
-      end
+    # Returns a type that allows all deep_merge options except 'preserve_unmergeables' since we force
+    # the setting of that option to false
+    #
+    # @return [Types::PAnyType] the puppet type used when validating the options hash
+    def self.options_t
+      @options_t ||= Types::TypeParser.singleton.parse('Struct[{'\
+          "strategy=>Optional[Pattern[#{key}]],"\
+          'knockout_prefix=>Optional[String],'\
+          'merge_debug=>Optional[Boolean],'\
+          'merge_hash_arrays=>Optional[Boolean],'\
+          'sort_merged_arrays=>Optional[Boolean],'\
+          '}]')
     end
 
     def value_t