puppet 5.3.7-x64-mingw32 → 5.4.0-x64-mingw32

data/lib/puppet/pops/loader/puppet_resource_type_impl_instantiator.rb

@@ -8,7 +8,7 @@ class PuppetResourceTypeImplInstantiator
   # given puppet source does not produce such an instance when evaluated.
   #
   # @param loader [Loader] The loader the function is associated with
-  # @param typed_name [TypedName] the type / name of the resoure type impl to load
+  # @param typed_name [TypedName] the type / name of the resource type impl to load
   # @param source_ref [URI, String] a reference to the source / origin of the puppet code to evaluate
   # @param pp_code_string [String] puppet code in a string
   #
@@ -20,14 +20,14 @@ class PuppetResourceTypeImplInstantiator
     # parse and validate
     model = parser.parse_string(pp_code_string, source_ref)
     statements = if model.is_a?(Model::Program)
-      if model.body.is_a?(Model::BlockExpression)
-        statements = model.body.statements
-      else
-        statements = [model.body]
-      end
-    else
-      statements = EMPTY_ARRAY
-    end
+                   if model.body.is_a?(Model::BlockExpression)
+                     model.body.statements
+                   else
+                     [model.body]
+                   end
+                 else
+                   EMPTY_ARRAY
+                 end
     statements = statements.reject { |s| s.is_a?(Model::Nop) }
     if statements.empty?
       raise ArgumentError, _("The code loaded from %{source_ref} does not create the resource type '%{type_name}' - it is empty") % { source_ref: source_ref, type_name: typed_name.name }

data/lib/puppet/pops/loader/ruby_data_type_instantiator.rb

@@ -0,0 +1,40 @@
+# The RubyTypeInstantiator instantiates a data type from the ruby source
+# that calls Puppet::DataTypes.create_type.
+#
+class Puppet::Pops::Loader::RubyDataTypeInstantiator
+  # Produces an instance of class derived from PAnyType class with the given typed_name, or fails with an error if the
+  # given ruby source does not produce this instance when evaluated.
+  #
+  # @param loader [Puppet::Pops::Loader::Loader] The loader the type is associated with
+  # @param typed_name [Puppet::Pops::Loader::TypedName] the type / name of the type to load
+  # @param source_ref [URI, String] a reference to the source / origin of the ruby code to evaluate
+  # @param ruby_code_string [String] ruby code in a string
+  #
+  # @return [Puppet::Pops::Types::PAnyType] - an instantiated data type associated with the given loader
+  #
+  def self.create(loader, typed_name, source_ref, ruby_code_string)
+    unless ruby_code_string.is_a?(String) && ruby_code_string =~ /Puppet\:\:DataTypes\.create_type/
+      raise ArgumentError, _("The code loaded from %{source_ref} does not seem to be a Puppet 5x API data type - no create_type call.") % { source_ref: source_ref }
+    end
+    # make the private loader available in a binding to allow it to be passed on
+    loader_for_type = loader.private_loader
+    here = get_binding(loader_for_type)
+    created = eval(ruby_code_string, here, source_ref, 1)
+    unless created.is_a?(Puppet::Pops::Types::PAnyType)
+      raise ArgumentError, _("The code loaded from %{source_ref} did not produce a data type when evaluated. Got '%{klass}'") % { source_ref: source_ref, klass: created.class }
+    end
+    unless created.name.casecmp(typed_name.name) == 0
+      raise ArgumentError, _("The code loaded from %{source_ref} produced mis-matched name, expected '%{type_name}', got %{created_name}") % { source_ref: source_ref, type_name: typed_name.name, created_name: created.name }
+    end
+    created
+  end
+
+  private
+
+  # Produces a binding where the given loader is bound as a local variable (loader_injected_arg). This variable can be used in loaded
+  # ruby code - e.g. to call Puppet::Function.create_loaded_function(:name, loader,...)
+  #
+  def self.get_binding(loader_injected_arg)
+    binding
+  end
+end

data/lib/puppet/pops/loader/runtime3_type_loader.rb

@@ -13,6 +13,11 @@ class Runtime3TypeLoader < BaseLoader
     @resource_3x_loader = env_path.nil? ? nil : ModuleLoaders.pcore_resource_type_loader_from(parent_loader, loaders, env_path)
   end
 
+  def discover(type, error_collector = nil, name_authority = Pcore::RUNTIME_NAME_AUTHORITY, &block)
+    # TODO: Use generated index of all known types (requires separate utility).
+    parent.discover(type, error_collector, name_authority, &block)
+  end
+
   def to_s()
     "(Runtime3TypeLoader '#{loader_name()}')"
   end
@@ -83,7 +88,7 @@ class Runtime3TypeLoader < BaseLoader
   end
   private :find_impl
 
-  # Allows shadowing since this loader is populalted with all loaded resource types at time
+  # Allows shadowing since this loader is populated with all loaded resource types at time
   # of loading. This loading will, for built in types override the aliases configured in the static
   # loader.
   #

data/lib/puppet/pops/loader/static_loader.rb

@@ -64,7 +64,7 @@ class StaticLoader < Loader
   BUILTIN_ALIASES = {
     'Data' => 'Variant[ScalarData,Undef,Hash[String,Data],Array[Data]]',
     'RichDataKey' => 'Variant[String,Numeric]',
-    'RichData' => 'Variant[Scalar,SemVerRange,Binary,Sensitive,Type,TypeSet,Undef,Default,Hash[RichDataKey,RichData],Array[RichData]]',
+    'RichData' => 'Variant[Scalar,SemVerRange,Binary,Sensitive,Type,TypeSet,URI,Object,Undef,Default,Hash[RichDataKey,RichData],Array[RichData]]',
 
     # Backward compatible aliases.
     'Puppet::LookupKey' => 'RichDataKey',
@@ -74,9 +74,16 @@ class StaticLoader < Loader
   attr_reader :loaded
   def initialize
     @loaded = {}
+    @runtime_3_initialized = false
     create_built_in_types
-    create_resource_type_references
-    register_aliases
+  end
+
+  def discover(type, error_collector = nil, name_authority = Pcore::RUNTIME_NAME_AUTHORITY)
+    # Static loader only contains runtime types
+    return EMPTY_ARRAY unless type == :type && name_authority == name_authority = Pcore::RUNTIME_NAME_AUTHORITY
+
+    typed_names = type == :type && name_authority == Pcore::RUNTIME_NAME_AUTHORITY ? @loaded.keys : EMPTY_ARRAY
+    block_given? ? typed_names.select { |tn| yield(tn) } : typed_names
   end
 
   def load_typed(typed_name)
@@ -108,6 +115,19 @@ class StaticLoader < Loader
     @loaded[typed_name]
   end
 
+  def runtime_3_init
+    unless @runtime_3_initialized
+      @runtime_3_initialized = true
+      create_resource_type_references
+    end
+    nil
+  end
+
+  def register_aliases
+    aliases = BUILTIN_ALIASES.map { |name, string| add_type(name, Types::PTypeAliasType.new(name, Types::TypeFactory.type_reference(string), nil)) }
+    aliases.each { |type| type.resolve(self) }
+  end
+
   private
 
   def load_constant(typed_name)
@@ -138,11 +158,6 @@ class StaticLoader < Loader
   def create_resource_type_reference(name)
     add_type(name, Types::TypeFactory.resource(name))
   end
-
-  def register_aliases
-    aliases = BUILTIN_ALIASES.map { |name, string| add_type(name, Types::PTypeAliasType.new(name, Types::TypeFactory.type_reference(string), nil)) }
-    aliases.each { |type| type.resolve(self) }
-  end
 end
 end
 end

data/lib/puppet/pops/loader/task_instantiator.rb

@@ -0,0 +1,69 @@
+# The TypeDefinitionInstantiator instantiates a type alias or a type definition
+#
+module Puppet::Pops
+module Loader
+class TaskInstantiator
+  def self.create(loader, typed_name, source_refs)
+    name = typed_name.name
+    metadata = nil
+    task_source = nil
+    source_refs.each do |source_ref|
+      if source_ref.end_with?('.json')
+        metadata = source_ref
+      elsif task_source.nil?
+        task_source = source_ref
+      else
+        raise ArgumentError, _('Only one file can exists besides the .json file for task %{name} in directory %{directory}') %
+          { name: name, directory: File.dirname(source_refs[0]) }
+      end
+    end
+
+    if task_source.nil?
+      raise ArgumentError, _('No source besides task metadata was found in directory %{directory} for task %{name}') %
+        { name: name, directory: File.dirname(source_refs[0]) }
+    end
+    create_task(loader, name, task_source, metadata)
+  end
+
+  def self.create_task(loader, name, task_source, metadata)
+    if metadata.nil?
+      create_task_from_hash(loader, name, task_source, EMPTY_HASH)
+    else
+      json_text = loader.get_contents(metadata)
+      begin
+        create_task_from_hash(loader, name, task_source, JSON.parse(json_text) || EMPTY_HASH)
+      rescue JSON::ParserError => ex
+        raise Puppet::ParseError.new(ex.message, metadata)
+      rescue Types::TypeAssertionError => ex
+        # Not strictly a parser error but from the users perspective, the file content didn't parse properly. The
+        # ParserError also conveys file info (even though line is unknown)
+        msg = _('Failed to load metadata for task %{name}: %{reason}') % { :name => name, :reason => ex.message }
+        raise Puppet::ParseError.new(msg, metadata)
+      end
+    end
+  end
+
+  def self.create_task_from_hash(loader, name, task_source, hash)
+    arguments = {
+      'name' => name,
+      'executable' => task_source
+    }
+    hash.each_pair do |key, value|
+      if 'parameters' == key || 'output' == key
+        ps = {}
+        value.each_pair do |k, v|
+          pd = v.dup
+          t = v['type']
+          pd['type'] = t.nil? ? Types::TypeFactory.data : Types::TypeParser.singleton.parse(t, loader)
+          ps[k] = pd
+        end
+        value = ps
+      end
+      arguments[key] = value
+    end
+
+    Types::TypeFactory.task.from_hash(arguments)
+  end
+end
+end
+end

data/lib/puppet/pops/loader/type_definition_instantiator.rb

@@ -88,6 +88,10 @@ class TypeDefinitionInstantiator
   def self.named_definition(te)
     te.is_a?(Model::AccessExpression) && (left = te.left_expr).is_a?(Model::QualifiedReference) ? left.cased_value : nil
   end
+
+  def several_paths?
+    false
+  end
 end
 end
 end

data/lib/puppet/pops/loaders.rb

@@ -63,6 +63,15 @@ class Loaders
     # 4. module loaders are set up from the create_environment_loader, they register themselves
   end
 
+  # Called after loader has been added to Puppet Context as :loaders so that dynamic types can
+  # be pre-loaded with a fully configured loader system
+  def pre_load
+    @puppet_system_loader.load(:type, 'error')
+
+    # Will move to Bolt
+    @puppet_system_loader.load(:type, 'executionresult')
+  end
+
   # Clears the cached static and puppet_system loaders (to enable testing)
   #
   def self.clear
@@ -84,7 +93,11 @@ class Loaders
 
   def self.static_loader
     # The static loader can only be changed after a reboot
-    @@static_loader ||= Loader::StaticLoader.new()
+    if !class_variable_defined?(:@@static_loader) || @@static_loader.nil?
+      @@static_loader = Loader::StaticLoader.new()
+      @@static_loader.register_aliases
+    end
+    @@static_loader
   end
 
   def self.implementation_registry
@@ -93,7 +106,7 @@ class Loaders
   end
 
   def register_implementations(obj_classes, name_authority)
-    self.class.register_implementations_with_loader(obj_classes, name_authority, loader = @private_environment_loader)
+    self.class.register_implementations_with_loader(obj_classes, name_authority, @private_environment_loader)
   end
 
   # Register implementations using the global static loader
@@ -233,8 +246,101 @@ class Loaders
     @loaders_by_name[name] = loader
   end
 
+  # Load the main manifest for the given environment
+  #
+  # There are two sources that can be used for the initial parse:
+  #
+  #   1. The value of `Puppet[:code]`: Puppet can take a string from
+  #     its settings and parse that as a manifest. This is used by various
+  #     Puppet applications to read in a manifest and pass it to the
+  #     environment as a side effect. This is attempted first.
+  #   2. The contents of the environment's +manifest+ attribute: Puppet will
+  #     try to load the environment manifest. The manifest must be a file.
+  #
+  # @return [Model::Program] The manifest parsed into a model object
+  def load_main_manifest
+    parser = Parser::EvaluatingParser.singleton
+    parsed_code = Puppet[:code]
+    program = if parsed_code != ""
+      parser.parse_string(parsed_code, 'unknown-source-location')
+    else
+      file = @environment.manifest
+
+      # if the manifest file is a reference to a directory, parse and combine
+      # all .pp files in that directory
+      if file == Puppet::Node::Environment::NO_MANIFEST
+        nil
+      elsif File.directory?(file)
+        raise Puppet::Error, "manifest of environment '#{@environment.name}' appoints directory '#{file}'. It must be a file"
+      elsif File.exists?(file)
+        parser.parse_file(file)
+      else
+        raise Puppet::Error, "manifest of environment '#{@environment.name}' appoints '#{file}'. It does not exist"
+      end
+    end
+    instantiate_definitions(program, public_environment_loader) unless program.nil?
+    program
+  rescue Puppet::ParseErrorWithIssue => detail
+    detail.environment = @environment.name
+    raise
+  rescue => detail
+    msg = _('Could not parse for environment %{env}: %{detail}') % { env: @environment, detail: detail }
+    error = Puppet::Error.new(msg)
+    error.set_backtrace(detail.backtrace)
+    raise error
+  end
+
+  # Add 4.x definitions found in the given program to the given loader.
+  def instantiate_definitions(program, loader)
+    program.definitions.each { |d| instantiate_definition(d, loader) }
+    nil
+  end
+
+  # Add given 4.x definition to the given loader.
+  def instantiate_definition(definition, loader)
+    case definition
+    when Model::PlanDefinition
+      instantiate_PlanDefinition(definition, loader)
+    when Model::FunctionDefinition
+      instantiate_FunctionDefinition(definition, loader)
+    when Model::TypeAlias
+      instantiate_TypeAlias(definition, loader)
+    when Model::TypeMapping
+      instantiate_TypeMapping(definition, loader)
+    else
+      raise Puppet::ParseError, "Internal Error: Unknown type of definition - got '#{definition.class}'"
+    end
+  end
+
   private
 
+  def instantiate_PlanDefinition(plan_definition, loader)
+    typed_name, f = Loader::PuppetPlanInstantiator.create_from_model(plan_definition, loader)
+    loader.set_entry(typed_name, f, plan_definition.locator.to_uri(plan_definition))
+    nil
+  end
+
+  def instantiate_FunctionDefinition(function_definition, loader)
+    # Instantiate Function, and store it in the loader
+    typed_name, f = Loader::PuppetFunctionInstantiator.create_from_model(function_definition, loader)
+    loader.set_entry(typed_name, f, function_definition.locator.to_uri(function_definition))
+    nil
+  end
+
+  def instantiate_TypeAlias(type_alias, loader)
+    # Bind the type alias to the loader using the alias
+    Puppet::Pops::Loader::TypeDefinitionInstantiator.create_from_model(type_alias, loader)
+    nil
+  end
+
+  def instantiate_TypeMapping(type_mapping, loader)
+    tf = Types::TypeParser.singleton
+    lhs = tf.interpret(type_mapping.type_expr, loader)
+    rhs = tf.interpret_any(type_mapping.mapping_expr, loader)
+    implementation_registry.register_type_mapping(lhs, rhs, loader)
+    nil
+  end
+
   def create_puppet_system_loader()
     Loader::ModuleLoaders.system_loader_from(static_loader, self)
   end
@@ -259,15 +365,20 @@ class Loaders
     env_conf = Puppet.lookup(:environments).get_conf(environment.name)
     env_path = env_conf.nil? || !env_conf.is_a?(Puppet::Settings::EnvironmentConf) ? nil : env_conf.path_to_env
 
-    # Create the 3.x resource type loader
-    @runtime3_type_loader = add_loader_by_name(Loader::Runtime3TypeLoader.new(puppet_system_loader, self, environment, env_conf.nil? ? nil : env_path))
-
-    if env_path.nil?
-      # Not a real directory environment, cannot work as a module TODO: Drop when legacy env are dropped?
-      loader = add_loader_by_name(Loader::SimpleEnvironmentLoader.new(@runtime3_type_loader, Loader::ENVIRONMENT))
+    if Puppet[:tasks]
+      loader = Loader::ModuleLoaders.environment_loader_from(puppet_system_loader, self, env_path)
     else
-      # View the environment as a module to allow loading from it - this module is always called 'environment'
-      loader = Loader::ModuleLoaders.environment_loader_from(@runtime3_type_loader, self, env_path)
+      # Create the 3.x resource type loader
+      static_loader.runtime_3_init
+      @runtime3_type_loader = add_loader_by_name(Loader::Runtime3TypeLoader.new(puppet_system_loader, self, environment, env_conf.nil? ? nil : env_path))
+
+      if env_path.nil?
+        # Not a real directory environment, cannot work as a module TODO: Drop when legacy env are dropped?
+        loader = add_loader_by_name(Loader::SimpleEnvironmentLoader.new(@runtime3_type_loader, Loader::ENVIRONMENT))
+      else
+        # View the environment as a module to allow loading from it - this module is always called 'environment'
+        loader = Loader::ModuleLoaders.environment_loader_from(@runtime3_type_loader, self, env_path)
+      end
     end
 
     # An environment has a module path even if it has a null loader
@@ -383,7 +494,7 @@ class Loaders
         nil
       else
         module_data.private_loader =
-          if module_data.restrict_to_dependencies?
+          if module_data.restrict_to_dependencies? && !Puppet[:tasks]
             create_loader_with_only_dependencies_visible(module_data)
           else
             create_loader_with_all_modules_visible(module_data)

data/lib/puppet/pops/lookup/data_dig_function_provider.rb

@@ -71,7 +71,7 @@ class V3BackendFunctionProvider < DataDigFunctionProvider
     # A merge_behavior retrieved from hiera.yaml must not be converted here. Instead, passing the symbol :hash
     # tells the V3 backend to pick it up from the config.
     resolution_type = lookup_invocation.hiera_v3_merge_behavior? ? :hash : convert_merge(merge)
-    @backend.lookup(key.to_s, lookup_invocation.scope, lookup_invocation.hiera_v3_location_overrides, resolution_type, context = {:recurse_guard => nil})
+    @backend.lookup(key.to_s, lookup_invocation.scope, lookup_invocation.hiera_v3_location_overrides, resolution_type, {:recurse_guard => nil})
   end
 
   def full_name

data/lib/puppet/pops/lookup/interpolation.rb

@@ -125,7 +125,7 @@ module Interpolation
     interpolate_method
   end
 
-  # Because the semanitcs of Puppet::Parser::Scope#include? differs from Hash#include?
+  # Because the semantics of Puppet::Parser::Scope#include? differs from Hash#include?
   def nil_in_scope?(scope, key)
     if scope.is_a?(Hash)
       scope.include?(key)

data/lib/puppet/pops/lookup/lookup_adapter.rb

@@ -48,7 +48,6 @@ class LookupAdapter < DataAdapter
 
     key = LookupKey.new(key)
     lookup_invocation.lookup(key, key.module_name) do
-      merge_explained = false
       if lookup_invocation.only_explain_options?
         catch(:no_such_key) { do_lookup(LookupKey::LOOKUP_OPTIONS, lookup_invocation, HASH) }
         nil

data/lib/puppet/pops/model/ast.pp

@@ -377,6 +377,9 @@ type Puppet::AST = TypeSet[{
         }
       }
     }],
+    PlanDefinition => Object[{
+      parent => FunctionDefinition,
+    }],
     LambdaExpression => Object[{
       parent => Expression,
       attributes => {