puppet 4.10.12 → 5.0.0

data/lib/puppet/pops/parser/evaluating_parser.rb

@@ -25,7 +25,7 @@ class EvaluatingParser
     # Also a possible improvement (if the YAML parser returns positions) is to provide correct output of position.
     #
     begin
-      assert_and_report(parser.parse_string(s, file_source))
+      assert_and_report(parser.parse_string(s, file_source)).model
     rescue Puppet::ParseErrorWithIssue => e
       raise e
     rescue Puppet::ParseError => e
@@ -38,7 +38,7 @@ class EvaluatingParser
   def parse_file(file)
     @file_source = file
     clear()
-    assert_and_report(parser.parse_file(file))
+    assert_and_report(parser.parse_file(file)).model
   end
 
   def evaluate_string(scope, s, file_source = nil)
@@ -99,10 +99,10 @@ class EvaluatingParser
 
   def assert_and_report(parse_result)
     return nil unless parse_result
-    if parse_result.source_ref.nil? || parse_result.source_ref == ''
-      parse_result.source_ref = @file_source
+    if parse_result['source_ref'].nil? || parse_result['source_ref'] == ''
+      parse_result['source_ref'] = @file_source
     end
-    validation_result = validate(parse_result)
+    validation_result = validate(parse_result.model)
 
     IssueReporter.assert_and_report(validation_result,
                                           :emit_warnings => true)

data/lib/puppet/pops/parser/heredoc_support.rb

@@ -133,7 +133,7 @@ module HeredocSupport
       lines = lines.collect {|s| s.gsub(leading_pattern, '') }
     end
     result = lines.join('')
-    result.gsub!(/\r?\n\z/m, '') if remove_break
+    result.gsub!(/\r?\n$/, '') if remove_break
     result
   end
 

data/lib/puppet/pops/parser/interpolation_support.rb

@@ -40,7 +40,7 @@ module Puppet::Pops::Parser::InterpolationSupport
         if varname = scn.scan(PATTERN_VARIABLE)
           # The $ is counted towards the variable
           enqueue_completed([:DQPRE, text, after-before-1], before)
-          enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after -1)
+          enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after - 1)
           break
         else
           # false $ variable start
@@ -81,7 +81,7 @@ module Puppet::Pops::Parser::InterpolationSupport
         if varname = scn.scan(PATTERN_VARIABLE)
           # The $ is counted towards the variable
           enqueue_completed([:DQMID, text, after-before-1], before)
-          enqueue_completed([:VARIABLE, varname, scn.pos - after +1], after -1)
+          enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after - 1)
           break
         else
           # false $ variable start
@@ -125,7 +125,7 @@ module Puppet::Pops::Parser::InterpolationSupport
         if varname = scn.scan(PATTERN_VARIABLE)
           # The $ is counted towards the variable
           enqueue_completed([:DQPRE, text, after-before-1], before)
-          enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after -1)
+          enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after - 1)
           break
         else
           # false $ variable start
@@ -165,7 +165,7 @@ module Puppet::Pops::Parser::InterpolationSupport
         if varname = scn.scan(PATTERN_VARIABLE)
           # The $ is counted towards the variable
           enqueue_completed([:DQMID, text, after-before-1], before)
-          enqueue_completed([:VARIABLE, varname, scn.pos - after +1], after -1)
+          enqueue_completed([:VARIABLE, varname, scn.pos - after + 1], after - 1)
           break
         else
           # false $ variable start

data/lib/puppet/pops/parser/lexer2.rb

@@ -134,42 +134,18 @@ class Lexer2
     'type'     => [:TYPE,     'type',     4],
     'attr'     => [:ATTR,     'attr',     4],
     'private'  => [:PRIVATE,  'private',  7],
+    'application' => [:APPLICATION, 'application',  11],
+    'consumes'    => [:CONSUMES,    'consumes',  8],
+    'produces'    => [:PRODUCES,    'produces',  8],
+    'site'        => [:SITE,        'site',  4],
   }
 
   KEYWORDS.each {|k,v| v[1].freeze; v.freeze }
   KEYWORDS.freeze
 
-  # We maintain two different tables of tokens for the constructs
-  # introduced by application management. Which ones we use is decided in
-  # +initvars+; by selecting one or the other variant, we select whether we
-  # hit the appmgmt-specific code paths
-  APP_MANAGEMENT_TOKENS = {
-    :with_appm => {
-      'application' => [:APPLICATION, 'application',  11],
-      'consumes'    => [:CONSUMES,    'consumes',  8],
-      'produces'    => [:PRODUCES,    'produces',  8],
-      'site'        => [:SITE,        'site',  4]
-    },
-    :without_appm => {
-      'application' => [:APPLICATION_R, 'application',  11],
-      'consumes'    => [:CONSUMES_R,    'consumes',  8],
-      'produces'    => [:PRODUCES_R,    'produces',  8],
-      'site'        => [:SITE_R,        'site',  4]
-    }
-  }
-
-  APP_MANAGEMENT_TOKENS.each do |_, variant|
-    variant.each { |_,v| v[1].freeze; v.freeze }
-    variant.freeze
-  end
-  APP_MANAGEMENT_TOKENS.freeze
-
   # Reverse lookup of keyword name to string
   KEYWORD_NAMES = {}
   KEYWORDS.each {|k, v| KEYWORD_NAMES[v[0]] = k }
-  APP_MANAGEMENT_TOKENS.each do |_, variant|
-    variant.each { |k,v| KEYWORD_NAMES[v[0]] = k }
-  end
   KEYWORD_NAMES.freeze
 
   PATTERN_WS        = %r{[[:blank:]\r]+}
@@ -552,7 +528,7 @@ class Lexer2
         before = scn.pos
         value = scn.scan(PATTERN_BARE_WORD)
         if value && value =~ PATTERN_NAME
-          emit_completed(KEYWORDS[value] || @appm_keywords[value] || [:NAME, value.freeze, scn.pos - before], before)
+          emit_completed(KEYWORDS[value] || [:NAME, value.freeze, scn.pos - before], before)
         elsif value
           emit_completed([:WORD, value.freeze, scn.pos - before], before)
         else
@@ -630,10 +606,10 @@ class Lexer2
   # overloading of = does not allow passing more than one argument).
   #
   def string=(string)
-    lex_string(string, '')
+    lex_string(string, nil)
   end
 
-  def lex_string(string, path='')
+  def lex_string(string, path=nil)
     initvars
     assert_not_bom(string)
     @scanner = StringScanner.new(string)
@@ -687,8 +663,6 @@ class Lexer2
       :after => nil,
       :line_lexical_start => 0
     }
-    appm_mode = Puppet[:app_management] ? :with_appm : :without_appm
-    @appm_keywords = APP_MANAGEMENT_TOKENS[appm_mode]
   end
 
   # Scans all of the content and returns it in an array

data/lib/puppet/pops/parser/locator.rb

@@ -22,6 +22,10 @@ class Locator
   def string
   end
 
+  def to_s
+    "Locator for file #{file}"
+  end
+
   # Returns the position on line (first position on a line is 1)
   def pos_on_line(offset)
   end
@@ -52,12 +56,37 @@ class Locator
   def extract_text(offset, length)
   end
 
+  def extract_tree_text(ast)
+    first = ast.offset
+    last = first + ast.length
+    ast._pcore_all_contents([]) do |m|
+      next unless m.is_a?(Model::Positioned)
+      m_offset = m.offset
+      m_last = m_offset + m.length
+      first = m_offset if m_offset < first
+      last = m_last if m_last > last
+    end
+    extract_text(first, last - first)
+  end
+
   # Returns the line index - an array of line offsets for the start position of each line, starting at 0 for
   # the first line.
   #
   def line_index()
   end
 
+  # Produces an URI with path?line=n&pos=n. If origin is unknown the URI is string:?line=n&pos=n
+  def to_uri(ast)
+    f = file
+    if f.nil? || f.empty?
+      f = 'string:'
+    else
+      f = Puppet::Util.path_to_uri(f).to_s
+    end
+    offset = ast.offset
+    URI("#{f}?line=#{line_for_offset(offset).to_s}&pos=#{pos_on_line(offset).to_s}")
+  end
+
   # A Sublocator locates a concrete locator (subspace) in a virtual space.
   # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
   # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
@@ -273,23 +302,17 @@ class Locator
   class Locator19 < AbstractLocator
     include Types::PuppetObject
 
-    def self._ptype
-      @type
-    end
-
-    # The Locator is not part of the Ecore model so no ObjectType is automatically inferred. Instead the
-    # type is explicitly added here.
-    # TODO: LocatorForChars is never added. It looks like it could be removed (remnant from Ruby 1.8 compatibility?)
-    # @api private
-    def self.register_ptype(loader, ir)
-      @type = Pcore::create_object_type(loader, ir, self, 'Puppet::AST::Locator', 'Any',
-        'string' => Types::PStringType::DEFAULT,
-        'file' => Types::PStringType::DEFAULT,
-        'line_index' => {
-          Types::KEY_TYPE => Types::POptionalType.new(Types::PArrayType.new(Types::PIntegerType::DEFAULT)),
-          Types::KEY_VALUE => nil
+    def self._pcore_type
+      @type ||= Types::PObjectType.new('Puppet::AST::Locator', {
+        'attributes' => {
+          'string' => Types::PStringType::DEFAULT,
+          'file' => Types::PStringType::DEFAULT,
+          'line_index' => {
+            Types::KEY_TYPE => Types::POptionalType.new(Types::PArrayType.new(Types::PIntegerType::DEFAULT)),
+            Types::KEY_VALUE => nil
+          }
         }
-      ).resolve(Types::TypeParser.singleton, loader)
+      })
     end
 
     # Returns the offset on line (first offset on a line is 0).

data/lib/puppet/pops/parser/parser_support.rb

@@ -27,11 +27,14 @@ class Parser
   # Returns the token text of the given lexer token, or nil, if token is nil
   def token_text t
     return t if t.nil?
-    t = t.current if t.respond_to?(:current)
-    return t.value if t.is_a? Model::QualifiedName
-
-    # else it is a lexer token
-    t[:value]
+    if t.is_a?(Factory) && t.model_class <= Model::QualifiedName
+      t['value']
+    elsif t.is_a?(Model::QualifiedName)
+      t.value
+    else
+      # else it is a lexer token
+      t[:value]
+    end
   end
 
   # Produces the fully qualified name, with the full (current) namespace for a given name.
@@ -54,21 +57,15 @@ class Parser
       except.line = semantic[:line];
       except.pos = semantic[:pos];
     else
-      semantic = semantic.current() if semantic.is_a?(Factory)
-
-      # Adapt the model so it is possible to get location information.
-      # The model may not have been added to the source tree, so give it the lexer's locator
-      # directly instead of searching for the root Program where the locator is normally stored.
-      #
-      if semantic.is_a?(Model::Positioned)
-        adapter = Adapters::SourcePosAdapter.adapt(semantic)
-        adapter.locator = @lexer.locator
-      else
-        adapter = nil
+      locator = @lexer.locator
+      except.file = locator.file
+      if semantic.is_a?(Factory)
+        offset = semantic['offset']
+        unless offset.nil?
+          except.line = locator.line_for_offset(offset)
+          except.pos = locator.pos_on_line(offset)
+        end
       end
-      except.file = @lexer.locator.file
-      except.line = adapter.line if adapter
-      except.pos = adapter.pos if adapter
     end
     raise except
   end
@@ -81,7 +78,7 @@ class Parser
       end
     end
     @lexer.file = file
-    _parse()
+    _parse
   end
 
   def initialize()
@@ -129,7 +126,7 @@ class Parser
 
   # Parses a String of pp DSL code.
   #
-  def parse_string(code, path = '')
+  def parse_string(code, path = nil)
     @lexer.lex_string(code, path)
     _parse()
   end
@@ -138,15 +135,15 @@ class Parser
   # @return [Factory] the given factory
   # @api private
   #
-  def loc(factory, start_locateable, end_locateable = nil)
-    factory.record_position(start_locateable, end_locateable)
+  def loc(factory, start_locatable, end_locatable = nil)
+    factory.record_position(@lexer.locator, start_locatable, end_locatable)
   end
 
   # Mark the factory wrapped heredoc model object with location information
   # @return [Factory] the given factory
   # @api private
   #
-  def heredoc_loc(factory, start_locateabke, end_locateable = nil)
+  def heredoc_loc(factory, start_locatable, end_locatable = nil)
     factory.record_heredoc_position(start_locatable, end_locatable)
   end
 
@@ -168,7 +165,7 @@ class Parser
   end
 
   def add_definition(definition)
-    @definitions << definition.current
+    @definitions << definition.model
     definition
   end
 
@@ -194,10 +191,10 @@ class Parser
     rescue Factory::ArgsToNonCallError => e
       # e.args[1] is the first comma token in the list
       # e.name_expr is the function name expression
-      if e.name_expr.is_a?(Model::QualifiedName)
-        error(e.args[1], "attempt to pass argument list to the function '#{e.name_expr.value}' which cannot be called without parentheses")
+      if e.name_expr.is_a?(Factory) && e.name_expr.model_class <= Model::QualifiedName
+        error(e.args[1], _("attempt to pass argument list to the function '%{name}' which cannot be called without parentheses") % { name: e.name_expr['value'] })
       else
-        error(e.args[1], "illegal comma separated argument list")
+        error(e.args[1], _("illegal comma separated argument list"))
       end
     end
   end
@@ -245,7 +242,6 @@ class Parser
     @namestack = []
     @definitions = []
   end
-
 end
 end
 end

data/lib/puppet/pops/patterns.rb

@@ -48,8 +48,7 @@ module Puppet::Pops::Patterns
 
   # VAR_NAME matches the name part of a variable (The $ character is not included)
   # Note, that only the final segment may start with an underscore.
-  # Note, regexp sensitive to backtracking
-  VAR_NAME = %r{\A(?:::)?(?:[a-z]\w*::)*[a-z_]\w*\z}
+  VAR_NAME = %r{\A(?:(::)?[a-z]\w*)*(?:(::)?[a-z_]\w*)\z}
 
   # PARAM_NAME matches the name part of a parameter (The $ character is not included)
   PARAM_NAME = %r{\A[a-z_]\w*\z}

data/lib/puppet/pops/pcore.rb

@@ -2,11 +2,14 @@ require 'uri'
 
 module Puppet::Pops
 module Pcore
+  include Types::PuppetObject
+
   TYPE_URI_RX = Types::TypeFactory.regexp(URI.regexp)
   TYPE_URI = Types::TypeFactory.pattern(TYPE_URI_RX)
   TYPE_URI_ALIAS = Types::PTypeAliasType.new('Pcore::URI', nil, TYPE_URI)
   TYPE_SIMPLE_TYPE_NAME = Types::TypeFactory.pattern(/\A[A-Z]\w*\z/)
   TYPE_QUALIFIED_REFERENCE = Types::TypeFactory.pattern(/\A[A-Z][\w]*(?:::[A-Z][\w]*)*\z/)
+  TYPE_MEMBER_NAME = Types::PPatternType.new([Types::PRegexpType.new(Patterns::PARAM_NAME)])
 
   KEY_PCORE_URI = 'pcore_uri'.freeze
   KEY_PCORE_VERSION = 'pcore_version'.freeze
@@ -17,28 +20,32 @@ module Pcore
 
   RUNTIME_NAME_AUTHORITY = 'http://puppet.com/2016.1/runtime'
 
+  def self._pcore_type
+    @type
+  end
+
+  def self.annotate(instance, annotations_hash)
+    annotations_hash.each_pair do |type, init_hash|
+      type.implementation_class.annotate(instance) { init_hash }
+    end
+    instance
+  end
+
   def self.init(loader, ir, for_agent)
     add_alias('Pcore::URI_RX', TYPE_URI_RX, loader)
     add_type(TYPE_URI_ALIAS, loader)
     add_alias('Pcore::SimpleTypeName', TYPE_SIMPLE_TYPE_NAME, loader)
+    add_alias('Pcore::MemberName', TYPE_MEMBER_NAME, loader)
     add_alias('Pcore::TypeName', TYPE_QUALIFIED_REFERENCE, loader)
     add_alias('Pcore::QRef', TYPE_QUALIFIED_REFERENCE, loader)
-    begin
     Types::TypedModelObject.register_ptypes(loader, ir)
-    rescue Exception => e
-      puts e.message
-    end
+
+    @type = create_object_type(loader, ir, Pcore, 'Pcore', nil)
 
     ir.register_implementation_namespace('Pcore', 'Puppet::Pops::Pcore', loader)
+    ir.register_implementation_namespace('Puppet::AST', 'Puppet::Pops::Model', loader)
+    ir.register_implementation('Puppet::AST::Locator', 'Puppet::Pops::Parser::Locator::Locator19', loader)
     unless for_agent
-      ir.register_implementation_namespace('Puppet::AST', 'Puppet::Pops::Model', loader)
-      ast_type_set = Serialization::RGen::TypeGenerator.new.generate_type_set('Puppet::AST', Puppet::Pops::Model, loader)
-
-      # Extend the Puppet::AST type set with the Locator (it's not an RGen class, but nevertheless, used in the model)
-      ast_ts_i12n = ast_type_set.i12n_hash
-      ast_ts_i12n['types'] = ast_ts_i12n['types'].merge('Locator' => Parser::Locator::Locator19.register_ptype(loader, ir))
-      add_type(Types::PTypeSetType.new(ast_ts_i12n), loader)
-
       Resource.register_ptypes(loader, ir)
       Lookup::Context.register_ptype(loader, ir);
       Lookup::DataProvider.register_types(loader)
@@ -59,17 +66,17 @@ module Pcore
   #
   # @api private
   def self.create_object_type(loader, ir, impl_class, type_name, parent_name, attributes_hash = EMPTY_HASH, functions_hash = EMPTY_HASH, equality = nil)
-    i12n_hash = {}
-    i12n_hash[Types::KEY_PARENT] = Types::PTypeReferenceType.new(parent_name) unless parent_name.nil?
-    i12n_hash[Types::KEY_ATTRIBUTES] = attributes_hash unless attributes_hash.empty?
-    i12n_hash[Types::KEY_FUNCTIONS] = functions_hash unless functions_hash.empty?
-    i12n_hash[Types::KEY_EQUALITY] = equality unless equality.nil?
+    init_hash = {}
+    init_hash[Types::KEY_PARENT] = Types::PTypeReferenceType.new(parent_name) unless parent_name.nil?
+    init_hash[Types::KEY_ATTRIBUTES] = attributes_hash unless attributes_hash.empty?
+    init_hash[Types::KEY_FUNCTIONS] = functions_hash unless functions_hash.empty?
+    init_hash[Types::KEY_EQUALITY] = equality unless equality.nil?
     ir.register_implementation(type_name, impl_class, loader)
-    add_type(Types::PObjectType.new(type_name, i12n_hash), loader)
+    add_type(Types::PObjectType.new(type_name, init_hash), loader)
   end
 
   def self.add_object_type(name, body, loader)
-    add_type(Types::PObjectType.new(name, Parser::EvaluatingParser.new.parse_string(body).current.body), loader)
+    add_type(Types::PObjectType.new(name, Parser::EvaluatingParser.new.parse_string(body).body), loader)
   end
 
   def self.add_alias(name, type, loader, name_authority = RUNTIME_NAME_AUTHORITY)

data/lib/puppet/pops/resource/param.rb

@@ -47,7 +47,7 @@ class Param
     name
   end
 
-  def self._ptype
+  def self._pcore_type
     @ptype
   end
 end