puppet 3.4.3 → 3.5.0.rc1

data/lib/puppet/pops/parser/lexer_support.rb

@@ -0,0 +1,107 @@
+# This is an integral part of the Lexer. It is broken out into a separate module
+# for maintainability of the code, and making the various parts of the lexer focused.
+#
+module Puppet::Pops::Parser::LexerSupport
+
+  # Formats given message by appending file, line and position if available.
+  def positioned_message(msg, pos = nil)
+    result = [msg]
+    file = @locator.file
+    line = @locator.line_for_offset(pos || @scanner.pos)
+    pos =  @locator.pos_on_line(pos || @scanner.pos)
+
+    result << "in file #{file}" if file && file.is_a?(String) && !file.empty?
+    result << "at line #{line}:#{pos}"
+    result.join(" ")
+  end
+
+  # Returns "<eof>" if at end of input, else the following 5 characters with \n \r \t escaped
+  def followed_by
+    return "<eof>" if @scanner.eos?
+    result = @scanner.rest[0,5] + "..."
+    result.gsub!("\t", '\t')
+    result.gsub!("\n", '\n')
+    result.gsub!("\r", '\r')
+    result
+  end
+
+  # Returns a quoted string using " or ' depending on the given a strings's content
+  def format_quote(q)
+    if q == "'"
+      '"\'"'
+    else
+      "'#{q}'"
+    end
+  end
+
+  # Raises a Puppet::LexError with the given message
+  def lex_error_without_pos msg
+    raise Puppet::LexError.new(msg)
+  end
+
+  # Raises a Puppet::LexError with the given message
+  def lex_error(msg, pos=nil)
+    raise Puppet::LexError.new(positioned_message(msg, pos))
+  end
+
+  # Asserts that the given string value is a float, or an integer in decimal, octal or hex form.
+  # An error is raised if the given value does not comply.
+  #
+  def assert_numeric(value, length)
+    if value =~ /^0[xX].*$/
+      lex_error("Not a valid hex number #{value}", length)     unless value =~ /^0[xX][0-9A-Fa-f]+$/
+
+    elsif value =~ /^0[^.].*$/
+      lex_error("Not a valid octal number #{value}", length)   unless value =~ /^0[0-7]+$/
+
+    else
+      lex_error("Not a valid decimal number #{value}", length) unless value =~ /0?\d+(?:\.\d+)?(?:[eE]-?\d+)?/
+    end
+  end
+
+  # A TokenValue keeps track of the token symbol, the lexed text for the token, its length
+  # and its position in its source container. There is a cost associated with computing the
+  # line and position on line information.
+  #
+  class TokenValue < Puppet::Pops::Parser::Locatable
+    attr_reader :token_array
+    attr_reader :offset
+    attr_reader :locator
+
+    def initialize(token_array, offset, locator)
+      @token_array = token_array
+      @offset = offset
+      @locator = locator
+    end
+
+    def length
+      @token_array[2]
+    end
+
+    def [](key)
+      case key
+      when :value
+        @token_array[1]
+      when :file
+        @locator.file
+      when :line
+        @locator.line_for_offset(@offset)
+      when :pos
+        @locator.pos_on_line(@offset)
+      when :length
+        @token_array[2]
+      when :locator
+        @locator
+      when :offset
+        @offset
+      else
+        nil
+      end
+    end
+
+    # TODO: Make this comparable for testing
+    # vs symbolic, vs array with symbol and non hash, array with symbol and hash)
+    #
+  end
+
+end

data/lib/puppet/pops/parser/locatable.rb

@@ -0,0 +1,23 @@
+# Interface for something that is "locateable" (holds offset and length).
+class Puppet::Pops::Parser::Locatable
+
+  # The offset in the locator's content
+  def offset
+  end
+
+  # The length in the locator from the given offset
+  def length
+  end
+
+  # This class is useful for testing
+  class Fixed < Puppet::Pops::Parser::Locatable
+    attr_reader :offset
+    attr_reader :length
+
+    def initialize(offset, length)
+      @offset = offset
+      @length = length
+    end
+  end
+
+end

data/lib/puppet/pops/parser/locator.rb

@@ -0,0 +1,291 @@
+# Helper class that keeps track of where line breaks are located and can answer questions about positions.
+#
+class Puppet::Pops::Parser::Locator
+
+  RUBY_1_9_3 = (1 << 16 | 9 << 8 | 3)
+  RUBY_2_0_0 = (2 << 16 | 0 << 8 | 0)
+  RUBYVER_ARRAY = RUBY_VERSION.split(".").collect {|s| s.to_i }
+  RUBYVER = (RUBYVER_ARRAY[0] << 16 | RUBYVER_ARRAY[1] << 8 | RUBYVER_ARRAY[2])
+
+  # Computes a symbol representing which ruby runtime this is running on
+  # This implementation will fail if there are more than 255 minor or micro versions of ruby
+  #
+  def self.locator_version
+    if RUBYVER >= RUBY_2_0_0
+      :ruby20
+    elsif RUBYVER >= RUBY_1_9_3
+      :ruby19
+    else
+      :ruby18
+    end
+  end
+  LOCATOR_VERSION = locator_version
+
+  # Constant set to true if multibyte is supported (includes multibyte extended regular expressions)
+  MULTIBYTE = !!(LOCATOR_VERSION == :ruby19 || LOCATOR_VERSION == :ruby20)
+
+  # Creates, or recreates a Locator. A Locator is created if index is not given (a scan is then
+  # performed of the given source string.
+  #
+  def self.locator(string, file, index = nil)
+    case LOCATOR_VERSION
+    when :ruby20, :ruby19
+      Locator19.new(string, file, index)
+    else
+      Locator18.new(string, file, index)
+    end
+  end
+
+  # Returns the file name associated with the string content
+  def file
+  end
+
+  # Returns the string content
+  def string
+  end
+
+  # Returns the position on line (first position on a line is 1)
+  def pos_on_line(offset)
+  end
+
+  # Returns the line number (first line is 1) for the given offset
+  def line_for_offset(offset)
+  end
+
+  # Returns the offset on line (first offset on a line is 0).
+  #
+  def offset_on_line(offset)
+  end
+
+  # Returns the character offset for a given reported offset
+  def char_offset(byte_offset)
+  end
+
+  # Returns the length measured in number of characters from the given start and end reported offseta
+  def char_length(offset, end_offset)
+  end
+
+  # Returns the line index - an array of line offsets for the start position of each line, starting at 0 for
+  # the first line.
+  #
+  def line_index()
+  end
+
+  # A Sublocator locates a concrete locator (subspace) in a virtual space.
+  # The `leading_line_count` is the (virtual) number of lines preceding the first line in the concrete locator.
+  # The `leading_offset` is the (virtual) byte offset of the first byte in the concrete locator.
+  # The `leading_line_offset` is the (virtual) offset / margin in characters for each line.
+  #
+  # This illustrates characters in the sublocator (`.`) inside the subspace (`X`):
+  #
+  #      1:XXXXXXXX
+  #      2:XXXX.... .. ... ..
+  #      3:XXXX. . .... ..
+  #      4:XXXX............
+  #
+  # This sublocator would be configured with leading_line_count = 1,
+  # leading_offset=8, and leading_line_offset=4
+  #
+  # Note that leading_offset must be the same for all lines and measured in characters.
+  #
+  class SubLocator < Puppet::Pops::Parser::Locator
+    attr_reader :locator
+    attr_reader :leading_line_count
+    attr_reader :leading_offset
+    attr_reader :leading_line_offset
+
+    def self.sub_locator(string, file, leading_line_count, leading_offset, leading_line_offset)
+      self.new(Puppet::Pops::Parser::Locator.locator(string, file),
+        leading_line_count,
+        leading_offset,
+        leading_line_offset)
+    end
+
+    def initialize(locator, leading_line_count, leading_offset, leading_line_offset)
+      @locator = locator
+      @leading_line_count = leading_line_count
+      @leading_offset = leading_offset
+      @leading_line_offset = leading_line_offset
+    end
+
+    def file
+      @locator.file
+    end
+
+    def string
+      @locator.string
+    end
+
+    # Given offset is offset in the subspace
+    def line_for_offset(offset)
+      @locator.line_for_offset(offset) + @leading_line_count
+    end
+
+    # Given offset is offset in the subspace
+    def offset_on_line(offset)
+      @locator.offset_on_line(offset) + @leading_line_offset
+    end
+
+    # Given offset is offset in the subspace
+    def char_offset(offset)
+      effective_line = @locator.line_for_offset(offset)
+      locator.char_offset(offset) + (effective_line * @leading_line_offset) + @leading_offset
+    end
+
+    # Given offsets are offsets in the subspace
+    def char_length(offset, end_offset)
+      effective_line = @locator.line_for_offset(end_offset) - @locator.line_for_offset(offset)
+      locator.char_length(offset, end_offset) + (effective_line * @leading_line_offset)
+    end
+
+    def pos_on_line(offset)
+      offset_on_line(offset) +1
+    end
+  end
+
+  private
+
+  class AbstractLocator < Puppet::Pops::Parser::Locator
+    attr_accessor :line_index
+    attr_accessor :string
+    attr_accessor :prev_offset
+    attr_accessor :prev_line
+    attr_reader   :string
+    attr_reader   :file
+
+    # Create a locator based on a content string, and a boolean indicating if ruby version support multi-byte strings
+    # or not.
+    #
+    def initialize(string, file, index = nil)
+      @string = string.freeze
+      @file = file.freeze
+      @prev_offset = nil
+      @prev_line = nil
+      @line_index = index
+      compute_line_index unless !index.nil?
+    end
+
+    # Returns the position on line (first position on a line is 1)
+    def pos_on_line(offset)
+      offset_on_line(offset) +1
+    end
+
+    def to_location_hash(reported_offset, end_offset)
+      pos        = pos_on_line(reported_offset)
+      offset     = char_offset(reported_offset)
+      length     = char_length(reported_offset, end_offset)
+      start_line = line_for_offset(reported_offset)
+      { :line => start_line, :pos => pos, :offset => offset, :length => length}
+    end
+
+    # Returns the index of the smallest item for which the item > the given value
+    # This is a min binary search. Although written in Ruby it is only slightly slower than
+    # the corresponding method in C in Ruby 2.0.0 - the main benefit to use this method over
+    # the Ruby C version is that it returns the index (not the value) which means there is not need
+    # to have an additional structure to get the index (or record the index in the structure). This
+    # saves both memory and CPU. It also does not require passing a block that is called since this
+    # method is specialized to search the line index.
+    #
+    def ary_bsearch_i(ary, value)
+      low = 0
+      high = ary.length
+      mid = nil
+      smaller = false
+      satisfied = false
+      v = nil
+
+      while low < high do
+          mid = low + ((high - low) / 2)
+          v = (ary[mid] > value)
+          if v == true
+            satisfied = true
+            smaller = true
+          elsif !v
+            smaller = false
+          else
+            raise TypeError, "wrong argument, must be boolean or nil, got '#{v.class}'"
+          end
+
+          if smaller
+            high = mid
+          else
+            low = mid + 1;
+          end
+      end
+
+      return nil if low == ary.length
+      return nil if !satisfied
+      return low
+    end
+
+    # Common impl for 18 and 19 since scanner is byte based
+    def compute_line_index
+      scanner = StringScanner.new(string)
+      result = [0] # first line starts at 0
+      while scanner.scan_until(/\n/)
+        result << scanner.pos
+      end
+      self.line_index = result.freeze
+    end
+
+    # Returns the line number (first line is 1) for the given offset
+    def line_for_offset(offset)
+      if prev_offset == offset
+        # use cache
+        return prev_line
+      end
+      if line_nbr = ary_bsearch_i(line_index, offset)
+        # cache
+        prev_offset = offset
+        prev_line = line_nbr
+        return line_nbr
+      end
+      # If not found it is after last
+      # clear cache
+      prev_offset = prev_line = nil
+      return line_index.size
+    end
+  end
+
+  class Locator18 < AbstractLocator
+
+    def offset_on_line(offset)
+      line_offset = line_index[ line_for_offset(offset)-1 ]
+      offset - line_offset
+    end
+
+    def char_offset(char_offset)
+      char_offset
+    end
+
+    def char_length(offset, end_offset)
+      end_offset - offset
+    end
+
+  end
+
+  # This implementation is for Ruby19 and Ruby20. It uses byteslice to get strings from byte based offsets.
+  # For Ruby20 this is faster than using the Stringscanner.charpos method (byteslice outperforms it, when
+  # strings are frozen).
+  #
+  class Locator19 < AbstractLocator
+
+    # Returns the offset on line (first offset on a line is 0).
+    # Ruby 19 is multibyte but has no character position methods, must use byteslice
+    def offset_on_line(offset)
+      line_offset = line_index[ line_for_offset(offset)-1 ]
+      string.byteslice(line_offset, offset-line_offset).length
+    end
+
+    # Returns the character offset for a given byte offset
+    # Ruby 19 is multibyte but has no character position methods, must use byteslice
+    def char_offset(byte_offset)
+      string.byteslice(0, byte_offset).length
+    end
+
+    # Returns the length measured in number of characters from the given start and end byte offseta
+    def char_length(offset, end_offset)
+      string.byteslice(offset, end_offset - offset).length
+    end
+  end
+end

data/lib/puppet/pops/parser/makefile

@@ -1,13 +1,6 @@
 
-#parser.rb: grammar.ra
-#	racc -o$@ grammar.ra
-#
-#grammar.output: grammar.ra
-#	racc -v -o$@ grammar.ra
-#
-
 eparser.rb: egrammar.ra
 	racc -o$@ egrammar.ra
 
 egrammar.output: egrammar.ra
-	racc -v -o$@ egrammar.ra
+	racc -v -o$@ egrammar.ra

data/lib/puppet/pops/parser/parser_support.rb

@@ -24,6 +24,7 @@ class Puppet::Pops::Parser::Parser
   include Puppet::Resource::TypeCollectionHelper
 
   attr_accessor :lexer
+  attr_reader :definitions
 
   # Returns the token text of the given lexer token, or nil, if token is nil
   def token_text t
@@ -42,28 +43,28 @@ class Puppet::Pops::Parser::Parser
   # before evaluation-time.
   #
   def classname(name)
-    [@lexer.namespace, name].join("::").sub(/^::/, '')
+    [namespace, name].join("::").sub(/^::/, '')
   end
 
-  # Reinitializes variables (i.e. creates a new lexer instance
-  #
-  def clear
-    initvars
-  end
+#  # Reinitializes variables (i.e. creates a new lexer instance
+#  #
+#  def clear
+#    initvars
+#  end
 
   # Raises a Parse error.
-  def error(message, options = {})
+  def error(value, message, options = {})
     except = Puppet::ParseError.new(message)
-    except.line = options[:line] || @lexer.line
-    except.file = options[:file] || @lexer.file
-    except.pos = options[:pos]   || @lexer.pos
+    except.line = options[:line] || value[:line]
+    except.file = options[:file] || value[:file] # @lexer.file
+    except.pos = options[:pos]   || value[:pos] # @lexer.pos
 
     raise except
   end
 
   # Parses a file expected to contain pp DSL logic.
   def parse_file(file)
-    unless Puppet::FileSystem::File.exist?(file)
+    unless Puppet::FileSystem.exist?(file)
       unless file =~ /\.pp$/
         file = file + ".pp"
       end
@@ -77,15 +78,16 @@ class Puppet::Pops::Parser::Parser
     # and there is no syntax that requires knowing if something referenced exists, it is safe
     # to assume that no environment is needed when parsing. (All that comes later).
     #
-    initvars
+    @lexer = Puppet::Pops::Parser::Lexer2.new
+    @namestack = []
+    @definitions = []
   end
 
-  # Initializes the parser support by creating a new instance of {Puppet::Pops::Parser::Lexer}
-  # @return [void]
-  #
-  def initvars
-    @lexer = Puppet::Pops::Parser::Lexer.new
-  end
+#  # Initializes the parser support by creating a new instance of {Puppet::Pops::Parser::Lexer}
+#  # @return [void]
+#  #
+#  def initvars
+#  end
 
   # This is a callback from the generated grammar (when an error occurs while parsing)
   # TODO Picks up origin information from the lexer, probably needs this from the caller instead
@@ -93,26 +95,36 @@ class Puppet::Pops::Parser::Parser
   #
   def on_error(token,value,stack)
     if token == 0 # denotes end of file
-      value = 'end of file'
+      value_at = 'end of file'
     else
-      value = "'#{value[:value]}'"
+      value_at = "'#{value[:value]}'"
     end
-    error = "Syntax error at #{value}"
+    error = "Syntax error at #{value_at}"
 
     # The 'expected' is only of value at end of input, otherwise any parse error involving a
-    # start of a pair will be reported as expecting the close of the pair - e.g. "$x.each |$x {", would
+    # start of a pair will be reported as expecting the close of the pair - e.g. "$x.each |$x {|", would
     # report that "seeing the '{', the '}' is expected. That would be wrong.
     # Real "expected" tokens are very difficult to compute (would require parsing of racc output data). Output of the stack
     # could help, but can require extensive backtracking and produce many options.
     #
-    if token == 0 && brace = @lexer.expected
-      error += "; expected '#{brace}'"
-    end
+    # The lexer should handle the "expected instead of end of file for strings, and interpolation", other expectancies
+    # must be handled by the grammar. The lexer may have enqueued tokens far ahead - the lexer's opinion about this
+    # is not trustworthy.
+    #
+#    if token == 0 && brace = @lexer.expected
+#      error += "; expected '#{brace}'"
+#    end
 
     except = Puppet::ParseError.new(error)
-    except.line = @lexer.line
-    except.file = @lexer.file if @lexer.file
-    except.pos  = @lexer.pos
+    if token != 0
+      path        = value[:file]
+      except.line = value[:line]
+      except.pos  = value[:pos]
+    else
+      # At end of input, use what the lexer thinks is the source file
+      path        = lexer.file
+    end
+    except.file = path if path.is_a?(String) && !path.empty?
 
     raise except
   end
@@ -130,8 +142,12 @@ class Puppet::Pops::Parser::Parser
   # @return [Puppet::Pops::Model::Factory] the given factory
   # @api private
   #
-  def loc(factory, start_token, end_token = nil)
-    factory.record_position(sourcepos(start_token), sourcepos(end_token))
+  def loc(factory, start_locateable, end_locateable = nil)
+    factory.record_position(start_locateable, end_locateable)
+  end
+
+  def heredoc_loc(factory, start_locateabke, end_locateable = nil)
+    factory.record_heredoc_position(start_locatable, end_locatable)
   end
 
   # Associate documentation with the factory wrapped model object.
@@ -141,28 +157,28 @@ class Puppet::Pops::Parser::Parser
     factory.doc = doc_string
   end
 
-  def sourcepos(o)
-    if !o
-      Puppet::Pops::Adapters::SourcePosAdapter.new
-    elsif o.is_a? Puppet::Pops::Model::Factory
-      # It is a built model element with loc set returns start at pos 0
-      o.loc
-    else
-      loc = Puppet::Pops::Adapters::SourcePosAdapter.new
-      # It must be a token
-      loc.line = o[:line]
-      loc.pos = o[:pos]
-      loc.offset = o[:offset]
-      loc.length = o[:length]
-      loc
-    end
-  end
-
   def aryfy(o)
     o = [o] unless o.is_a?(Array)
     o
   end
 
+  def namespace
+    @namestack.join('::')
+  end
+
+  def namestack(name)
+    @namestack << name
+  end
+
+  def namepop()
+    @namestack.pop
+  end
+
+  def add_definition(definition)
+    @definitions << definition.current
+    definition
+  end
+
   # Transforms an array of expressions containing literal name expressions to calls if followed by an
   # expression, or expression list
   #
@@ -170,6 +186,17 @@ class Puppet::Pops::Parser::Parser
     Factory.transform_calls(expressions)
   end
 
+  # Transforms a LEFT followed by the result of attribute_operations, this may be a call or an invalid sequence
+  def transform_resource_wo_title(left, resource)
+    Factory.transform_resource_wo_title(left, resource)
+  end
+
+  # If there are definitions that require initialization a Program is produced, else the body
+  def create_program(body)
+    locator = @lexer.locator
+    Factory.PROGRAM(body, definitions, locator)
+  end
+
   # Performs the parsing and returns the resulting model.
   # The lexer holds state, and this is setup with {#parse_string}, or {#parse_file}.
   #
@@ -194,10 +221,11 @@ class Puppet::Pops::Parser::Parser
       #      rescue => except
       #        raise Puppet::ParseError.new(except.message, @lexer.file, @lexer.line, @lexer.pos, except)
     end
-    main.record_origin(@lexer.file) if main
     return main
   ensure
     @lexer.clear
+    @namestack = []
+    @definitions = []
   end
 
 end