puppet 0.25.5 → 2.6.0

data/lib/puppet/parser/lexer.rb

@@ -5,517 +5,575 @@ require 'puppet'
 
 
 module Puppet
-    class LexError < RuntimeError; end
+  class LexError < RuntimeError; end
 end
 
 module Puppet::Parser; end
 
 class Puppet::Parser::Lexer
-    attr_reader :last, :file, :lexing_context
+  attr_reader :last, :file, :lexing_context, :token_queue
 
-    attr_accessor :line, :indefine
+  attr_accessor :line, :indefine
 
-    # Our base token class.
-    class Token
-        attr_accessor :regex, :name, :string, :skip, :incr_line, :skip_text, :accumulate
+  def lex_error msg
+    raise Puppet::LexError.new(msg)
+  end
 
-        def initialize(regex, name)
-            if regex.is_a?(String)
-                @name, @string = name, regex
-                @regex = Regexp.new(Regexp.escape(@string))
-            else
-                @name, @regex = name, regex
-            end
-        end
-
-        %w{skip accumulate}.each do |method|
-            define_method(method+"?") do
-                self.send(method)
-            end
-        end
+  class Token
+    attr_accessor :regex, :name, :string, :skip, :incr_line, :skip_text, :accumulate
 
-        def to_s
-            if self.string
-                @string
-            else
-                @name.to_s
-            end
-        end
-        
-        def acceptable?(context={})
-            # By default tokens are aceeptable in any context
-            true 
-        end
+    def initialize(regex, name)
+      if regex.is_a?(String)
+        @name, @string = name, regex
+        @regex = Regexp.new(Regexp.escape(@string))
+      else
+        @name, @regex = name, regex
+      end
     end
 
-    # Maintain a list of tokens.
-    class TokenList
-        attr_reader :regex_tokens, :string_tokens
-
-        def [](name)
-            @tokens[name]
-        end
-
-        # Create a new token.
-        def add_token(name, regex, options = {}, &block)
-            token = Token.new(regex, name)
-            raise(ArgumentError, "Token %s already exists" % name) if @tokens.include?(name)
-            @tokens[token.name] = token
-            if token.string
-                @string_tokens << token
-                @tokens_by_string[token.string] = token
-            else
-                @regex_tokens << token
-            end
-
-            options.each do |name, option|
-                token.send(name.to_s + "=", option)
-            end
-
-            token.meta_def(:convert, &block) if block_given?
-
-            token
-        end
+    # MQR: Why not just alias?
+    %w{skip accumulate}.each do |method|
+      define_method(method+"?") do
+        self.send(method)
+      end
+    end
 
-        def initialize
-            @tokens = {}
-            @regex_tokens = []
-            @string_tokens = []
-            @tokens_by_string = {}
-        end
+    def to_s
+      if self.string
+        @string
+      else
+        @name.to_s
+      end
+    end
 
-        # Look up a token by its value, rather than name.
-        def lookup(string)
-            @tokens_by_string[string]
-        end
+    def acceptable?(context={})
+      # By default tokens are aceeptable in any context
+      true
+    end
+  end
 
-        # Define more tokens.
-        def add_tokens(hash)
-            hash.each do |regex, name|
-                add_token(name, regex)
-            end
-        end
+  # Maintain a list of tokens.
+  class TokenList
+    attr_reader :regex_tokens, :string_tokens
 
-        # Sort our tokens by length, so we know once we match, we're done.
-        # This helps us avoid the O(n^2) nature of token matching.
-        def sort_tokens
-            @string_tokens.sort! { |a, b| b.string.length <=> a.string.length }
-        end
+    def [](name)
+      @tokens[name]
     end
 
-    TOKENS = TokenList.new
-    TOKENS.add_tokens(
-        '[' => :LBRACK,
-        ']' => :RBRACK,
-        '{' => :LBRACE,
-        '}' => :RBRACE,
-        '(' => :LPAREN,
-        ')' => :RPAREN,
-        '=' => :EQUALS,
-        '+=' => :APPENDS,
-        '==' => :ISEQUAL,
-        '>=' => :GREATEREQUAL,
-        '>' => :GREATERTHAN,
-        '<' => :LESSTHAN,
-        '<=' => :LESSEQUAL,
-        '!=' => :NOTEQUAL,
-        '!' => :NOT,
-        ',' => :COMMA,
-        '.' => :DOT,
-        ':' => :COLON,
-        '@' => :AT,
-        '<<|' => :LLCOLLECT,
-        '|>>' => :RRCOLLECT,
-        '<|' => :LCOLLECT,
-        '|>' => :RCOLLECT,
-        ';' => :SEMIC,
-        '?' => :QMARK,
-        '\\' => :BACKSLASH,
-        '=>' => :FARROW,
-        '+>' => :PARROW,
-        '+' => :PLUS,
-        '-' => :MINUS,
-        '/' => :DIV,
-        '*' => :TIMES,
-        '<<' => :LSHIFT,
-        '>>' => :RSHIFT,
-        '=~' => :MATCH,
-        '!~' => :NOMATCH,
-        %r{([a-z][-\w]*)?(::[a-z][-\w]*)+} => :CLASSNAME, # Require '::' in the class name, else we'd compete with NAME
-        %r{((::){0,1}[A-Z][-\w]*)+} => :CLASSREF
-    )
-
-    TOKENS.add_tokens "Whatever" => :DQTEXT, "Nomatter" => :SQTEXT, "alsonomatter" => :BOOLEAN
-
-    TOKENS.add_token :NUMBER, %r{\b(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b} do |lexer, value|
-        [TOKENS[:NAME], value]
-    end
+    # Create a new token.
+    def add_token(name, regex, options = {}, &block)
+      token = Token.new(regex, name)
+      raise(ArgumentError, "Token #{name} already exists") if @tokens.include?(name)
+      @tokens[token.name] = token
+      if token.string
+        @string_tokens << token
+        @tokens_by_string[token.string] = token
+      else
+        @regex_tokens << token
+      end
 
-    TOKENS.add_token :NAME, %r{[a-z0-9][-\w]*} do |lexer, value|
-        string_token = self
-        # we're looking for keywords here
-        if tmp = KEYWORDS.lookup(value)
-            string_token = tmp
-            if [:TRUE, :FALSE].include?(string_token.name)
-                value = eval(value)
-                string_token = TOKENS[:BOOLEAN]
-            end
-        end
-        [string_token, value]
-    end
+      options.each do |name, option|
+        token.send(name.to_s + "=", option)
+      end
 
-    TOKENS.add_token :COMMENT, %r{#.*}, :accumulate => true, :skip => true do |lexer,value|
-        value.sub!(/# ?/,'')
-        [self, value]
-    end
+      token.meta_def(:convert, &block) if block_given?
 
-    TOKENS.add_token :MLCOMMENT, %r{/\*(.*?)\*/}m, :accumulate => true, :skip => true do |lexer, value|
-        lexer.line += value.count("\n")
-        value.sub!(/^\/\* ?/,'')
-        value.sub!(/ ?\*\/$/,'')
-        [self,value]
+      token
     end
 
-    regex_token = TOKENS.add_token :REGEX, %r{/[^/\n]*/} do |lexer, value|
-        # Make sure we haven't matched an escaped /
-        while value[-2..-2] == '\\'
-            other = lexer.scan_until(%r{/})
-            value += other
-        end
-        regex = value.sub(%r{\A/}, "").sub(%r{/\Z}, '').gsub("\\/", "/")
-        [self, Regexp.new(regex)]
+    def initialize
+      @tokens = {}
+      @regex_tokens = []
+      @string_tokens = []
+      @tokens_by_string = {}
     end
 
-    def regex_token.acceptable?(context={})
-        [:NODE,:LBRACE,:RBRACE,:MATCH,:NOMATCH,:COMMA].include? context[:after]
+    # Look up a token by its value, rather than name.
+    def lookup(string)
+      @tokens_by_string[string]
     end
 
-    TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true
-
-    TOKENS.add_token :SQUOTE, "'" do |lexer, value|
-        value = lexer.slurpstring(value)
-        [TOKENS[:SQTEXT], value]
+    # Define more tokens.
+    def add_tokens(hash)
+      hash.each do |regex, name|
+        add_token(name, regex)
+      end
     end
 
-    TOKENS.add_token :DQUOTE, '"' do |lexer, value|
-        value = lexer.slurpstring(value)
-        [TOKENS[:DQTEXT], value]
+    # Sort our tokens by length, so we know once we match, we're done.
+    # This helps us avoid the O(n^2) nature of token matching.
+    def sort_tokens
+      @string_tokens.sort! { |a, b| b.string.length <=> a.string.length }
     end
+  end
 
-    TOKENS.add_token :VARIABLE, %r{\$(\w*::)*\w+} do |lexer, value|
-        value = value.sub(/^\$/, '')
-        [self, value]
-    end
+  TOKENS = TokenList.new
 
-    TOKENS.sort_tokens
+    TOKENS.add_tokens(
 
-    @@pairs = {
-        "{" => "}",
-        "(" => ")",
-        "[" => "]",
-        "<|" => "|>",
-        "<<|" => "|>>"
-    }
+      '[' => :LBRACK,
+      ']' => :RBRACK,
+      '{' => :LBRACE,
+      '}' => :RBRACE,
+      '(' => :LPAREN,
+
+      ')' => :RPAREN,
+      '=' => :EQUALS,
+      '+=' => :APPENDS,
+      '==' => :ISEQUAL,
+      '>=' => :GREATEREQUAL,
+      '>' => :GREATERTHAN,
+      '<' => :LESSTHAN,
+      '<=' => :LESSEQUAL,
+      '!=' => :NOTEQUAL,
+      '!' => :NOT,
+      ',' => :COMMA,
+      '.' => :DOT,
+      ':' => :COLON,
+      '@' => :AT,
+      '<<|' => :LLCOLLECT,
+      '->' => :IN_EDGE,
+      '<-' => :OUT_EDGE,
+      '~>' => :IN_EDGE_SUB,
+      '<~' => :OUT_EDGE_SUB,
+      '|>>' => :RRCOLLECT,
+      '<|' => :LCOLLECT,
+      '|>' => :RCOLLECT,
+      ';' => :SEMIC,
+      '?' => :QMARK,
+      '\\' => :BACKSLASH,
+      '=>' => :FARROW,
+      '+>' => :PARROW,
+      '+' => :PLUS,
+      '-' => :MINUS,
+      '/' => :DIV,
+      '*' => :TIMES,
+      '<<' => :LSHIFT,
+      '>>' => :RSHIFT,
+      '=~' => :MATCH,
+      '!~' => :NOMATCH,
+      %r{([a-z][-\w]*)?(::[a-z][-\w]*)+} => :CLASSNAME, # Require '::' in the class name, else we'd compete with NAME
+      %r{((::){0,1}[A-Z][-\w]*)+} => :CLASSREF,
+      "<string>" => :STRING,
+      "<dqstring up to first interpolation>" => :DQPRE,
+      "<dqstring between two interpolations>" => :DQMID,
+      "<dqstring after final interpolation>" => :DQPOST,
+      "<boolean>" => :BOOLEAN
+      )
+
+  TOKENS.add_token :NUMBER, %r{\b(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b} do |lexer, value|
+    [TOKENS[:NAME], value]
+  end
+  #:stopdoc: # Issue #4161
+  def (TOKENS[:NUMBER]).acceptable?(context={})
+    ![:DQPRE,:DQMID].include? context[:after]
+  end
+  #:startdoc:
+
+  TOKENS.add_token :NAME, %r{[a-z0-9][-\w]*} do |lexer, value|
+    string_token = self
+    # we're looking for keywords here
+    if tmp = KEYWORDS.lookup(value)
+      string_token = tmp
+      if [:TRUE, :FALSE].include?(string_token.name)
+        value = eval(value)
+        string_token = TOKENS[:BOOLEAN]
+      end
+    end
+    [string_token, value]
+  end
+  [:NAME,:CLASSNAME,:CLASSREF].each { |name_token|
+    #:stopdoc: # Issue #4161
+    def (TOKENS[name_token]).acceptable?(context={})
+      ![:DQPRE,:DQMID].include? context[:after]
+    end
+    #:startdoc:
+  }
+
+  TOKENS.add_token :COMMENT, %r{#.*}, :accumulate => true, :skip => true do |lexer,value|
+    value.sub!(/# ?/,'')
+    [self, value]
+  end
+
+  TOKENS.add_token :MLCOMMENT, %r{/\*(.*?)\*/}m, :accumulate => true, :skip => true do |lexer, value|
+    lexer.line += value.count("\n")
+    value.sub!(/^\/\* ?/,'')
+    value.sub!(/ ?\*\/$/,'')
+    [self,value]
+  end
+
+  TOKENS.add_token :REGEX, %r{/[^/\n]*/} do |lexer, value|
+    # Make sure we haven't matched an escaped /
+    while value[-2..-2] == '\\'
+      other = lexer.scan_until(%r{/})
+      value += other
+    end
+    regex = value.sub(%r{\A/}, "").sub(%r{/\Z}, '').gsub("\\/", "/")
+    [self, Regexp.new(regex)]
+  end
+
+  #:stopdoc: # Issue #4161
+  def (TOKENS[:REGEX]).acceptable?(context={})
+    [:NODE,:LBRACE,:RBRACE,:MATCH,:NOMATCH,:COMMA].include? context[:after]
+  end
+  #:startdoc:
+
+  TOKENS.add_token :RETURN, "\n", :skip => true, :incr_line => true, :skip_text => true
+
+  TOKENS.add_token :SQUOTE, "'" do |lexer, value|
+    [TOKENS[:STRING], lexer.slurpstring(value).first ]
+  end
+
+  DQ_initial_token_types      = {'$' => :DQPRE,'"' => :STRING}
+  DQ_continuation_token_types = {'$' => :DQMID,'"' => :DQPOST}
+
+  TOKENS.add_token :DQUOTE, /"/ do |lexer, value|
+    lexer.tokenize_interpolated_string(DQ_initial_token_types)
+  end
+
+  TOKENS.add_token :DQCONT, /\}/ do |lexer, value|
+    lexer.tokenize_interpolated_string(DQ_continuation_token_types)
+  end
+  #:stopdoc: # Issue #4161
+  def (TOKENS[:DQCONT]).acceptable?(context={})
+    context[:string_interpolation_depth] > 0
+  end
+  #:startdoc:
+
+  TOKENS.add_token :DOLLAR_VAR, %r{\$(\w*::)*\w+} do |lexer, value|
+    [TOKENS[:VARIABLE],value[1..-1]]
+  end
+
+  TOKENS.add_token :VARIABLE, %r{(\w*::)*\w+}
+  #:stopdoc: # Issue #4161
+  def (TOKENS[:VARIABLE]).acceptable?(context={})
+    [:DQPRE,:DQMID].include? context[:after]
+  end
+  #:startdoc:
+
+
+  TOKENS.sort_tokens
+
+  @@pairs = {
+    "{" => "}",
+    "(" => ")",
+    "[" => "]",
+    "<|" => "|>",
+    "<<|" => "|>>"
+  }
+
+  KEYWORDS = TokenList.new
 
-    KEYWORDS = TokenList.new
 
     KEYWORDS.add_tokens(
-        "case" => :CASE,
-        "class" => :CLASS,
-        "default" => :DEFAULT,
-        "define" => :DEFINE,
-        "import" => :IMPORT,
-        "if" => :IF,
-        "elsif" => :ELSIF,
-        "else" => :ELSE,
-        "inherits" => :INHERITS,
-        "node" => :NODE,
-        "and"  => :AND,
-        "or"   => :OR,
-        "undef"   => :UNDEF,
-        "false" => :FALSE,
-        "true" => :TRUE
-    )
-
-    def clear
-        initvars
-    end
 
-    def expected
-        return nil if @expected.empty?
-        name = @expected[-1]
-        raise "Could not find expected token %s" % name unless token = TOKENS.lookup(name)
+      "case" => :CASE,
+      "class" => :CLASS,
+      "default" => :DEFAULT,
+      "define" => :DEFINE,
+      "import" => :IMPORT,
+      "if" => :IF,
+      "elsif" => :ELSIF,
+      "else" => :ELSE,
+      "inherits" => :INHERITS,
+      "node" => :NODE,
+      "and"  => :AND,
+      "or"   => :OR,
+      "undef"   => :UNDEF,
+      "false" => :FALSE,
+      "true" => :TRUE,
+
+      "in" => :IN
+  )
+
+  def clear
+    initvars
+  end
+
+  def expected
+    return nil if @expected.empty?
+    name = @expected[-1]
+    TOKENS.lookup(name) or lex_error "Could not find expected token #{name}"
+  end
+
+  # scan the whole file
+  # basically just used for testing
+  def fullscan
+    array = []
+
+    self.scan { |token, str|
+      # Ignore any definition nesting problems
+      @indefine = false
+      array.push([token,str])
+    }
+    array
+  end
+
+  def file=(file)
+    @file = file
+    @line = 1
+    @scanner = StringScanner.new(File.read(file))
+  end
+
+  def shift_token
+    @token_queue.shift
+  end
+
+  def find_string_token
+    # We know our longest string token is three chars, so try each size in turn
+    # until we either match or run out of chars.  This way our worst-case is three
+    # tries, where it is otherwise the number of string token we have.  Also,
+    # the lookups are optimized hash lookups, instead of regex scans.
+    #
+    s = @scanner.peek(3)
+    token = TOKENS.lookup(s[0,3]) || TOKENS.lookup(s[0,2]) || TOKENS.lookup(s[0,1])
+    [ token, token && @scanner.scan(token.regex) ]
+  end
+
+  # Find the next token that matches a regex.  We look for these first.
+  def find_regex_token
+    @regex += 1
+    best_token = nil
+    best_length = 0
+
+    # I tried optimizing based on the first char, but it had
+    # a slightly negative affect and was a good bit more complicated.
+    TOKENS.regex_tokens.each do |token|
+      if length = @scanner.match?(token.regex) and token.acceptable?(lexing_context)
+        # We've found a longer match
+        if length > best_length
+          best_length = length
+          best_token = token
+        end
+      end
+    end
 
-        return token
-    end
+    return best_token, @scanner.scan(best_token.regex) if best_token
+  end
 
-    # scan the whole file
-    # basically just used for testing
-    def fullscan
-        array = []
-
-        self.scan { |token, str|
-            # Ignore any definition nesting problems
-            @indefine = false
-            array.push([token,str])
-        }
-        return array
-    end
+  # Find the next token, returning the string and the token.
+  def find_token
+    @find += 1
+    shift_token || find_regex_token || find_string_token
+  end
 
-    def file=(file)
-        @file = file
-        @line = 1
-        @scanner = StringScanner.new(File.read(file))
+  def indefine?
+    if defined?(@indefine)
+      @indefine
+    else
+      false
     end
+  end
 
-    def find_string_token
-        matched_token = value = nil
-
-        # We know our longest string token is three chars, so try each size in turn
-        # until we either match or run out of chars.  This way our worst-case is three
-        # tries, where it is otherwise the number of string chars we have.  Also,
-        # the lookups are optimized hash lookups, instead of regex scans.
-        [3, 2, 1].each do |i|
-            str = @scanner.peek(i)
-            if matched_token = TOKENS.lookup(str)
-                value = @scanner.scan(matched_token.regex)
-                break
-            end
-        end
+  def initialize
+    @find = 0
+    @regex = 0
+    initvars
+  end
 
-        return matched_token, value
-    end
+  def initvars
+    @line = 1
+    @previous_token = nil
+    @scanner = nil
+    @file = nil
+    # AAARRGGGG! okay, regexes in ruby are bloody annoying
+    # no one else has "\n" =~ /\s/
+    @skip = %r{[ \t\r]+}
 
-    # Find the next token that matches a regex.  We look for these first.
-    def find_regex_token
-        @regex += 1
-        best_token = nil
-        best_length = 0
-
-        # I tried optimizing based on the first char, but it had
-        # a slightly negative affect and was a good bit more complicated.
-        TOKENS.regex_tokens.each do |token|
-            if length = @scanner.match?(token.regex) and token.acceptable?(lexing_context)
-                # We've found a longer match
-                if length > best_length
-                    best_length = length
-                    best_token = token
-                end
-            end
-        end
+    @namestack = []
+    @token_queue = []
+    @indefine = false
+    @expected = []
+    @commentstack = [ ['', @line] ]
+    @lexing_context = {
+      :after => nil,
+      :start_of_line => true,
+      :string_interpolation_depth => 0
+      }
+  end
 
-        return best_token, @scanner.scan(best_token.regex) if best_token
-    end
+  # Make any necessary changes to the token and/or value.
+  def munge_token(token, value)
+    @line += 1 if token.incr_line
 
-    # Find the next token, returning the string and the token.
-    def find_token
-        @find += 1
-        find_regex_token || find_string_token
-    end
+    skip if token.skip_text
 
-    def indefine?
-        if defined? @indefine
-            @indefine
-        else
-            false
-        end
-    end
+    return if token.skip and not token.accumulate?
 
-    def initialize
-        @find = 0
-        @regex = 0
-        initvars()
-    end
+    token, value = token.convert(self, value) if token.respond_to?(:convert)
 
-    def initvars
-        @line = 1
-        @previous_token = nil
-        @scanner = nil
-        @file = nil
-        # AAARRGGGG! okay, regexes in ruby are bloody annoying
-        # no one else has "\n" =~ /\s/
-        @skip = %r{[ \t]+}
-
-        @namestack = []
-        @indefine = false
-        @expected = []
-        @commentstack = [ ['', @line] ]
-        @lexing_context = {:after => nil, :start_of_line => true}
-    end
+    return unless token
 
-    # Make any necessary changes to the token and/or value.
-    def munge_token(token, value)
-        @line += 1 if token.incr_line
+    if token.accumulate?
+      comment = @commentstack.pop
+      comment[0] << value + "\n"
+      @commentstack.push(comment)
+    end
 
-        skip() if token.skip_text
+    return if token.skip
 
-        return if token.skip and not token.accumulate?
+    return token, { :value => value, :line => @line }
+  end
 
-        token, value = token.convert(self, value) if token.respond_to?(:convert)
+  # Go up one in the namespace.
+  def namepop
+    @namestack.pop
+  end
 
-        return unless token
+  # Collect the current namespace.
+  def namespace
+    @namestack.join("::")
+  end
 
-        if token.accumulate?
-            comment = @commentstack.pop
-            comment[0] << value + "\n"
-            @commentstack.push(comment)
-        end
+  # This value might have :: in it, but we don't care -- it'll be
+  # handled normally when joining, and when popping we want to pop
+  # this full value, however long the namespace is.
+  def namestack(value)
+    @namestack << value
+  end
 
-        return if token.skip
+  def rest
+    @scanner.rest
+  end
 
-        return token, { :value => value, :line => @line }
-    end
+  # this is the heart of the lexer
+  def scan
+    #Puppet.debug("entering scan")
+    lex_error "Invalid or empty string" unless @scanner
 
-    # Go up one in the namespace.
-    def namepop
-        @namestack.pop
-    end
+    # Skip any initial whitespace.
+    skip
 
-    # Collect the current namespace.
-    def namespace
-        @namestack.join("::")
-    end
+    until token_queue.empty? and @scanner.eos? do
+      yielded = false
+      matched_token, value = find_token
 
-    # This value might have :: in it, but we don't care -- it'll be
-    # handled normally when joining, and when popping we want to pop
-    # this full value, however long the namespace is.
-    def namestack(value)
-        @namestack << value
-    end
+      # error out if we didn't match anything at all
+      lex_error "Could not match #{@scanner.rest[/^(\S+|\s+|.*)/]}" unless matched_token
 
-    def rest
-        @scanner.rest
-    end
+      newline = matched_token.name == :RETURN
+
+      # this matches a blank line; eat the previously accumulated comments
+      getcomment if lexing_context[:start_of_line] and newline
+      lexing_context[:start_of_line] = newline
 
-    # this is the heart of the lexer
-    def scan
-        #Puppet.debug("entering scan")
-        raise Puppet::LexError.new("Invalid or empty string") unless @scanner
-
-        # Skip any initial whitespace.
-        skip()
-
-        until @scanner.eos? do
-            yielded = false
-            matched_token, value = find_token
-
-            # error out if we didn't match anything at all
-            if matched_token.nil?
-                nword = nil
-                # Try to pull a 'word' out of the remaining string.
-                if @scanner.rest =~ /^(\S+)/
-                    nword = $1
-                elsif @scanner.rest =~ /^(\s+)/
-                    nword = $1
-                else
-                    nword = @scanner.rest
-                end
-                raise "Could not match '%s'" % nword
-            end
-
-            newline = matched_token.name == :RETURN
-
-            # this matches a blank line; eat the previously accumulated comments
-            getcomment if lexing_context[:start_of_line] and newline
-            lexing_context[:start_of_line] = newline
-
-            final_token, token_value = munge_token(matched_token, value)
-
-            unless final_token
-                skip()
-                next
-            end
-
-            lexing_context[:after]         = final_token.name unless newline
-
-            value = token_value[:value]
-
-            if match = @@pairs[value] and final_token.name != :DQUOTE and final_token.name != :SQUOTE
-                @expected << match
-            elsif exp = @expected[-1] and exp == value and final_token.name != :DQUOTE and final_token.name != :SQUOTE
-                @expected.pop
-            end
-
-            if final_token.name == :LBRACE
-                commentpush
-            end
-
-            yield [final_token.name, token_value]
-
-            if @previous_token
-                namestack(value) if @previous_token.name == :CLASS
-
-                if @previous_token.name == :DEFINE
-                    if indefine?
-                        msg = "Cannot nest definition %s inside %s" % [value, @indefine]
-                        self.indefine = false
-                        raise Puppet::ParseError, msg
-                    end
-
-                    @indefine = value
-                end
-            end
-            @previous_token = final_token
-            skip()
-        end
-        @scanner = nil
+      final_token, token_value = munge_token(matched_token, value)
 
-        # This indicates that we're done parsing.
-        yield [false,false]
-    end
+      unless final_token
+        skip
+        next
+      end
 
-    # Skip any skipchars in our remaining string.
-    def skip
-        @scanner.skip(@skip)
-    end
+      lexing_context[:after]         = final_token.name unless newline
+      lexing_context[:string_interpolation_depth] += 1 if final_token.name == :DQPRE
+      lexing_context[:string_interpolation_depth] -= 1 if final_token.name == :DQPOST
 
-    # Provide some limited access to the scanner, for those
-    # tokens that need it.
-    def scan_until(regex)
-        @scanner.scan_until(regex)
-    end
+      value = token_value[:value]
 
-    # we've encountered an opening quote...
-    # slurp in the rest of the string and return it
-    def slurpstring(quote)
-        # we search for the next quote that isn't preceded by a
-        # backslash; the caret is there to match empty strings
-        str = @scanner.scan_until(/([^\\]|^)#{quote}/)
-        if str.nil?
-            raise Puppet::LexError.new("Unclosed quote after '%s' in '%s'" %
-                [self.last,self.rest])
-        else
-            str.sub!(/#{quote}\Z/,"")
-            str.gsub!(/\\#{quote}/,quote)
-        end
+      if match = @@pairs[value] and final_token.name != :DQUOTE and final_token.name != :SQUOTE
+        @expected << match
+      elsif exp = @expected[-1] and exp == value and final_token.name != :DQUOTE and final_token.name != :SQUOTE
+        @expected.pop
+      end
 
-        # Add to our line count for every carriage return in multi-line strings.
-        @line += str.count("\n")
+      if final_token.name == :LBRACE
+        commentpush
+      end
 
-        return str
-    end
+      yield [final_token.name, token_value]
 
-    # just parse a string, not a whole file
-    def string=(string)
-        @scanner = StringScanner.new(string)
-    end
+      if @previous_token
+        namestack(value) if @previous_token.name == :CLASS
 
-    # returns the content of the currently accumulated content cache
-    def commentpop
-        return @commentstack.pop[0]
-    end
+        if @previous_token.name == :DEFINE
+          if indefine?
+            msg = "Cannot nest definition #{value} inside #{@indefine}"
+            self.indefine = false
+            raise Puppet::ParseError, msg
+          end
 
-    def getcomment(line = nil)
-        comment = @commentstack.last
-        if line.nil? or comment[1] <= line
-            @commentstack.pop
-            @commentstack.push(['', @line])
-            return comment[0]
+          @indefine = value
         end
-        return ''
-    end
-
-    def commentpush
-        @commentstack.push(['', @line])
-    end
+      end
+      @previous_token = final_token
+      skip
+    end
+    @scanner = nil
+
+    # This indicates that we're done parsing.
+    yield [false,false]
+  end
+
+  # Skip any skipchars in our remaining string.
+  def skip
+    @scanner.skip(@skip)
+  end
+
+  # Provide some limited access to the scanner, for those
+  # tokens that need it.
+  def scan_until(regex)
+    @scanner.scan_until(regex)
+  end
+
+  # we've encountered the start of a string...
+  # slurp in the rest of the string and return it
+  Valid_escapes_in_strings = %w{ \\  $ ' " n t s }+["\n"]
+  def slurpstring(terminators)
+    # we search for the next quote that isn't preceded by a
+    # backslash; the caret is there to match empty strings
+    str = @scanner.scan_until(/([^\\]|^)[#{terminators}]/) or lex_error "Unclosed quote after '#{last}' in '#{rest}'"
+    @line += str.count("\n") # literal carriage returns add to the line count.
+    str.gsub!(/\\(.)/) {
+      case ch=$1
+      when 'n'; "\n"
+      when 't'; "\t"
+      when 's'; " "
+      else
+        if Valid_escapes_in_strings.include? ch and not (ch == '"' and terminators == "'")
+          ch
+        else
+          Puppet.warning "Unrecognised escape sequence '\\#{ch}'#{file && " in file #{file}"}#{line && " at line #{line}"}"
+          "\\#{ch}"
+        end
+      end
+    }
+    [ str[0..-2],str[-1,1] ]
+  end
+
+  def tokenize_interpolated_string(token_type,preamble='')
+    value,terminator = slurpstring('"$')
+    token_queue << [TOKENS[token_type[terminator]],preamble+value]
+    if terminator != '$' or @scanner.scan(/\{/)
+      token_queue.shift 
+    elsif var_name = @scanner.scan(%r{(\w*::)*\w+|[0-9]})
+      token_queue << [TOKENS[:VARIABLE],var_name]
+      tokenize_interpolated_string(DQ_continuation_token_types)
+    else
+      tokenize_interpolated_string(token_type,token_queue.pop.last + terminator)
+    end
+  end
+
+  # just parse a string, not a whole file
+  def string=(string)
+    @scanner = StringScanner.new(string)
+  end
+
+  # returns the content of the currently accumulated content cache
+  def commentpop
+    @commentstack.pop[0]
+  end
+
+  def getcomment(line = nil)
+    comment = @commentstack.last
+    if line.nil? or comment[1] <= line
+      @commentstack.pop
+      @commentstack.push(['', @line])
+      return comment[0]
+    end
+    ''
+  end
+
+  def commentpush
+    @commentstack.push(['', @line])
+  end
 end