twig_ruby 0.0.1 → 0.0.2

data/lib/twig/lexer.rb

@@ -18,10 +18,16 @@ module Twig
     REGEX_EXPONENT = /[eE][+-]?#{REGEX_LNUM}/
     REGEX_DNUM = /#{REGEX_LNUM}(?:#{REGEX_FRAC})?/
 
-    REGEX_NAME = /[a-zA-Z_][a-zA-Z0-9_]*/
+    REGEX_NAME = /[a-zA-Z_\u{007f}-\u{00ff}][a-zA-Z0-9_\u{007f}-\u{00ff}]*/u
     REGEX_SYMBOL = /:#{REGEX_NAME}/
-    REGEX_STRING = /\A"([^#"\\]*(?:\\\\.[^#"\\]*)*)"|'([^'\\]*(?:\\\\.[^'\\]*)*)'/s
-    REGEX_NUMBER = /\A(?:#{REGEX_DNUM}(?:#{REGEX_EXPONENT})?)/x
+    REGEX_CVAR = /@#{REGEX_NAME}/
+    REGEX_STRING = /\G"([^#"\\]*(?:\\.[^#"\\]*)*)"|'([^'\\]*(?:\\.[^'\\]*)*)'/mu
+    REGEX_DQ_STRING_PART = /\G[^#"\\]*(?:(?:\.|#(?!\{))[^#"\\]*)*/mu
+    REGEX_INLINE_COMMENT = /#[^\n]*/
+    REGEX_DQ_STRING_DELIM = /\G"/
+    REGEX_INTERP_START = /\G#\{[[:space:]]*/
+    REGEX_INTERP_END = /\G[[:space:]]*\}/
+    REGEX_NUMBER = /\G(?:#{REGEX_DNUM}(?:#{REGEX_EXPONENT})?)/x
 
     STATE_DATA = 0
     STATE_BLOCK = 1
@@ -29,6 +35,14 @@ module Twig
     STATE_STRING = 3
     STATE_INTERPOLATION = 4
 
+    SPECIAL_CHARS = {
+      'f' => "\f",
+      'n' => "\n",
+      'r' => "\r",
+      't' => "\t",
+      'v' => "\v",
+    }.freeze
+
     # @param [Environment] environment
     def initialize(environment)
       @environment = environment
@@ -56,6 +70,10 @@ module Twig
           lex_block
         when STATE_VAR
           lex_var
+        when STATE_STRING
+          lex_string
+        when STATE_INTERPOLATION
+          lex_interpolation
         else
           raise "Unknown state: #{@state}"
         end
@@ -91,17 +109,26 @@ module Twig
       # Push the template text first
       text = text_content = @code[@cursor, (position.begin(0) - @cursor)]
 
-      # TODO: Trim
+      # trim?
+      char = @positions[@position][2]
+
+      if char == WHITESPACE_TRIM
+        text = text.rstrip
+      elsif char == WHITESPACE_LINE_TRIM
+        text = text.gsub(/[ \t\0\x0B]*\z/, '')
+      end
 
       push_token(Token::TEXT_TYPE, text)
       move_cursor(text_content + position.to_s)
 
       case @positions[@position][1]
+      when TAG_COMMENT[0]
+        lex_comment
       when TAG_BLOCK[0]
-        if (match = @code[@cursor...].match(lex_block_raw_regex))
+        if (match = @code.match(lex_block_raw_regex, @cursor))
           move_cursor(match.to_s)
           lex_raw_data
-        elsif (match = @code[@cursor...].match(lex_block_line_regex))
+        elsif (match = @code.match(lex_block_line_regex, @cursor))
           move_cursor(match[0].to_s)
           @lineno = match[1].to_i
         else
@@ -114,24 +141,24 @@ module Twig
         push_state(STATE_VAR)
         @current_var_block_line = @lineno
       else
-        raise "Invalid start token #{@positions[@position]}"
+        raise Error::Syntax.new("Invalid start token #{@positions[@position]}", @lineno)
       end
     end
 
     def lex_raw_data
-      unless (match = @code[@cursor...].match(lex_raw_data_regex))
-        raise "Uexpected end of file. Unclosed 'verbatim' block"
+      unless (match = @code.match(lex_raw_data_regex, @cursor))
+        raise Error::Syntax.new("Uexpected end of file. Unclosed 'verbatim' block.", @lineno, @source)
       end
 
-      text = @code[@cursor, match.begin(0)]
-      move_cursor(@code[@cursor, (match.begin(0) + match.to_s.length)])
+      text = @code[@cursor, match.begin(0) - @cursor]
+      move_cursor(@code[@cursor, (text.length + match.to_s.length)])
 
       # trim
       if match[1]
         text = if match[1] == WHITESPACE_TRIM
-                 text.gsub(/ *$/, '') # space trim
-               else
                  text.rstrip # line trim
+               else
+                 text.gsub(/[ \t\0\x0B]*\z/, '') # space trim
                end
       end
 
@@ -139,7 +166,7 @@ module Twig
     end
 
     def lex_block
-      if @brackets.empty? && (match = @code[@cursor..].match(lex_block_regex))
+      if @brackets.empty? && (match = @code.match(lex_block_regex, @cursor))
         push_token(Token::BLOCK_END_TYPE)
         move_cursor(match.to_s)
         pop_state
@@ -149,7 +176,7 @@ module Twig
     end
 
     def lex_var
-      match = @code[@cursor...].match(lex_var_regex)
+      match = @code.match(lex_var_regex, @cursor)
 
       if @brackets.empty? && match
         push_token(Token::VAR_END_TYPE)
@@ -161,75 +188,144 @@ module Twig
     end
 
     def lex_expression
-      @code[@cursor..].match(/\A\s+/) do |match|
+      @code.match(/\G[[:space:]]+/, @cursor) do |match|
         move_cursor(match.to_s)
 
         if @cursor >= @end
-          raise "Unclosed #{@state == STATE_BLOCK ? 'block' : 'variable'}"
+          raise Error::Syntax.new(
+            "Unclosed \"#{@state == STATE_BLOCK ? 'block' : 'variable'}\".",
+            @lineno,
+            @source
+          )
         end
       end
 
-      # Spread operator
-      if code_at?(0, '.') && (@cursor + 2 < @end) && code_at?(1, '.') && code_at?(2, '.')
-        push_token(Token::SPREAD_TYPE)
-        move_cursor('...')
-      # Arrow function
-      elsif code_at?(0, '=') && (@cursor + 1 < @end) && code_at?(1, '>')
-        push_token(Token::ARROW_TYPE)
-        move_cursor('=>')
-      elsif (match = @code[@cursor..].match(operator_regex))
-        push_token(Token::OPERATOR_TYPE, match.to_s.gsub('/\s+/', ' '))
+      if (match = @code.match(operator_regex, @cursor))
+        operator = match.to_s.gsub(/[[:space:]]+/, ' ')
+
+        if OPENING_BRACKET.include?(operator)
+          check_brackets(operator)
+        end
+
+        push_token(Token::OPERATOR_TYPE, operator)
         move_cursor(match.to_s)
-      elsif (match = @code[@cursor..].match(/\A#{REGEX_NAME}/))
+      elsif (match = @code.match(/\G#{REGEX_NAME}(?:\?(?!\?))?/, @cursor)) # Optional ? but not ??
         push_token(Token::NAME_TYPE, match.to_s)
         move_cursor(match.to_s)
-      elsif (match = @code[@cursor..].match(/\A#{REGEX_SYMBOL}/))
+      elsif (match = @code.match(/\G#{REGEX_SYMBOL}/, @cursor))
         push_token(Token::SYMBOL_TYPE, match.to_s[1..])
         move_cursor(match.to_s)
-      elsif (match = @code[@cursor..].match(REGEX_NUMBER))
+      elsif (match = @code.match(/\G#{REGEX_CVAR}/, @cursor))
+        push_token(Token::CLASS_VAR_TYPE, match.to_s)
+        move_cursor(match.to_s)
+      elsif (match = @code.match(REGEX_NUMBER, @cursor))
         value = match.to_s.tr('_', '')
         value = value.to_i.to_s == value ? value.to_i : value.to_f
         push_token(Token::NUMBER_TYPE, value)
         move_cursor(match.to_s)
       elsif code_at?(0, PUNCTUATION)
-        # opening bracket
-        if code_at?(0, OPENING_BRACKET)
-          @brackets << [code_at, @lineno]
-        elsif code_at?(0, CLOSING_BRACKET)
-          if @brackets.empty?
-            raise Error::Syntax.new("Unexpected closing bracket: #{code_at}", @lineno, @source)
-          end
+        check_brackets(code_at)
+        push_token(Token::PUNCTUATION_TYPE, code_at)
+        @cursor += 1
+      elsif (match = @code.match(REGEX_STRING, @cursor))
+        push_token(Token::STRING_TYPE, stripcslashes(match.to_s[1...-1], match.to_s[0]))
+        move_cursor(match.to_s)
+      elsif (match = @code.match(REGEX_DQ_STRING_DELIM, @cursor))
+        @brackets << ['"', @lineno]
+        push_state(STATE_STRING)
+        move_cursor(match.to_s)
+      elsif (match = @code.match(REGEX_INLINE_COMMENT, @cursor))
+        move_cursor(match.to_s)
+      else
+        raise Error::Syntax.new("Unexpected character '#{code_at}'", @lineno, @source)
+      end
+    end
 
-          expect, lineno = @brackets.pop
+    def lex_comment
+      unless (match = @code.match(lex_comment_regex, @cursor))
+        raise Error::Syntax.new('Unclosed comment.', @lineno, @source)
+      end
 
-          unless code_at?(0, expect.tr(OPENING_BRACKET.join, CLOSING_BRACKET.join))
-            raise Error::Syntax.new("Unclosed bracket: #{code_at}", lineno, @source)
-          end
+      move_cursor(@code[@cursor...match.offset(0)[1]])
+    end
+
+    def lex_string
+      if (match = @code.match(REGEX_INTERP_START, @cursor))
+        @brackets << ['#{', @lineno]
+        push_token(Token::INTERPOLATION_START_TYPE)
+        move_cursor(match.to_s)
+        push_state(STATE_INTERPOLATION)
+      elsif (match = @code.match(REGEX_DQ_STRING_PART, @cursor)) && match.to_s != ''
+        push_token(Token::STRING_TYPE, stripcslashes(match.to_s, '"'))
+        move_cursor(match.to_s)
+      elsif @code.match(REGEX_DQ_STRING_DELIM, @cursor)
+        expect, lineno = @brackets.pop
+
+        unless code_at?(0, '"')
+          raise Error::Syntax.new("Unclosed '#{expect}'.", lineno, @source)
         end
 
-        push_token(Token::PUNCTUATION_TYPE, code_at)
+        pop_state
         @cursor += 1
-      elsif (match = @code[@cursor..].match(REGEX_STRING))
-        push_token(Token::STRING_TYPE, match.to_s[1...-1])
-        move_cursor(match.to_s)
+      else
+        Error::Syntax.new("Unexpected character '#{code_at}'.", @lineno, @source)
+      end
+    end
+
+    # @return [String]
+    def stripcslashes(str, quote_type)
+      result = ''
+      length = str.length
+
+      i = 0
+      while i < length
+        pos = str.index('\\', i)
+
+        unless pos
+          result += str[i..]
+          break
+        end
+
+        result += str[i...pos]
+        i = pos + 1
+
+        if i >= length
+          result += '\\'
+          break
+        end
+
+        next_char = str[i]
+
+        if SPECIAL_CHARS.key?(next_char)
+          result += SPECIAL_CHARS[next_char]
+        elsif next_char == '\\' || next_char == quote_type
+          result += next_char
+        elsif next_char == '#' && i + 1 < length && str[i + 1] == '{'
+          result += '#{'
+          i += 1
+        elsif next_char == 'x' && i + 1 < length && str[i + 1].match?(/[0-9a-fA-F]/)
+          hex = str[i + 1]
+          i += 1
+          if i + 1 < length && str[i + 1].match?(/[0-9a-fA-F]/)
+            hex += str[i + 1]
+            i += 1
+          end
+          result += [hex.to_i(16)].pack('C')
+        elsif next_char.match?(/[0-7]/) # octal
+          octal = next_char
+          while i + 1 < length && str[i + 1].match?(/[0-7]/) && octal.length < 3
+            octal += str[i + 1]
+            i += 1
+          end
+          result += [octal.to_i(8)].pack('C')
+        else
+          result += "\\#{next_char}"
+        end
+
+        i += 1
       end
 
-      <<-TEMP
-        // opening double quoted string
-        elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
-            $this->brackets[] = ['"', $this->lineno];
-            $this->pushState(self::STATE_STRING);
-            $this->moveCursor($match[0]);
-        }
-        // inline comment
-        elseif (preg_match(self::REGEX_INLINE_COMMENT, $this->code, $match, 0, $this->cursor)) {
-            $this->moveCursor($match[0]);
-        }
-        // unlexable
-        else {
-            throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
-        }
-      TEMP
+      result
     end
 
     def push_token(type, value = '')
@@ -238,6 +334,18 @@ module Twig
       @tokens << Token.new(type, value, @lineno)
     end
 
+    def lex_interpolation
+      bracket = @brackets.last
+      if bracket[0] == '#{' && (match = @code.match(REGEX_INTERP_END, @cursor))
+        @brackets.pop
+        push_token(Token::INTERPOLATION_END_TYPE)
+        move_cursor(match.to_s)
+        pop_state
+      else
+        lex_expression
+      end
+    end
+
     def push_state(state)
       @states << @state
       @state = state
@@ -287,36 +395,47 @@ module Twig
 
     def lex_var_regex
       @lex_var_regex ||=
-        /\A\s*(?:
+        /\G[[:space:]]*(?:
           #{Regexp.union(
-            "#{WHITESPACE_TRIM}#{TAG_VARIABLE[1]}\\s*",
-            WHITESPACE_LINE_TRIM + TAG_VARIABLE[1] + "[#{WHITESPACE_LINE_CHARS}]*",
+            /#{WHITESPACE_TRIM}#{TAG_VARIABLE[1]}[[:space:]]*/,
+            /#{WHITESPACE_LINE_TRIM}#{TAG_VARIABLE[1]}[#{WHITESPACE_LINE_CHARS}]*/,
             TAG_VARIABLE[1]
           )}
         )/x
     end
 
+    def lex_comment_regex
+      @lex_comment_regex ||=
+        /
+          (?:#{Regexp.union(
+            /#{WHITESPACE_TRIM}#{TAG_COMMENT[1]}[[:space:]]*\n?/,
+            /#{WHITESPACE_LINE_TRIM}#{TAG_COMMENT[1]}[#{WHITESPACE_LINE_CHARS}]*/,
+            /#{TAG_COMMENT[1]}\n?/
+          )})
+        /mxu
+    end
+
     def lex_block_raw_regex
       @lex_block_raw_regex ||=
-        /\A\s*verbatim\s*(?:
+        /\G[[:space:]]*verbatim[[:space:]]*(?:
           #{Regexp.union(
-            "#{WHITESPACE_TRIM}#{TAG_BLOCK[1]}\\s*",
-            WHITESPACE_LINE_TRIM + TAG_BLOCK[1] + "[#{WHITESPACE_LINE_CHARS}]*",
+            /#{WHITESPACE_TRIM}#{TAG_BLOCK[1]}[[:space:]]*/,
+            /#{WHITESPACE_LINE_TRIM}#{TAG_BLOCK[1]}[#{WHITESPACE_LINE_CHARS}]*/,
             TAG_BLOCK[1]
           )}
-        )/sx
+        )/mxu
     end
 
     def lex_block_line_regex
-      @lex_block_line_regex ||= /\A\s*line\s+(\d+)\s*#{Regexp.escape(TAG_BLOCK[1])}/s
+      @lex_block_line_regex ||= /\G[[:space:]]*line[[:space:]]+(\d+)[[:space:]]*#{Regexp.escape(TAG_BLOCK[1])}/mu
     end
 
     def lex_block_regex
       @lex_block_regex ||=
-        /\A\s*(?:
+        /\G[[:space:]]*(?:
           #{Regexp.union(
-            /#{WHITESPACE_TRIM}#{TAG_BLOCK[1]}\s*\n?/,
-            WHITESPACE_LINE_TRIM + TAG_BLOCK[1] + "[#{WHITESPACE_LINE_CHARS}]*",
+            /#{WHITESPACE_TRIM}#{TAG_BLOCK[1]}[[:space:]]*\n?/,
+            /#{WHITESPACE_LINE_TRIM}#{TAG_BLOCK[1]}[#{WHITESPACE_LINE_CHARS}]*/,
             /#{TAG_BLOCK[1]}\n?/
           )}
         )/x
@@ -326,20 +445,25 @@ module Twig
       @lex_raw_data_regex ||=
         /
           #{TAG_BLOCK[0]}
-          (#{Regexp.union(WHITESPACE_TRIM, WHITESPACE_LINE_TRIM)})?\s*endverbatim\s*
+          (#{Regexp.union(WHITESPACE_TRIM, WHITESPACE_LINE_TRIM)})?[[:space:]]*endverbatim[[:space:]]*
           (?:#{Regexp.union(
-            "#{WHITESPACE_TRIM}#{TAG_BLOCK[1]}\\s*",
-            WHITESPACE_LINE_TRIM + TAG_BLOCK[1] + "[#{WHITESPACE_LINE_CHARS}]*",
+            /#{WHITESPACE_TRIM}#{TAG_BLOCK[1]}[[:space:]]*/,
+            /#{WHITESPACE_LINE_TRIM}#{TAG_BLOCK[1]}[#{WHITESPACE_LINE_CHARS}]*/,
             TAG_BLOCK[1]
           )})
-        /sx
+        /xu
     end
 
     def operator_regex
       return @operator_regex if defined?(@operator_regex)
 
-      unary, binary = @environment.operators
-      operators = ([:'='] + unary.keys + binary.keys).
+      expression_parsers = ['=']
+
+      environment.expression_parsers.each do |expression_parser|
+        expression_parsers.concat([expression_parser.name], expression_parser.aliases)
+      end
+
+      operators = expression_parsers.
         to_h { |op| [op, op.length] }.
         sort_by { |_, length| -length }.
         to_h
@@ -352,7 +476,7 @@ module Twig
         # an operator that ends with a character must be followed by
         # a whitespace, a parenthesis, an opening map [ or sequence {
         if operator[-1].match(/\w/)
-          regex << '(?=[\s()\[{])'
+          regex << '(?=[[[:space:]]()\[{])'
         end
 
         # an operator that begins with a character must not have a dot or pipe before
@@ -361,12 +485,32 @@ module Twig
         end
 
         # an operator with a space can be any amount of whitespaces
-        regex.gsub!(/\s+/, '\s+')
+        regex.gsub!(/(\\\s)+/, '[[:space:]]+')
 
         chain << regex
       end
 
-      @operator_regex = Regexp.new("\\A(?:#{chain.join('|')})")
+      @operator_regex = Regexp.new("\\G(?:#{chain.join('|')})")
     end
+
+    # @param [String]
+    def check_brackets(code)
+      if OPENING_BRACKET.include?(code)
+        @brackets << [code, @lineno]
+      elsif CLOSING_BRACKET.include?(code)
+        if @brackets.empty?
+          raise Error::Syntax.new("Unexpected \"#{code}\".", @lineno, @source)
+        end
+
+        expect, lineno = @brackets.pop
+
+        unless code_at?(0, expect.tr(OPENING_BRACKET.join, CLOSING_BRACKET.join))
+          raise Error::Syntax.new("Unclosed \"#{expect}\".", lineno, @source)
+        end
+      end
+    end
+
+    # @return [Environment]
+    attr_reader :environment
   end
 end

data/lib/twig/loader/array.rb

@@ -7,33 +7,50 @@ module Twig
       def initialize(templates)
         super()
 
-        @templates = templates.transform_keys(&:to_sym)
+        @templates = templates.transform_keys { |name| normalize_name(name) }
       end
 
       def get_source_context(name)
-        name = name.to_sym
-        raise "LoaderError: Template #{name} is not defined" unless @templates[name]
+        name = normalize_name(name)
+
+        unless @templates[name]
+          raise Error::Loader, "Template \"#{name}\" is not defined."
+        end
 
         ::Twig::Source.new(@templates[name], name)
       end
 
       def exists?(name)
-        @templates.key?(name.to_sym)
+        name = normalize_name(name)
+
+        @templates.key?(name)
       end
 
       def get_cache_key(name)
-        name = name.to_sym
-        raise "LoaderError: Template #{name} is not defined" unless @templates[name]
+        name = normalize_name(name)
+
+        unless @templates[name]
+          raise Error::Loader, "Template \"#{name}\" is not defined."
+        end
 
         "#{name}:#{@templates[name]}"
       end
 
       def fresh?(name, time)
-        name = name.to_sym
-        raise "LoaderError: Template #{name} is not defined" unless @templates[name]
+        name = normalize_name(name)
+
+        unless @templates[name]
+          raise Error::Loader, "Template \"#{name}\" is not defined."
+        end
 
         true
       end
+
+      private
+
+      def normalize_name(name)
+        name.to_s
+      end
     end
   end
 end

data/lib/twig/loader/chain.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Twig
+  module Loader
+    class Chain < Loader::Base
+      # @param [Array<Loader::Base>] loaders
+      def initialize(loaders)
+        super()
+
+        @loaders = loaders
+      end
+
+      def get_source_context(name)
+        exceptions = []
+
+        loaders.each do |loader|
+          unless loader.exists?(name)
+            next
+          end
+
+          begin
+            return loader.get_source_context(name)
+          rescue Twig::Error::Loader => e
+            exceptions << "#{loader.class.name}: #{e.message}"
+          end
+        end
+
+        exceptions = exceptions.any? ? "(#{exceptions.join(', ')})" : ''
+
+        raise Twig::Error::Loader, "Template \"#{name}\" is not defined#{exceptions}."
+      end
+
+      def exists?(name)
+        @has_source_cache ||= {}
+
+        return @has_source_cache[name] if @has_source_cache.key?(name)
+
+        loaders.each do |loader|
+          if loader.exists?(name)
+            return @has_source_cache[name] = true
+          end
+        end
+
+        @has_source_cache[name] = false
+      end
+
+      def get_cache_key(name)
+        exceptions = []
+
+        loaders.each do |loader|
+          unless loader.exists?(name)
+            next
+          end
+
+          begin
+            return loader.get_cache_key(name)
+          rescue Twig::Error::Loader => e
+            exceptions << "#{loader.class.name}: #{e.message}"
+          end
+        end
+
+        exceptions = exceptions.any? ? "(#{exceptions.join(', ')})" : ''
+
+        raise Twig::Error::Loader, "Template \"#{name}\" is not defined#{exceptions}."
+      end
+
+      def fresh?(name, time)
+        exceptions = []
+
+        loaders.each do |loader|
+          unless loader.exists?(name)
+            next
+          end
+
+          begin
+            return loader.fresh?(name, time)
+          rescue Twig::Error::Loader => e
+            exceptions << "#{loader.class.name}: #{e.message}"
+          end
+        end
+
+        exceptions = exceptions.any? ? "(#{exceptions.join(', ')})" : ''
+
+        raise Twig::Error::Loader, "Template \"#{name}\" is not defined#{exceptions}."
+      end
+
+      private
+
+      # @return [::Array<Loader::Base>]
+      attr_accessor :loaders
+    end
+  end
+end