better_html 0.0.12 → 1.0.0

data/lib/better_html/test_helper/safe_lodash_tester.rb

@@ -1,4 +1,6 @@
 require 'better_html/test_helper/safety_error'
+require 'better_html/ast/iterator'
+require 'better_html/tree/tag'
 
 module BetterHtml
   module TestHelper
@@ -51,7 +53,7 @@ EOF
           @data = data
           @config = config
           @errors = Errors.new
-          @nodes = BetterHtml::NodeIterator.new(data, template_language: :lodash)
+          @parser = BetterHtml::Parser.new(data, template_language: :lodash)
           validate!
         end
 
@@ -60,73 +62,73 @@ EOF
         end
 
         def validate!
-          @nodes.each_with_index do |node, index|
-            case node
-            when BetterHtml::NodeIterator::Element
-              validate_element(node)
-
-              if node.name == 'script' && !node.closing?
-                add_error(
-                  "No script tags allowed nested in lodash templates",
-                  location: node.name_parts.first.location
-                )
-              end
-            when BetterHtml::NodeIterator::CData, BetterHtml::NodeIterator::Comment
-              validate_no_statements(node)
+          @parser.nodes_with_type(:tag).each do |tag_node|
+            tag = Tree::Tag.from_node(tag_node)
+            validate_tag_attributes(tag)
+            validate_no_statements(tag_node)
+
+            if tag.name == 'script' && !tag.closing?
+              add_error(
+                "No script tags allowed nested in lodash templates",
+                location: tag_node.loc
+              )
             end
           end
+
+          @parser.nodes_with_type(:cdata, :comment).each do |node|
+            validate_no_statements(node)
+          end
         end
 
-        def validate_element(element)
-          element.attributes.each do |attribute|
-            attribute.name_parts.each do |token|
-              add_no_statement_error(attribute, token) if token.type == :stmt
+        def lodash_nodes(node)
+          Enumerator.new do |yielder|
+            next if node.nil?
+            node.descendants(:lodash).each do |lodash_node|
+              indicator_node, code_node = *lodash_node
+              yielder.yield(lodash_node, indicator_node, code_node)
             end
+          end
+        end
+
+        def validate_tag_attributes(tag)
+          tag.attributes.each do |attribute|
+            lodash_nodes(attribute.value_node).each do |lodash_node, indicator_node, code_node|
+              next if indicator_node.nil?
 
-            attribute.value_parts.each do |token|
-              case token.type
-              when :stmt
-                add_no_statement_error(attribute, token)
-              when :expr_literal
-                validate_tag_expression(element, attribute.name, token)
-              when :expr_escaped
+              if indicator_node.loc.source == '='
+                validate_tag_expression(attribute, lodash_node)
+              elsif indicator_node.loc.source == '!'
                 add_error(
                   "lodash interpolation with '[%!' inside html attribute is never safe",
-                  location: token.location
+                  location: lodash_node.loc
                 )
               end
             end
           end
         end
 
-        def validate_tag_expression(node, attr_name, value_token)
-          if javascript_attribute_name?(attr_name) && !lodash_safe_javascript_expression?(value_token.code.strip)
+        def validate_tag_expression(attribute, lodash_node)
+          _, code_node = *lodash_node
+          source = code_node.loc.source.strip
+          if @config.javascript_attribute_name?(attribute.name) && !@config.lodash_safe_javascript_expression?(source)
             add_error(
               "lodash interpolation in javascript attribute "\
-              "`#{attr_name}` must call `JSON.stringify(#{value_token.code.strip})`",
-              location: value_token.location
+              "`#{attribute.name}` must call `JSON.stringify(#{source})`",
+              location: lodash_node.loc
             )
           end
         end
 
-        def javascript_attribute_name?(name)
-          @config.javascript_attribute_names.any?{ |other| other === name }
-        end
-
-        def lodash_safe_javascript_expression?(code)
-          @config.lodash_safe_javascript_expression.any?{ |other| other === code }
-        end
-
         def validate_no_statements(node)
-          node.content_parts.each do |token|
-            add_no_statement_error(node, token) if token.type == :stmt
+          lodash_nodes(node).each do |lodash_node, indicator_node, code_node|
+            add_no_statement_error(lodash_node.loc) if indicator_node.nil?
           end
         end
 
-        def add_no_statement_error(node, token)
+        def add_no_statement_error(loc)
           add_error(
             "javascript statement not allowed here; did you mean '[%=' ?",
-            location: token.location
+            location: loc
           )
         end
       end

data/lib/better_html/tokenizer/base_erb.rb

@@ -0,0 +1,79 @@
+require 'erubi'
+require_relative 'token'
+require_relative 'location'
+
+module BetterHtml
+  module Tokenizer
+    class BaseErb < ::Erubi::Engine
+      REGEXP_WITHOUT_TRIM = /<%(={1,2}|%)?(.*?)()?%>([ \t]*\r?\n)?/m
+      STMT_TRIM_MATCHER = /\A(-|#)?(.*?)(-)?\z/m
+      EXPR_TRIM_MATCHER = /\A(.*?)(-)?\z/m
+
+      attr_reader :tokens
+      attr_reader :current_position
+
+      def initialize(document)
+        @document = document
+        @tokens = []
+        @current_position = 0
+        super(document, regexp: REGEXP_WITHOUT_TRIM, trim: false)
+      end
+
+      private
+
+      def append(text)
+        @current_position += text.length
+      end
+
+      def add_code(code)
+        _, ltrim_or_comment, code, rtrim = *STMT_TRIM_MATCHER.match(code)
+        ltrim = ltrim_or_comment if ltrim_or_comment == '-'
+        indicator = ltrim_or_comment if ltrim_or_comment == '#'
+        add_erb_tokens(ltrim, indicator, code, rtrim)
+        append("<%#{ltrim}#{indicator}#{code}#{rtrim}%>")
+      end
+
+      def add_expression(indicator, code)
+        _, code, rtrim = *EXPR_TRIM_MATCHER.match(code)
+        add_erb_tokens(nil, indicator, code, rtrim)
+        append("<%#{indicator}#{code}#{rtrim}%>")
+      end
+
+      def add_erb_tokens(ltrim, indicator, code, rtrim)
+        pos = current_position
+
+        token = add_token(:erb_begin, pos, pos + 2)
+        pos += 2
+
+        if ltrim
+          token = add_token(:trim, pos, pos + ltrim.length)
+          pos += ltrim.length
+        end
+
+        if indicator
+          token = add_token(:indicator, pos, pos + indicator.length)
+          pos += indicator.length
+        end
+
+        token = add_token(:code, pos, pos + code.length)
+        pos += code.length
+
+        if rtrim
+          token = add_token(:trim, pos, pos + rtrim.length)
+          pos += rtrim.length
+        end
+
+        token = add_token(:erb_end, pos, pos + 2)
+      end
+
+      def add_token(type, start, stop, line = nil, column = nil)
+        token = Token.new(
+          type: type,
+          loc: Location.new(@document, start, stop - 1, line, column)
+        )
+        @tokens << token
+        token
+      end
+    end
+  end
+end

data/lib/better_html/tokenizer/html_erb.rb

@@ -0,0 +1,31 @@
+require 'html_tokenizer'
+require_relative 'base_erb'
+
+module BetterHtml
+  module Tokenizer
+    class HtmlErb < BaseErb
+      attr_reader :parser
+
+      def initialize(document)
+        @parser = HtmlTokenizer::Parser.new
+        super(document)
+      end
+
+      def current_position
+        @parser.document_length
+      end
+
+      private
+
+      def append(text)
+        @parser.append_placeholder(text)
+      end
+
+      def add_text(text)
+        @parser.parse(text) do |type, start, stop, line, column|
+          add_token(type, start, stop, line, column)
+        end
+      end
+    end
+  end
+end

data/lib/better_html/{node_iterator → tokenizer}/html_lodash.rb

@@ -1,8 +1,9 @@
+require 'active_support'
 require_relative 'token'
 require_relative 'location'
 
 module BetterHtml
-  class NodeIterator
+  module Tokenizer
     class HtmlLodash
       attr_reader :tokens
       attr_reader :parser
@@ -12,9 +13,9 @@ module BetterHtml
       self.lodash_evaluate = %r{(?:\[\%)(.+?)(?:\%\])}m
       self.lodash_interpolate = %r{(?:\[\%)!(.+?)(?:\%\])}m
 
-      def initialize(source)
-        @source = source
-        @scanner = StringScanner.new(source)
+      def initialize(document)
+        @document = document
+        @scanner = StringScanner.new(document)
         @parser = HtmlTokenizer::Parser.new
         @tokens = []
         scan!
@@ -28,20 +29,21 @@ module BetterHtml
           if scanned.present?
             captures = scan_pattern.match(scanned).captures
             if pre_match = captures[0]
-              add_text(pre_match) unless pre_match.blank?
+              add_text(pre_match) if pre_match.present?
             end
             match = captures[1]
             if code = lodash_escape.match(match)
-              add_expr_escape(match, code.captures[0])
+              add_lodash_tokens("=", code.captures[0])
             elsif code = lodash_interpolate.match(match)
-              add_expr_interpolate(match, code.captures[0])
+              add_lodash_tokens("!", code.captures[0])
             elsif code = lodash_evaluate.match(match)
-              add_stmt(match, code.captures[0])
+              add_lodash_tokens(nil, code.captures[0])
             else
               raise RuntimeError, 'unexpected match'
             end
+            @parser.append_placeholder(match)
           else
-            text = @source[(@scanner.pos)..(@source.size)]
+            text = @document[(@scanner.pos)..(@document.size)]
             add_text(text) unless text.blank?
             break
           end
@@ -61,40 +63,34 @@ module BetterHtml
 
       def add_text(text)
         @parser.parse(text) do |type, start, stop, line, column|
-          add_token(type, @parser.extract(start, stop), start: start, stop: stop, line: line, column: column)
+          add_token(type, start: start, stop: stop, line: line, column: column)
         end
       end
 
-      def add_stmt(text, code)
-        add_token(:stmt, text, code: code)
-        @parser.append_placeholder(text)
-      end
+      def add_lodash_tokens(indicator, code)
+        pos = @parser.document_length
 
-      def add_expr_interpolate(text, code)
-        add_token(:expr_escaped, text, code: code)
-        @parser.append_placeholder(text)
-      end
+        add_token(:lodash_begin, start: pos, stop: pos + 2)
+        pos += 2
+
+        if indicator
+          add_token(:indicator, start: pos, stop: pos + indicator.length)
+          pos += indicator.length
+        end
+
+        add_token(:code, start: pos, stop: pos + code.length)
+        pos += code.length
 
-      def add_expr_escape(text, code)
-        add_token(:expr_literal, text, code: code)
-        @parser.append_placeholder(text)
+        add_token(:lodash_end, start: pos, stop: pos + 2)
       end
 
-      def add_token(type, text, code: nil, start: nil, stop: nil, line: nil, column: nil)
-        start ||= @parser.document_length
-        stop ||= start + text.size
-        extra_attributes = if type == :tag_end
-          {
-            self_closing: @parser.self_closing_tag?
-          }
-        end
-        @tokens << Token.new(
+      def add_token(type, start: nil, stop: nil, line: nil, column: nil)
+        token = Token.new(
           type: type,
-          text: text,
-          code: code,
-          location: Location.new(@source, start, stop, line || @parser.line_number, column || @parser.column_number),
-          **(extra_attributes || {})
+          loc: Location.new(@document, start, stop-1, line, column)
         )
+        @tokens << token
+        token
       end
     end
   end

data/lib/better_html/tokenizer/javascript_erb.rb

@@ -0,0 +1,15 @@
+require_relative 'base_erb'
+
+module BetterHtml
+  module Tokenizer
+    class JavascriptErb < BaseErb
+      private
+
+      def add_text(text)
+        pos = current_position
+        add_token(:text, pos, pos + text.size) if text.present?
+        append(text)
+      end
+    end
+  end
+end

data/lib/better_html/{node_iterator → tokenizer}/location.rb

@@ -1,9 +1,13 @@
 module BetterHtml
-  class NodeIterator
+  module Tokenizer
     class Location
       attr_accessor :start, :stop
 
       def initialize(document, start, stop, line = nil, column = nil)
+        raise ArgumentError, "start location #{start} is out of range for document of size #{document.size}" if start > document.size
+        raise ArgumentError, "stop location #{stop} is out of range for document of size #{document.size}" if stop > document.size
+        raise ArgumentError, "end of range must be greater than start of range (#{stop} < #{start})" if stop < start
+
         @document = document
         @start = start
         @stop = stop
@@ -12,7 +16,7 @@ module BetterHtml
       end
 
       def range
-        Range.new(start, stop-1)
+        Range.new(start, stop)
       end
 
       def source
@@ -31,17 +35,19 @@ module BetterHtml
         line_content = extract_line(line: line)
         spaces = line_content.scan(/\A\s*/).first
         column_without_spaces = [column - spaces.length, 0].max
-        underscore_length = [[stop - start, line_content.length - column_without_spaces].min, 1].max
+        underscore_length = [[stop - start + 1, line_content.length - column_without_spaces].min, 1].max
         "#{line_content.gsub(/\A\s*/, '')}\n#{' ' * column_without_spaces}#{'^' * underscore_length}"
       end
 
       private
 
       def calculate_line
+        return 1 if start == 0
         @document[0..start-1].scan("\n").count + 1
       end
 
       def calculate_column
+        return 0 if start == 0
         @document[0..start-1]&.split("\n", -1)&.last&.length || 0
       end
 

data/lib/better_html/tokenizer/token.rb

@@ -0,0 +1,16 @@
+module BetterHtml
+  module Tokenizer
+    class Token
+      attr_reader :type, :loc
+
+      def initialize(type:, loc:)
+        @type = type
+        @loc = loc
+      end
+
+      def inspect
+        "t(#{type.inspect}, #{loc&.source.inspect})"
+      end
+    end
+  end
+end

data/lib/better_html/tokenizer/token_array.rb

@@ -0,0 +1,54 @@
+module BetterHtml
+  module Tokenizer
+    class TokenArray
+      def initialize(list)
+        @list = list
+        @current = 0
+        @last = @list.size
+      end
+
+      def shift
+        raise RuntimeError, 'no tokens left to shift' if empty?
+        item = @list[@current]
+        @current += 1
+        item
+      end
+
+      def pop
+        raise RuntimeError, 'no tokens left to pop' if empty?
+        item = @list[@last - 1]
+        @last -= 1
+        item
+      end
+
+      def trim(type)
+        while current&.type == type
+          shift
+        end
+        while last&.type == type
+          pop
+        end
+      end
+
+      def empty?
+        size <= 0
+      end
+
+      def any?
+        !empty?
+      end
+
+      def current
+        @list[@current] unless empty?
+      end
+
+      def last
+        @list[@last - 1] unless empty?
+      end
+
+      def size
+        @last - @current
+      end
+    end
+  end
+end