better_html 0.0.12 → 1.0.0

data/lib/better_html/tree/attribute.rb

@@ -0,0 +1,31 @@
+require 'ast'
+
+module BetterHtml
+  module Tree
+    class Attribute
+      attr_reader :node, :name_node, :equal_node, :value_node
+
+      def initialize(node)
+        @node = node
+        @name_node, @equal_node, @value_node = *node
+      end
+
+      def self.from_node(node)
+        new(node)
+      end
+
+      def loc
+        @node.loc
+      end
+
+      def name
+        @name_node&.loc&.source&.downcase
+      end
+
+      def value
+        parts = value_node.to_a.reject{ |node| node.is_a?(::AST::Node) && node.type == :quote }
+        parts.map { |s| s.is_a?(::AST::Node) ? s.loc.source : CGI.unescapeHTML(s) }.join
+      end
+    end
+  end
+end

data/lib/better_html/tree/attributes_list.rb

@@ -0,0 +1,25 @@
+require 'better_html/tree/attribute'
+
+module BetterHtml
+  module Tree
+    class AttributesList
+      def initialize(list)
+        @list = list || []
+      end
+
+      def self.from_nodes(nodes)
+        new(nodes&.map { |node| Tree::Attribute.from_node(node) })
+      end
+
+      def [](name)
+        @list.find do |attribute|
+          attribute.name == name.downcase
+        end
+      end
+
+      def each(&block)
+        @list.each(&block)
+      end
+    end
+  end
+end

data/lib/better_html/tree/tag.rb

@@ -0,0 +1,39 @@
+require 'better_html/tree/attributes_list'
+require 'better_html/ast/iterator'
+
+module BetterHtml
+  module Tree
+    class Tag
+      attr_reader :node
+
+      def initialize(node)
+        @node = node
+        @start_solidus, @name_node, @attributes_node, @end_solidus = *node
+      end
+
+      def self.from_node(node)
+        new(node)
+      end
+
+      def loc
+        @node.loc
+      end
+
+      def name
+        @name_node&.loc&.source&.downcase
+      end
+
+      def closing?
+        @start_solidus&.type == :solidus
+      end
+
+      def self_closing?
+        @end_solidus&.type == :solidus
+      end
+
+      def attributes
+        @attributes ||= AttributesList.from_nodes(@attributes_node.to_a)
+      end
+    end
+  end
+end

data/lib/better_html/version.rb

@@ -1,3 +1,3 @@
 module BetterHtml
-  VERSION = "0.0.12"
+  VERSION = "1.0.0"
 end

data/test/better_html/parser_test.rb

@@ -0,0 +1,279 @@
+require 'test_helper'
+require 'better_html/parser'
+require 'ast'
+
+module BetterHtml
+  class ParserTest < ActiveSupport::TestCase
+    include ::AST::Sexp
+
+    test "parse empty document" do
+      tree = Parser.new('')
+
+      assert_equal s(:document), tree.ast
+    end
+
+    test "consume cdata nodes" do
+      code = "<![CDATA[ foo ]]>"
+      tree = Parser.new(code)
+
+      assert_equal s(:document, s(:cdata, ' foo ')), tree.ast
+      assert_equal code, tree.ast.loc.source
+    end
+
+    test "unterminated cdata nodes are consumed until end" do
+      code = "<![CDATA[ foo"
+      tree = Parser.new(code)
+
+      assert_equal s(:document, s(:cdata, ' foo')), tree.ast
+      assert_equal code, tree.ast.loc.source
+    end
+
+    test "consume cdata with interpolation" do
+      code = "<![CDATA[ foo <%= bar %> baz ]]>"
+      tree = Parser.new(code)
+
+      assert_equal s(:document,
+        s(:cdata,
+          " foo ",
+          s(:erb, s(:indicator, '='), nil, s(:code, " bar "), nil),
+          " baz "
+        )),
+        tree.ast
+      assert_equal code, tree.ast.loc.source
+    end
+
+    test "consume comment nodes" do
+      tree = Parser.new("<!-- foo -->")
+
+      assert_equal s(:document, s(:comment, ' foo ')), tree.ast
+    end
+
+    test "unterminated comment nodes are consumed until end" do
+      tree = Parser.new("<!-- foo")
+
+      assert_equal s(:document, s(:comment, ' foo')), tree.ast
+    end
+
+    test "consume comment with interpolation" do
+      tree = Parser.new("<!-- foo <%= bar %> baz -->")
+
+      assert_equal s(:document,
+        s(:comment,
+          " foo ",
+          s(:erb, s(:indicator, "="), nil, s(:code, " bar "), nil),
+          " baz "
+        )),
+        tree.ast
+    end
+
+    test "consume tag nodes" do
+      tree = Parser.new("<div>")
+      assert_equal s(:document, s(:tag, nil, s(:tag_name, "div"), nil, nil)), tree.ast
+    end
+
+    test "tag without name" do
+      tree = Parser.new("foo < bar")
+      assert_equal s(:document,
+        s(:text, "foo "),
+        s(:tag, nil, nil,
+          s(:tag_attributes,
+            s(:attribute, s(:attribute_name, 'bar'), nil, nil)
+          ),
+          nil
+        )
+      ), tree.ast
+    end
+
+    test "consume tag nodes with solidus" do
+      tree = Parser.new("</div>")
+      assert_equal s(:document, s(:tag, s(:solidus), s(:tag_name, "div"), nil, nil)), tree.ast
+    end
+
+    test "sets self_closing when appropriate" do
+      tree = Parser.new("<div/>")
+      assert_equal s(:document, s(:tag, nil, s(:tag_name, "div"), nil, s(:solidus))), tree.ast
+    end
+
+    test "consume tag nodes until name ends" do
+      tree = Parser.new("<div/>")
+      assert_equal s(:document, s(:tag, nil, s(:tag_name, "div"), nil, s(:solidus))), tree.ast
+
+      tree = Parser.new("<div ")
+      assert_equal s(:document, s(:tag, nil, s(:tag_name, "div"), nil, nil)), tree.ast
+    end
+
+    test "consume tag nodes with interpolation" do
+      tree = Parser.new("<ns:<%= name %>-thing>")
+      assert_equal s(:document,
+        s(:tag,
+          nil,
+          s(:tag_name, "ns:", s(:erb, s(:indicator, "="), nil, s(:code, " name "), nil), "-thing"),
+          nil,
+          nil
+        )), tree.ast
+    end
+
+    test "consume tag attributes nodes unquoted value" do
+      tree = Parser.new("<div foo=bar>")
+      assert_equal s(:document,
+        s(:tag, nil,
+          s(:tag_name, "div"),
+          s(:tag_attributes,
+            s(:attribute,
+              s(:attribute_name, "foo"),
+              s(:equal),
+              s(:attribute_value, "bar")
+            )
+          ),
+          nil
+        )), tree.ast
+    end
+
+    test "consume attributes without name" do
+      tree = Parser.new("<div 'thing'>")
+      assert_equal s(:document,
+        s(:tag, nil,
+          s(:tag_name, "div"),
+          s(:tag_attributes,
+            s(:attribute,
+              nil,
+              nil,
+              s(:attribute_value, s(:quote, "'"), "thing", s(:quote, "'"))
+            )
+          ),
+          nil
+        )), tree.ast
+    end
+
+    test "consume tag attributes nodes quoted value" do
+      tree = Parser.new("<div foo=\"bar\">")
+      assert_equal s(:document,
+        s(:tag, nil,
+          s(:tag_name, "div"),
+          s(:tag_attributes,
+            s(:attribute,
+              s(:attribute_name, "foo"),
+              s(:equal),
+              s(:attribute_value, s(:quote, "\""), "bar", s(:quote, "\""))
+            )
+          ),
+          nil
+        )), tree.ast
+    end
+
+    test "consume tag attributes nodes interpolation in name and value" do
+      tree = Parser.new("<div data-<%= foo %>=\"some <%= value %> foo\">")
+      assert_equal s(:document,
+        s(:tag, nil,
+          s(:tag_name, "div"),
+          s(:tag_attributes,
+            s(:attribute,
+              s(:attribute_name, "data-", s(:erb, s(:indicator, "="), nil, s(:code, " foo "), nil)),
+              s(:equal),
+              s(:attribute_value,
+                s(:quote, "\""),
+                "some ",
+                s(:erb, s(:indicator, "="), nil, s(:code, " value "), nil),
+                " foo",
+                s(:quote, "\""),
+              ),
+            )
+          ),
+          nil
+        )), tree.ast
+    end
+
+    test "consume text nodes" do
+      tree = Parser.new("here is <%= some %> text")
+
+      assert_equal s(:document,
+        s(:text,
+          "here is ",
+          s(:erb, s(:indicator, "="), nil, s(:code, " some "), nil),
+          " text"
+        )), tree.ast
+    end
+
+    test "javascript template parsing works" do
+      tree = Parser.new("here is <%= some %> text", template_language: :javascript)
+
+      assert_equal s(:document,
+        s(:text,
+          "here is ",
+          s(:erb, s(:indicator, "="), nil, s(:code, " some "), nil),
+          " text"
+        )), tree.ast
+    end
+
+    test "javascript template does not consume html tags" do
+      tree = Parser.new("<div <%= some %> />", template_language: :javascript)
+
+      assert_equal s(:document,
+        s(:text,
+          "<div ",
+          s(:erb, s(:indicator, "="), nil, s(:code, " some "), nil),
+          " />"
+        )), tree.ast
+    end
+
+    test "lodash template parsing works" do
+      tree = Parser.new('<div class="[%= foo %]">', template_language: :lodash)
+
+      assert_equal s(:document,
+        s(:tag,
+          nil,
+          s(:tag_name, "div"),
+          s(:tag_attributes,
+            s(:attribute,
+              s(:attribute_name, "class"),
+              s(:equal),
+              s(:attribute_value,
+                s(:quote, "\""),
+                s(:lodash, s(:indicator, "="), s(:code, " foo ")),
+                s(:quote, "\"")
+              )
+            )
+          ),
+          nil
+        )
+      ), tree.ast
+    end
+
+    test "nodes are all nested under document" do
+      tree = Parser.new(<<~HTML)
+        some text
+        <!-- a comment -->
+        some more text
+        <%= an erb tag -%>
+        <div class="foo">
+          content
+        </div>
+      HTML
+
+      assert_equal s(:document,
+        s(:text, "some text\n"),
+        s(:comment, " a comment "),
+        s(:text,
+          "\nsome more text\n",
+          s(:erb, s(:indicator, '='), nil, s(:code, ' an erb tag '), s(:trim)),
+          "\n"
+        ),
+        s(:tag,
+          nil,
+          s(:tag_name, "div"),
+          s(:tag_attributes,
+            s(:attribute,
+              s(:attribute_name, "class"),
+              s(:equal),
+              s(:attribute_value, s(:quote, "\""), "foo", s(:quote, "\""))
+            )
+          ),
+          nil
+        ),
+        s(:text, "\n  content\n"),
+        s(:tag, s(:solidus), s(:tag_name, 'div'), nil, nil),
+        s(:text, "\n"),
+      ), tree.ast
+    end
+  end
+end

data/test/better_html/test_helper/safe_erb_tester_test.rb

@@ -281,6 +281,17 @@ module BetterHtml
         assert_equal "erb statement not allowed here; did you mean '<%=' ?", errors.first.message
       end
 
+      test "disallowed script types" do
+        errors = parse(<<-EOF).errors
+          <script type="text/bogus">
+          </script>
+        EOF
+
+        assert_equal 1, errors.size
+        assert_equal 'type="text/bogus"', errors.first.location.source
+        assert_equal "text/bogus is not a valid type, valid types are text/javascript, text/template, text/html", errors.first.message
+      end
+
       test "statements not allowed in javascript template" do
         errors = parse(<<-JS, template_language: :javascript).errors
           <% if foo %>

data/test/better_html/test_helper/safe_lodash_tester_test.rb

@@ -46,7 +46,17 @@ module BetterHtml
         EOF
 
         assert_equal 1, errors.size
-        assert_equal 'script', errors.first.location.source
+        assert_equal '<script type="text/javascript">', errors.first.location.source
+        assert_equal "No script tags allowed nested in lodash templates", errors.first.message
+      end
+
+      test "script tag names are unescaped" do
+        errors = parse(<<-EOF).errors
+          <script type="text/j&#x61;v&#x61;script"></script>
+        EOF
+
+        assert_equal 1, errors.size
+        assert_equal '<script type="text/j&#x61;v&#x61;script">', errors.first.location.source
         assert_equal "No script tags allowed nested in lodash templates", errors.first.message
       end
 

data/test/better_html/tokenizer/html_erb_test.rb

@@ -0,0 +1,158 @@
+require 'test_helper'
+require 'better_html/tokenizer/html_erb'
+
+module BetterHtml
+  module Tokenizer
+    class HtmlErbTest < ActiveSupport::TestCase
+      test "text" do
+        scanner = HtmlErb.new("just some text")
+        assert_equal 1, scanner.tokens.size
+
+        assert_attributes ({
+          type: :text,
+          loc: { start: 0, stop: 13, source: 'just some text' }
+        }), scanner.tokens[0]
+      end
+
+      test "statement" do
+        scanner = HtmlErb.new("<% statement %>")
+        assert_equal 3, scanner.tokens.size
+
+        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :code, loc: { start: 2, stop: 12, source: ' statement ' } }), scanner.tokens[1]
+        assert_attributes ({ type: :erb_end, loc: { start: 13, stop: 14, source: '%>' } }), scanner.tokens[2]
+      end
+
+      test "debug statement" do
+        scanner = HtmlErb.new("<%# statement %>")
+        assert_equal 4, scanner.tokens.size
+
+        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :indicator, loc: { start: 2, stop: 2, source: '#' } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { start: 3, stop: 13, source: ' statement ' } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_end, loc: { start: 14, stop: 15, source: '%>' } }), scanner.tokens[3]
+      end
+
+      test "when multi byte characters are present in erb" do
+        code = "<% ui_helper 'your store’s' %>"
+        scanner = HtmlErb.new(code)
+        assert_equal 3, scanner.tokens.size
+
+        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :code, loc: { start: 2, stop: 27, source: " ui_helper 'your store’s' " } }), scanner.tokens[1]
+        assert_attributes ({ type: :erb_end, loc: { start: 28, stop: 29, source: '%>' } }), scanner.tokens[2]
+        assert_equal code.length, scanner.current_position
+      end
+
+      test "when multi byte characters are present in text" do
+        code = "your store’s"
+        scanner = HtmlErb.new(code)
+        assert_equal 1, scanner.tokens.size
+
+        assert_attributes ({ type: :text, loc: { start: 0, stop: 11, source: 'your store’s' } }), scanner.tokens[0]
+        assert_equal code.length, scanner.current_position
+      end
+
+      test "when multi byte characters are present in html" do
+        code = "<div title='your store’s'>foo</div>"
+        scanner = HtmlErb.new(code)
+        assert_equal 14, scanner.tokens.size
+
+        assert_attributes ({ type: :tag_start, loc: { start: 0, stop: 0, source: '<' } }), scanner.tokens[0]
+        assert_attributes ({ type: :tag_name, loc: { start: 1, stop: 3, source: "div" } }), scanner.tokens[1]
+        assert_attributes ({ type: :whitespace, loc: { start: 4, stop: 4, source: " " } }), scanner.tokens[2]
+        assert_attributes ({ type: :attribute_name, loc: { start: 5, stop: 9, source: "title" } }), scanner.tokens[3]
+        assert_attributes ({ type: :equal, loc: { start: 10, stop: 10, source: "=" } }), scanner.tokens[4]
+        assert_attributes ({ type: :attribute_quoted_value_start, loc: { start: 11, stop: 11, source: "'" } }), scanner.tokens[5]
+        assert_attributes ({ type: :attribute_quoted_value, loc: { start: 12, stop: 23, source: "your store’s" } }), scanner.tokens[6]
+        assert_attributes ({ type: :attribute_quoted_value_end, loc: { start: 24, stop: 24, source: "'" } }), scanner.tokens[7]
+        assert_equal code.length, scanner.current_position
+      end
+
+      test "expression literal" do
+        scanner = HtmlErb.new("<%= literal %>")
+        assert_equal 4, scanner.tokens.size
+
+        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :indicator, loc: { start: 2, stop: 2, source: '=' } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { start: 3, stop: 11, source: ' literal ' } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_end, loc: { start: 12, stop: 13, source: '%>' } }), scanner.tokens[3]
+      end
+
+      test "expression escaped" do
+        scanner = HtmlErb.new("<%== escaped %>")
+        assert_equal 4, scanner.tokens.size
+
+        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :indicator, loc: { start: 2, stop: 3, source: '==' } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { start: 4, stop: 12, source: ' escaped ' } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_end, loc: { start: 13, stop: 14, source: '%>' } }), scanner.tokens[3]
+      end
+
+      test "line number for multi-line statements" do
+        scanner = HtmlErb.new("before <% multi\nline %> after")
+        assert_equal 5, scanner.tokens.size
+
+        assert_attributes ({ type: :text, loc: { line: 1, source: 'before ' } }), scanner.tokens[0]
+        assert_attributes ({ type: :erb_begin, loc: { line: 1, source: '<%' } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { line: 1, source: " multi\nline " } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_end, loc: { line: 2, source: "%>" } }), scanner.tokens[3]
+        assert_attributes ({ type: :text, loc: { line: 2, source: " after" } }), scanner.tokens[4]
+      end
+
+      test "multi-line statements with trim" do
+        scanner = HtmlErb.new("before\n<% multi\nline -%>\nafter")
+        assert_equal 7, scanner.tokens.size
+
+        assert_attributes ({ type: :text, loc: { line: 1, source: "before\n" } }), scanner.tokens[0]
+        assert_attributes ({ type: :erb_begin, loc: { line: 2, source: '<%' } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { line: 2, source: " multi\nline " } }), scanner.tokens[2]
+        assert_attributes ({ type: :trim, loc: { line: 3, source: "-" } }), scanner.tokens[3]
+        assert_attributes ({ type: :erb_end, loc: { line: 3, source: "%>" } }), scanner.tokens[4]
+        assert_attributes ({ type: :text, loc: { line: 3, source: "\n" } }), scanner.tokens[5]
+        assert_attributes ({ type: :text, loc: { line: 4, source: "after" } }), scanner.tokens[6]
+      end
+
+      test "multi-line expression with trim" do
+        scanner = HtmlErb.new("before\n<%= multi\nline -%>\nafter")
+        assert_equal 8, scanner.tokens.size
+
+        assert_attributes ({ type: :text, loc: { line: 1, source: "before\n" } }), scanner.tokens[0]
+        assert_attributes ({ type: :erb_begin, loc: { line: 2, source: '<%' } }), scanner.tokens[1]
+        assert_attributes ({ type: :indicator, loc: { line: 2, source: '=' } }), scanner.tokens[2]
+        assert_attributes ({ type: :code, loc: { line: 2, source: " multi\nline " } }), scanner.tokens[3]
+        assert_attributes ({ type: :trim, loc: { line: 3, source: "-" } }), scanner.tokens[4]
+        assert_attributes ({ type: :erb_end, loc: { line: 3, source: "%>" } }), scanner.tokens[5]
+        assert_attributes ({ type: :text, loc: { line: 3, source: "\n" } }), scanner.tokens[6]
+        assert_attributes ({ type: :text, loc: { line: 4, source: "after" } }), scanner.tokens[7]
+      end
+
+      test "line counts with comments" do
+        scanner = HtmlErb.new("before\n<%# BO$$ Mode %>\nafter")
+        assert_equal 7, scanner.tokens.size
+
+        assert_attributes ({ type: :text, loc: { line: 1, source: "before\n" } }), scanner.tokens[0]
+        assert_attributes ({ type: :erb_begin, loc: { line: 2, source: '<%' } }), scanner.tokens[1]
+        assert_attributes ({ type: :indicator, loc: { line: 2, source: '#' } }), scanner.tokens[2]
+        assert_attributes ({ type: :code, loc: { line: 2, source: " BO$$ Mode " } }), scanner.tokens[3]
+        assert_attributes ({ type: :erb_end, loc: { line: 2, source: "%>" } }), scanner.tokens[4]
+        assert_attributes ({ type: :text, loc: { line: 2, source: "\n" } }), scanner.tokens[5]
+        assert_attributes ({ type: :text, loc: { line: 3, source: "after" } }), scanner.tokens[6]
+      end
+
+      private
+
+      def assert_attributes(attributes, token)
+        attributes.each do |key, value|
+          if value.nil?
+            assert_nil token.send(key)
+          elsif value.is_a?(Hash)
+            assert_attributes(value, token.send(key))
+          else
+            assert_equal value, token.send(key)
+          end
+        end
+      end
+    end
+  end
+end