better_html 1.0.14 → 2.0.0

data/ext/better_html_ext/tokenizer.h

@@ -0,0 +1,80 @@
+#pragma once
+
+enum tokenizer_context {
+  TOKENIZER_NONE = 0,
+  TOKENIZER_HTML,
+  TOKENIZER_OPEN_TAG,
+  TOKENIZER_SOLIDUS_OR_TAG_NAME,
+  TOKENIZER_TAG_NAME,
+  TOKENIZER_CDATA,
+  TOKENIZER_RCDATA, // title, textarea
+  TOKENIZER_RAWTEXT, // style, xmp, iframe, noembed, noframes
+  TOKENIZER_SCRIPT_DATA, // script
+  TOKENIZER_PLAINTEXT, // plaintext
+  TOKENIZER_COMMENT,
+  TOKENIZER_ATTRIBUTE_NAME,
+  TOKENIZER_ATTRIBUTE_VALUE,
+  TOKENIZER_ATTRIBUTE_UNQUOTED,
+  TOKENIZER_ATTRIBUTE_QUOTED,
+};
+
+enum token_type {
+  TOKEN_NONE = 0,
+  TOKEN_TEXT,
+  TOKEN_WHITESPACE,
+  TOKEN_COMMENT_START,
+  TOKEN_COMMENT_END,
+  TOKEN_TAG_START,
+  TOKEN_TAG_NAME,
+  TOKEN_TAG_END,
+  TOKEN_ATTRIBUTE_NAME,
+  TOKEN_ATTRIBUTE_QUOTED_VALUE_START,
+  TOKEN_ATTRIBUTE_QUOTED_VALUE,
+  TOKEN_ATTRIBUTE_QUOTED_VALUE_END,
+  TOKEN_ATTRIBUTE_UNQUOTED_VALUE,
+  TOKEN_CDATA_START,
+  TOKEN_CDATA_END,
+  TOKEN_SOLIDUS,
+  TOKEN_EQUAL,
+  TOKEN_MALFORMED,
+};
+
+struct scan_t {
+  char *string;
+  long unsigned int cursor;
+  long unsigned int length;
+
+  int enc_index;
+  long unsigned int mb_cursor;
+};
+
+struct tokenizer_t
+{
+  enum tokenizer_context context[1000];
+  uint32_t current_context;
+
+  void *callback_data;
+  void (*f_callback)(struct tokenizer_t *tk, enum token_type type, long unsigned int length, void *data);
+
+  char attribute_value_start;
+  int found_attribute;
+
+  char *current_tag;
+
+  int is_closing_tag;
+  enum token_type last_token;
+
+  struct scan_t scan;
+};
+
+
+void Init_html_tokenizer_tokenizer(VALUE mHtmlTokenizer);
+void tokenizer_init(struct tokenizer_t *tk);
+void tokenizer_free_members(struct tokenizer_t *tk);
+void tokenizer_set_scan_string(struct tokenizer_t *tk, const char *string, long unsigned int length);
+void tokenizer_free_scan_string(struct tokenizer_t *tk);
+void tokenizer_scan_all(struct tokenizer_t *tk);
+VALUE token_type_to_symbol(enum token_type type);
+
+extern const rb_data_type_t ht_tokenizer_data_type;
+#define Tokenizer_Get_Struct(obj, sval) TypedData_Get_Struct(obj, struct tokenizer_t, &ht_tokenizer_data_type, sval)

data/lib/better_html/ast/iterator.rb

@@ -1,9 +1,20 @@
-require 'ast'
-require 'active_support/core_ext/array/wrap'
+# frozen_string_literal: true
+
+require "ast"
+require "active_support/core_ext/array/wrap"
 
 module BetterHtml
   module AST
     class Iterator
+      class << self
+        def descendants(root_node, type)
+          Enumerator.new do |yielder|
+            t = new(type) { |node| yielder << node }
+            t.traverse(root_node)
+          end
+        end
+      end
+
       def initialize(types, &block)
         @types = types.nil? ? nil : Array.wrap(types)
         @block = block
@@ -11,6 +22,7 @@ module BetterHtml
 
       def traverse(node)
         return unless node.is_a?(::AST::Node)
+
         @block.call(node) if @types.nil? || @types.include?(node.type)
         traverse_all(node)
       end
@@ -20,13 +32,6 @@ module BetterHtml
           traverse(node) if node.is_a?(::AST::Node)
         end
       end
-
-      def self.descendants(root_node, type)
-        Enumerator.new do |yielder|
-          t = new(type) { |node| yielder << node }
-          t.traverse(root_node)
-        end
-      end
     end
   end
 end

data/lib/better_html/ast/node.rb

@@ -1,5 +1,7 @@
-require 'ast'
-require_relative 'iterator'
+# frozen_string_literal: true
+
+require "ast"
+require_relative "iterator"
 
 module BetterHtml
   module AST

data/lib/better_html/better_erb/erubi_implementation.rb

@@ -1,50 +1,54 @@
-require 'action_view'
-require_relative 'runtime_checks'
-
-class BetterHtml::BetterErb
-  class ErubiImplementation < ActionView::Template::Handlers::ERB::Erubi
-    include RuntimeChecks
-
-    def add_text(text)
-      return if text.empty?
-
-      if text == "\n"
-        @parser.parse("\n")
-        @newline_pending += 1
-      else
-        src << "@output_buffer.safe_append='"
-        src << "\n" * @newline_pending if @newline_pending > 0
-        src << escape_text(text)
-        src << "'.freeze;"
-
-        @parser.parse(text) do |*args|
-          check_token(*args)
-        end
+# frozen_string_literal: true
+
+require "action_view"
+require_relative "runtime_checks"
+
+module BetterHtml
+  class BetterErb
+    class ErubiImplementation < ActionView::Template::Handlers::ERB::Erubi
+      include RuntimeChecks
+
+      def add_text(text)
+        return if text.empty?
 
-        @newline_pending = 0
+        if text == "\n"
+          @parser.parse("\n")
+          @newline_pending += 1
+        else
+          src << "@output_buffer.safe_append='"
+          src << "\n" * @newline_pending if @newline_pending > 0
+          src << escape_text(text)
+          src << "'.freeze;"
+
+          @parser.parse(text) do |*args|
+            check_token(*args)
+          end
+
+          @newline_pending = 0
+        end
       end
-    end
 
-    def add_expression(indicator, code)
-      if (indicator == "==") || @escape
-        add_expr_auto_escaped(src, code, false)
-      else
-        add_expr_auto_escaped(src, code, true)
+      def add_expression(indicator, code)
+        if (indicator == "==") || @escape
+          add_expr_auto_escaped(src, code, false)
+        else
+          add_expr_auto_escaped(src, code, true)
+        end
       end
-    end
 
-    def add_code(code)
-      flush_newline_if_pending(src)
+      def add_code(code)
+        flush_newline_if_pending(src)
 
-      block_check(src, "<%#{code}%>")
-      @parser.append_placeholder(code)
-      super
-    end
+        block_check(src, "<%#{code}%>")
+        @parser.append_placeholder(code)
+        super
+      end
 
-    private
+      private
 
-    def escape_text(text)
-      text.gsub(/['\\]/, '\\\\\&')
+      def escape_text(text)
+        text.gsub(/['\\]/, '\\\\\&')
+      end
     end
   end
 end

data/lib/better_html/better_erb/runtime_checks.rb

@@ -1,162 +1,169 @@
-require 'html_tokenizer'
-require 'action_view'
-
-class BetterHtml::BetterErb
-  module RuntimeChecks
-    def initialize(erb, config: BetterHtml.config, **options)
-      @parser = HtmlTokenizer::Parser.new
-      @config = config
-      super(erb, **options)
-    end
+# frozen_string_literal: true
 
-    def validate!
-      check_parser_errors
+require "action_view"
 
-      unless @parser.context == :none
-        raise BetterHtml::HtmlError, 'Detected an open tag at the end of this document.'
+module BetterHtml
+  class BetterErb
+    module RuntimeChecks
+      def initialize(erb, config: BetterHtml.config, **options)
+        @parser = HtmlTokenizer::Parser.new
+        @config = config
+        super(erb, **options)
       end
-    end
 
-    private
+      def validate!
+        check_parser_errors unless @config.disable_parser_validation
 
-    def class_name
-      "BetterHtml::BetterErb::ValidatedOutputBuffer"
-    end
+        unless @parser.context == :none
+          raise BetterHtml::HtmlError, "Detected an open tag at the end of this document."
+        end
+      end
 
-    def wrap_method
-      "#{class_name}.wrap"
-    end
+      private
 
-    def add_expr_auto_escaped(src, code, auto_escape)
-      flush_newline_if_pending(src)
+      def class_name
+        "BetterHtml::BetterErb::ValidatedOutputBuffer"
+      end
 
-      src << "#{wrap_method}(@output_buffer, (#{parser_context.inspect}), '#{escape_text(code)}'.freeze, #{auto_escape})"
-      method_name = "safe_#{@parser.context}_append"
-      if code =~ self.class::BLOCK_EXPR
-        block_check(src, "<%=#{code}%>")
-        src << ".#{method_name}= " << code
-      else
-        src << ".#{method_name}=(" << code << ");"
+      def wrap_method
+        "#{class_name}.wrap"
       end
-      @parser.append_placeholder("<%=#{code}%>")
-    end
 
-    def parser_context
-      if [:quoted_value, :unquoted_value, :space_after_attribute].include?(@parser.context)
-        {
-          tag_name: @parser.tag_name,
-          attribute_name: @parser.attribute_name,
-          attribute_value: @parser.attribute_value,
-          attribute_quoted: @parser.attribute_quoted?,
-          quote_character: @parser.quote_character,
-        }
-      elsif [:attribute_name, :after_attribute_name, :after_equal].include?(@parser.context)
-        {
-          tag_name: @parser.tag_name,
-          attribute_name: @parser.attribute_name,
-        }
-      elsif [:tag, :tag_name, :tag_end].include?(@parser.context)
-        {
-          tag_name: @parser.tag_name,
-        }
-      elsif @parser.context == :rawtext
-        {
-          tag_name: @parser.tag_name,
-          rawtext_text: @parser.rawtext_text,
-        }
-      elsif @parser.context == :comment
-        {
-          comment_text: @parser.comment_text,
-        }
-      elsif [:none, :solidus_or_tag_name].include?(@parser.context)
-        {}
-      else
-        raise RuntimeError, "Tried to interpolate into unknown location #{@parser.context}."
+      def add_expr_auto_escaped(src, code, auto_escape)
+        flush_newline_if_pending(src)
+
+        escaped_code = escape_text(code)
+
+        src << "#{wrap_method}(@output_buffer, (#{parser_context.inspect}), '#{escaped_code}'.freeze, #{auto_escape})"
+        method_name = "safe_#{@parser.context}_append"
+        if code =~ self.class::BLOCK_EXPR
+          block_check(src, "<%=#{code}%>")
+          src << ".#{method_name}= " << code
+        else
+          src << ".#{method_name}=(" << code << ");"
+        end
+        @parser.append_placeholder("<%=#{code}%>")
       end
-    end
 
-    def block_check(src, code)
-      unless @parser.context == :none || @parser.context == :rawtext
-        s = "Ruby statement not allowed.\n"
-        s << "In '#{@parser.context}' on line #{@parser.line_number} column #{@parser.column_number}:\n"
-        prefix = extract_line(@parser.line_number)
-        code = code.lines.first
-        s << "#{prefix}#{code}\n"
-        s << "#{' ' * prefix.size}#{'^' * code.size}"
-        raise BetterHtml::DontInterpolateHere, s
+      def parser_context
+        if [:quoted_value, :unquoted_value, :space_after_attribute].include?(@parser.context)
+          {
+            tag_name: @parser.tag_name,
+            attribute_name: @parser.attribute_name,
+            attribute_value: @parser.attribute_value,
+            attribute_quoted: @parser.attribute_quoted?,
+            quote_character: @parser.quote_character,
+          }
+        elsif [:attribute_name, :after_attribute_name, :after_equal].include?(@parser.context)
+          {
+            tag_name: @parser.tag_name,
+            attribute_name: @parser.attribute_name,
+          }
+        elsif [:tag, :tag_name, :tag_end].include?(@parser.context)
+          {
+            tag_name: @parser.tag_name,
+          }
+        elsif @parser.context == :rawtext
+          {
+            tag_name: @parser.tag_name,
+            rawtext_text: @parser.rawtext_text,
+          }
+        elsif @parser.context == :comment
+          {
+            comment_text: @parser.comment_text,
+          }
+        elsif [:none, :solidus_or_tag_name].include?(@parser.context)
+          {}
+        else
+          raise "Tried to interpolate into unknown location #{@parser.context}."
+        end
       end
-    end
 
-    def check_parser_errors
-      errors = @parser.errors
-      return if errors.empty?
-
-      s = "#{errors.size} error(s) found in HTML document.\n"
-      errors.each do |error|
-        s = "#{error.message}\n"
-        s << "On line #{error.line} column #{error.column}:\n"
-        line = extract_line(error.line)
-        s << "#{line}\n"
-        s << "#{' ' * (error.column)}#{'^' * (line.size - error.column)}"
+      def block_check(src, code)
+        unless @parser.context == :none || @parser.context == :rawtext
+          s = +"Ruby statement not allowed.\n"
+          s << "In '#{@parser.context}' on line #{@parser.line_number} column #{@parser.column_number}:\n"
+          prefix = extract_line(@parser.line_number)
+          code = code.lines.first
+          s << "#{prefix}#{code}\n"
+          s << "#{" " * prefix.size}#{"^" * code.size}"
+          raise BetterHtml::DontInterpolateHere, s
+        end
       end
 
-      raise BetterHtml::HtmlError, s
-    end
+      def check_parser_errors
+        errors = @parser.errors
+        return if errors.empty?
 
-    def check_token(type, *args)
-      check_tag_name(type, *args) if type == :tag_name
-      check_attribute_name(type, *args) if type == :attribute_name
-      check_quoted_value(type, *args) if type == :attribute_quoted_value_start
-      check_unquoted_value(type, *args) if type == :attribute_unquoted_value
-    end
+        s = +"#{errors.size} error(s) found in HTML document.\n"
+        errors.each do |error|
+          s << "#{error.message}\n"
+          s << "On line #{error.line} column #{error.column}:\n"
+          line = extract_line(error.line)
+          s << "#{line}\n"
+          s << "#{" " * error.column}#{"^" * (line.size - error.column)}"
+        end
 
-    def check_tag_name(type, start, stop, line, column)
-      text = @parser.document[start...stop]
-      return if text.upcase == "!DOCTYPE"
-      return if @config.partial_tag_name_pattern === text
+        raise BetterHtml::HtmlError, s
+      end
 
-      s = "Invalid tag name #{text.inspect} does not match "\
-        "regular expression #{@config.partial_tag_name_pattern.inspect}\n"
-      s << build_location(line, column, text.size)
-      raise BetterHtml::HtmlError, s
-    end
+      def check_token(type, *args)
+        check_tag_name(type, *args) if type == :tag_name
+        check_attribute_name(type, *args) if type == :attribute_name
+        check_quoted_value(type, *args) if type == :attribute_quoted_value_start
+        check_unquoted_value(type, *args) if type == :attribute_unquoted_value
+      end
 
-    def check_attribute_name(type, start, stop, line, column)
-      text = @parser.document[start...stop]
-      return if @config.partial_attribute_name_pattern === text
+      def check_tag_name(type, start, stop, line, column)
+        text = @parser.document[start...stop]
+        return if text.upcase == "!DOCTYPE"
+        return if @config.partial_tag_name_pattern.match?(text)
 
-      s = "Invalid attribute name #{text.inspect} does not match "\
-        "regular expression #{@config.partial_attribute_name_pattern.inspect}\n"
-      s << build_location(line, column, text.size)
-      raise BetterHtml::HtmlError, s
-    end
+        s = +"Invalid tag name #{text.inspect} does not match "\
+          "regular expression #{@config.partial_tag_name_pattern.inspect}\n"
+        s << build_location(line, column, text.size)
+        raise BetterHtml::HtmlError, s
+      end
 
-    def check_quoted_value(type, start, stop, line, column)
-      return if @config.allow_single_quoted_attributes
-      text = @parser.document[start...stop]
-      return if text == '"'
+      def check_attribute_name(type, start, stop, line, column)
+        text = @parser.document[start...stop]
+        return if @config.partial_attribute_name_pattern.match?(text)
 
-      s = "Single-quoted attributes are not allowed\n"
-      s << build_location(line, column, text.size)
-      raise BetterHtml::HtmlError, s
-    end
+        s = +"Invalid attribute name #{text.inspect} does not match "\
+          "regular expression #{@config.partial_attribute_name_pattern.inspect}\n"
+        s << build_location(line, column, text.size)
+        raise BetterHtml::HtmlError, s
+      end
 
-    def check_unquoted_value(type, start, stop, line, column)
-      return if @config.allow_unquoted_attributes
-      s = "Unquoted attribute values are not allowed\n"
-      s << build_location(line, column, stop-start)
-      raise BetterHtml::HtmlError, s
-    end
+      def check_quoted_value(type, start, stop, line, column)
+        return if @config.allow_single_quoted_attributes
 
-    def build_location(line, column, length)
-      s = "On line #{line} column #{column}:\n"
-      s << "#{extract_line(line)}\n"
-      s << "#{' ' * column}#{'^' * length}"
-    end
+        text = @parser.document[start...stop]
+        return if text == '"'
 
-    def extract_line(line)
-      line = @parser.document.lines[line-1]
-      line.nil? ? "" : line.gsub(/\n$/, '')
+        s = +"Single-quoted attributes are not allowed\n"
+        s << build_location(line, column, text.size)
+        raise BetterHtml::HtmlError, s
+      end
+
+      def check_unquoted_value(type, start, stop, line, column)
+        return if @config.allow_unquoted_attributes
+
+        s = +"Unquoted attribute values are not allowed\n"
+        s << build_location(line, column, stop - start)
+        raise BetterHtml::HtmlError, s
+      end
+
+      def build_location(line, column, length)
+        s = +"On line #{line} column #{column}:\n"
+        s << "#{extract_line(line)}\n"
+        s << "#{" " * column}#{"^" * length}"
+      end
+
+      def extract_line(line)
+        line = @parser.document.lines[line - 1]
+        line.nil? ? "" : line.gsub(/\n$/, "")
+      end
     end
   end
 end