haml 6.4.0 → 7.5.1

data/lib/haml/engine.rb

@@ -17,13 +17,13 @@ module Haml
       :buffer_class,
       generator:    Temple::Generators::StringBuffer,
       format:       :html,
-      attr_quote:   "'",
+      attr_quote:   '"',
       escape_html:  true,
       escape_attrs: true,
       autoclose:    %w(area base basefont br col command embed frame
                        hr img input isindex keygen link menuitem meta
                        param source track wbr),
-      filename:     "",
+      filename:     '',
       disable_capture: false,
       remove_whitespace: false,
     )
@@ -53,7 +53,7 @@ module Haml
     end
 
     def precompiled_with_ambles(_local_names, after_preamble:)
-      "#{after_preamble.tr("\n", ';')}#{@precompiled}".dup
+      "#{after_preamble.tr("\n", ';')}#{@precompiled}"
     end
   end
 end

data/lib/haml/filters/plain.rb

@@ -14,7 +14,12 @@ module Haml
 
       def compile_plain(text)
         string_literal = ::Haml::Util.unescape_interpolation(text)
-        StringSplitter.compile(string_literal).map do |temple|
+        compiled = StringSplitter.try_compile(string_literal)
+        # Ruby that does not parse: let the generated code report it against the template,
+        # rather than failing the whole compilation.
+        return [[:escape, false, [:dynamic, string_literal]]] if compiled.nil?
+
+        compiled.map do |temple|
           type, str = temple
           case type
           when :dynamic

data/lib/haml/helpers.rb

@@ -3,7 +3,7 @@ module Haml
   module Helpers
     def self.preserve(input)
       s = input.to_s.chomp("\n")
-      s.gsub!(/\n/, '
')
+      s.gsub!("\n", '
')
       s.delete!("\r")
       s
     end

data/lib/haml/parser.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'ripper'
+require 'prism'
 require 'strscan'
 require 'haml/error'
 require 'haml/util'
@@ -182,7 +182,7 @@ module Haml
 
       tabs = line.whitespace.length / @indentation.length
       return tabs if line.whitespace == @indentation * tabs
-      return @template_tabs + 1 if flat? && line.whitespace =~ /^#{@flat_spaces}/
+      return @template_tabs + 1 if flat? && /^#{@flat_spaces}/.match?(line.whitespace)
 
       message = Error.message(:inconsistent_indentation,
         human_indentation(line.whitespace),
@@ -227,7 +227,7 @@ module Haml
       end
 
       def inspect
-        %Q[(#{type} #{value.inspect}#{children.each_with_object(''.dup) {|c, s| s << "\n#{c.inspect.gsub!(/^/, '  ')}"}})].dup
+        %Q[(#{type} #{value.inspect}#{children.each_with_object(+'') {|c, s| s << "\n#{c.inspect.gsub!(/^/, '  ')}"}})]
       end
     end
 
@@ -236,7 +236,7 @@ module Haml
     DynamicAttributes = Struct.new(:new, :old) do
       undef :old=
       def old=(value)
-        unless value =~ /\A{.*}\z/m
+        unless /\A{.*}\z/m.match?(value)
           raise ArgumentError.new('Old attributes must start with "{" and end with "}"')
         end
         self[:old] = value
@@ -247,6 +247,12 @@ module Haml
         [new, stripped_old].compact.join(', ')
       end
 
+      # The lines a tag spans beyond its first because of these attributes, which is what
+      # the compiled tag has to span too for the line numbers after it to hold.
+      def newline_count
+        (new ? new.count("\n") : 0) + (old ? old.count("\n") : 0)
+      end
+
       private
 
       # For `%foo{ { foo: 1 }, bar: 2 }`, :old is "{ { foo: 1 }, bar: 2 }" and this method returns " { foo: 1 }, bar: 2 " for last argument.
@@ -397,7 +403,7 @@ module Haml
     def haml_comment(text)
       if filter_opened?
         @flat = true
-        @filter_buffer = String.new
+        @filter_buffer = +''
         @filter_buffer << "#{text}\n" unless text.empty?
         text = @filter_buffer
         # If we don't know the indentation by now, it'll be set in Line#tabs
@@ -444,7 +450,8 @@ module Haml
         end
       else
         if Util.contains_interpolation?(value)
-          value = Util.unescape_interpolation(value, escape_html)
+          value = Util.unescape_interpolation(value)
+          escape_interpolation = true if escape_html
           parse = true
           escape_html = false
         end
@@ -528,11 +535,11 @@ module Haml
     end
 
     def filter(name)
-      raise Error.new(Error.message(:invalid_filter_name, name)) unless name =~ /^\w+$/
+      raise Error.new(Error.message(:invalid_filter_name, name)) unless /^\w+$/.match?(name)
 
       if filter_opened?
         @flat = true
-        @filter_buffer = String.new
+        @filter_buffer = +''
         # If we don't know the indentation by now, it'll be set in Line#tabs
         @flat_spaces = @indentation * (@template_tabs+1) if @indentation
       end
@@ -601,8 +608,7 @@ module Haml
       attributes
     end
 
-    # This method doesn't use Haml::HamlAttributeParser because currently it depends on Ripper and Rubinius doesn't provide it.
-    # Ideally this logic should be placed in Haml::HamlAttributeParser instead of here and this method should use it.
+    # Ideally this logic should be placed in Haml::AttributeParser instead of here and this method should use it.
     #
     # @param  [String] text - Hash literal or text inside old attributes
     # @return [Hash,nil] - Return nil if text is not static Hash literal
@@ -630,7 +636,7 @@ module Haml
 
       tag_name, attributes, rest = match
 
-      if !attributes.empty? && (attributes =~ /[.#](\.|#|\z)/)
+      if !attributes.empty? && /[.#](\.|#|\z)/.match?(attributes)
         raise SyntaxError.new(Error.message(:illegal_element))
       end
 
@@ -686,10 +692,19 @@ module Haml
         # Old attributes often look like a valid Hash literal, but it sometimes allow code like
         # `{ hash, foo: bar }`, which is compiled to `_hamlout.attributes({}, nil, hash, foo: bar)`.
         #
-        # To scan such code correctly, this scans `a( hash, foo: bar }` instead, stops when there is
-        # 1 more :on_embexpr_end (the last '}') than :on_embexpr_beg, and resurrects '{' afterwards.
-        balanced, rest = balance_tokens(text.sub(?{, METHOD_CALL_PREFIX), :on_embexpr_beg, :on_embexpr_end, count: 1)
-        attributes_hash = balanced.sub(METHOD_CALL_PREFIX, ?{)
+        # To scan such code correctly, this scans `a( hash, foo: bar }` instead, stops on the '}'
+        # that closes one more brace than was opened, and resurrects '{' afterwards.
+        #
+        # Old attributes allow invalid Hash constructs (e.g., `{ hash, foo: bar }`).
+        # Replacing the opening `{` with `a(` makes the syntax look like a method call so
+        # it can be lexed reliably.
+
+        attributes_hash, rest = balance_tokens(
+          text.sub(?{, METHOD_CALL_PREFIX),
+          [:BRACE_LEFT, :LAMBDA_BEGIN, :EMBEXPR_BEGIN],
+          [:BRACE_RIGHT, :EMBEXPR_END],
+          count: 1)
+        attributes_hash = attributes_hash.sub(METHOD_CALL_PREFIX, ?{)
       rescue SyntaxError => e
         if e.message == Error.message(:unbalanced_brackets) && !@template.empty?
           text << "\n#{@next_line.text}"
@@ -734,7 +749,7 @@ module Haml
       end
 
       static_attributes = {}
-      dynamic_attributes = "{".dup
+      dynamic_attributes = +"{"
       attributes.each do |name, (type, val)|
         if type == :static
           static_attributes[name] = val
@@ -774,7 +789,7 @@ module Haml
 
       return name, [:static, content.first[1]] if content.size == 1
       return name, [:dynamic,
-        %!"#{content.each_with_object(''.dup) {|(t, v), s| s << (t == :str ? Util.inspect_obj(v)[1...-1] : "\#{#{v}}")}}"!]
+        %!"#{content.each_with_object(+'') {|(t, v), s| s << (t == :str ? Util.inspect_obj(v)[1...-1] : "\#{#{v}}")}}"!]
     end
 
     def next_line
@@ -786,7 +801,7 @@ module Haml
       line_defined = instance_variable_defined?(:@line)
       @line.tabs if line_defined
       unless (flat? && !closes_flat?(line) && !closes_flat?(@line)) ||
-          (line_defined && @line.text[0] == ?: && line.full =~ %r[^#{@line.full[/^\s+/]}\s])
+          (line_defined && @line.text[0] == ?: && %r[^#{@line.full[/^\s+/]}\s].match?(line.full))
         return next_line if line.text.empty?
 
         handle_multiline(line)
@@ -796,7 +811,7 @@ module Haml
     end
 
     def closes_flat?(line)
-      line && !line.text.empty? && line.full !~ /^#{@flat_spaces}/
+      line && !line.text.empty? && !(/^#{@flat_spaces}/.match?(line.full))
     end
 
     def handle_multiline(line)
@@ -814,7 +829,7 @@ module Haml
 
     # Checks whether or not `line` is in a multiline sequence.
     def is_multiline?(text)
-      text && text.length > 1 && text[-1] == MULTILINE_CHAR_VALUE && text[-2] == ?\s && text !~ BLOCK_WITH_SPACES
+      text && text.length > 1 && text[-1] == MULTILINE_CHAR_VALUE && text[-2] == ?\s && !BLOCK_WITH_SPACES.match?(text)
     end
 
     def handle_ruby_multiline(line)
@@ -838,27 +853,30 @@ module Haml
     # - and not "?\," which is a character literal
     def is_ruby_multiline?(text)
       text && text.length > 1 && text[-1] == ?, &&
-        !((text[-3, 2] =~ /\W\?/) || text[-3, 2] == "?\\")
+        !(/\W\?/.match?(text[-3, 2]) || text[-3, 2] == "?\\")
     end
 
     def balance(*args)
       Haml::Util.balance(*args) or raise(SyntaxError.new(Error.message(:unbalanced_brackets)))
     end
 
-    # Unlike #balance, this balances Ripper tokens to balance something like `{ a: "}" }` correctly.
+    # Unlike #balance, this balances lexed tokens to balance something like `{ a: "}" }` correctly.
     def balance_tokens(buf, start, finish, count: 0)
-      text = ''.dup
-      Ripper.lex(buf).each do |_, token, str|
-        text << str
-        case token
-        when start
+      Prism.lex(buf).value.each do |token, _|
+        type = token.type
+        next if type == :EOF
+
+        if start.include?(type)
           count += 1
-        when finish
+        elsif finish.include?(type)
           count -= 1
         end
 
         if count == 0
-          return text, buf.sub(text, '')
+          # Splitting on the offset instead of the tokens seen so far: whitespace and comments
+          # are not tokens, so the text cannot be rebuilt by concatenating them.
+          offset = token.location.end_offset
+          return buf.byteslice(0, offset), buf.byteslice(offset..)
         end
       end
       raise SyntaxError.new(Error.message(:unbalanced_brackets))
@@ -871,7 +889,7 @@ module Haml
     # Same semantics as block_opened?, except that block_opened? uses Line#tabs,
     # which doesn't interact well with filter lines
     def filter_opened?
-      @next_line.full =~ (@indentation ? /^#{@indentation * (@template_tabs + 1)}/ : /^\s/)
+      (@indentation ? /^#{@indentation * (@template_tabs + 1)}/ : /^\s/).match?(@next_line.full)
     end
 
     def flat?

data/lib/haml/preserver.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+module Haml
+  # find_and_preserve, shared by compiled templates and RailsHelpers.
+  #
+  # @api private
+  module Preserver
+    DEFAULT_TAGS = %w[textarea pre code].freeze
+
+    def self.build_regex(tags)
+      # An empty entry would add an alternative matching `<>`.
+      pattern = tags.reject(&:empty?).map { |tag| Regexp.escape(tag) }.join('|')
+      /<(#{pattern})([^>]*)>(.*?)(<\/\1>)/im
+    end
+    private_class_method :build_regex
+
+    DEFAULT_REGEX = build_regex(DEFAULT_TAGS)
+    private_constant :DEFAULT_REGEX
+
+    # Past this many lists the merge below turns quadratic, so the cache starts over.
+    MAX_REGEXES = 64
+    private_constant :MAX_REGEXES
+
+    INITIAL_REGEXES = { DEFAULT_TAGS => DEFAULT_REGEX }.freeze
+    private_constant :INITIAL_REGEXES
+
+    @regexes = INITIAL_REGEXES
+
+    # The cache is replaced, never mutated, so a racing writer at worst rebuilds a regex.
+    def self.regex(tags)
+      return DEFAULT_REGEX if tags.equal?(DEFAULT_TAGS)
+
+      cache = @regexes
+      cache[tags] || begin
+        regex = build_regex(tags)
+        # Copied so a caller mutating its own list, or a String in it, cannot strand the entry.
+        key = tags.map { |tag| tag.dup.freeze }.freeze
+        cache = INITIAL_REGEXES if cache.size >= MAX_REGEXES
+        @regexes = cache.merge(key => regex).freeze
+        regex
+      end
+    end
+
+    def self.find_and_preserve(input, tags)
+      re = regex(tags)
+      input.to_s.gsub(re) do |s|
+        s =~ re # Can't rely on $1, etc. existing since Rails' SafeBuffer#gsub is incompatible
+        name, attributes, content = $1, $2, $3
+        "<#{name}#{attributes}>#{yield(content)}</#{name}>"
+      end
+    end
+  end
+end

data/lib/haml/rails_helpers.rb

@@ -1,5 +1,6 @@
-# frozen_string_literal: false
+# frozen_string_literal: true
 require 'haml/helpers'
+require 'haml/preserver'
 
 # There are only helpers that depend on ActionView internals.
 module Haml
@@ -7,21 +8,12 @@ module Haml
     include Helpers
     extend self
 
-    DEFAULT_PRESERVE_TAGS = %w[textarea pre code].freeze
+    DEFAULT_PRESERVE_TAGS = Preserver::DEFAULT_TAGS
 
     def find_and_preserve(input = nil, tags = DEFAULT_PRESERVE_TAGS, &block)
       return find_and_preserve(capture_haml(&block), input || tags) if block
 
-      tags = tags.each_with_object('') do |t, s|
-        s << '|' unless s.empty?
-        s << Regexp.escape(t)
-      end
-
-      re = /<(#{tags})([^>]*)>(.*?)(<\/\1>)/im
-      input.to_s.gsub(re) do |s|
-        s =~ re # Can't rely on $1, etc. existing since Rails' SafeBuffer#gsub is incompatible
-        "<#{$1}#{$2}>#{preserve($3)}</#{$1}>"
-      end
+      Preserver.find_and_preserve(input, tags) { |content| preserve(content) }
     end
 
     def preserve(input = nil, &block)

data/lib/haml/ruby_expression.rb

@@ -1,32 +1,39 @@
 # frozen_string_literal: true
-require 'ripper'
+require 'prism'
 
 module Haml
-  class RubyExpression < Ripper
-    class ParseError < StandardError; end
+  class RubyExpression
+    # A character literal (`?a`) is a StringNode for Prism, but it has no quotes
+    # to split on, so it must not be reported as a string literal.
+    CHAR_LITERAL_OPENING = '?'
+
+    # A template is compiled into a method body, so a jump like `yield` is valid here even
+    # though it would not be at the top level of a script.
+    PARSE_OPTIONS = { partial_script: true }.freeze
 
     def self.syntax_error?(code)
-      self.new(code).parse
-      false
-    rescue ParseError
-      true
+      Prism.parse_failure?(code, **PARSE_OPTIONS)
     end
 
     def self.string_literal?(code)
-      return false if syntax_error?(code)
-
-      type, instructions = Ripper.sexp(code)
-      return false if type != :program
-      return false if instructions.size > 1
-
-      type, _ = instructions.first
-      type == :string_literal
+      !string_literal_node(code).nil?
     end
 
-    private
+    # @return [Prism::Node, nil] - the node of a string literal StringSplitter can split, if any.
+    #   Its locations are byte offsets into `code` as given, so `code` must not be stripped here.
+    def self.string_literal_node(code)
+      result = Prism.parse(code, **PARSE_OPTIONS)
+      return if result.failure?
+
+      statements = result.value.statements.body
+      return if statements.size > 1
 
-    def on_parse_error(*)
-      raise ParseError
+      case (node = statements.first)
+      when Prism::StringNode, Prism::InterpolatedStringNode
+        # Adjacent concatenation (`"a" "b"`) is one node without an opening
+        # delimiter of its own, and each of its parts keeps its own quotes.
+        node if !node.opening.nil? && node.opening != CHAR_LITERAL_OPENING
+      end
     end
   end
 end

data/lib/haml/string_splitter.rb

@@ -1,140 +1,88 @@
 # frozen_string_literal: true
-begin
-  require 'ripper'
-rescue LoadError
-end
+require 'prism'
+require 'haml/ruby_expression'
 
 module Haml
   # Compile [:dynamic, "foo#{bar}"] to [:multi, [:static, 'foo'], [:dynamic, 'bar']]
   class StringSplitter < Temple::Filter
-    if defined?(Ripper) && RUBY_VERSION >= "2.0.0" && Ripper.respond_to?(:lex)
-      class << self
-        # `code` param must be valid string literal
-        def compile(code)
-          [].tap do |exps|
-            tokens = Ripper.lex(code.strip)
-            tokens.pop while tokens.last && [:on_comment, :on_sp].include?(tokens.last[1])
-
-            if tokens.size < 2
-              raise(Haml::InternalError, "Expected token size >= 2 but got: #{tokens.size}")
-            end
-            compile_tokens!(exps, tokens)
-          end
-        end
-
-        private
-
-        def strip_quotes!(tokens)
-          _, type, beg_str = tokens.shift
-          if type != :on_tstring_beg
-            raise(Haml::InternalError, "Expected :on_tstring_beg but got: #{type}")
-          end
-
-          _, type, end_str = tokens.pop
-          if type != :on_tstring_end
-            raise(Haml::InternalError, "Expected :on_tstring_end but got: #{type}")
-          end
-
-          [beg_str, end_str]
+    class << self
+      # `code` param must be a string literal, as RubyExpression.string_literal? defines it.
+      def compile(code, node: RubyExpression.string_literal_node(code))
+        case node
+        when Prism::StringNode
+          node.unescaped.empty? ? [] : [[:static, in_source_encoding(node.unescaped, code)]]
+        when Prism::InterpolatedStringNode
+          compile_parts(node.parts, code)
+        else
+          raise(Haml::InternalError, "Expected a string literal but got: #{code}")
         end
+      end
 
-        def compile_tokens!(exps, tokens)
-          beg_str, end_str = strip_quotes!(tokens)
+      # Like compile, but nil instead of raising, for callers that built `code` themselves
+      # and would rather emit it untouched than fail the whole compilation.
+      def try_compile(code)
+        node = RubyExpression.string_literal_node(code)
+        compile(code, node: node) unless node.nil?
+      end
 
-          until tokens.empty?
-            _, type, str = tokens.shift
+      private
 
-            case type
-            when :on_tstring_content
-              beg_str, end_str = escape_quotes(beg_str, end_str)
-              exps << [:static, eval("#{beg_str}#{str}#{end_str}").to_s]
-            when :on_embexpr_beg
-              embedded = shift_balanced_embexpr(tokens)
+      def compile_parts(parts, code)
+        [].tap do |exps|
+          parts.each do |part|
+            case part
+            when Prism::StringNode
+              content = part.unescaped
+              exps << [:static, in_source_encoding(content, code)] unless content.empty?
+            when Prism::EmbeddedStatementsNode
+              embedded = embedded_source(part, code)
               exps << [:dynamic, embedded] unless embedded.empty?
-            end
-          end
-        end
-
-        # Some quotes are split-unsafe. Replace such quotes with null characters.
-        def escape_quotes(beg_str, end_str)
-          case [beg_str[-1], end_str]
-          when ['(', ')'], ['[', ']'], ['{', '}']
-            [beg_str.sub(/.\z/) { "\0" }, "\0"]
-          else
-            [beg_str, end_str]
-          end
-        end
-
-        def shift_balanced_embexpr(tokens)
-          String.new.tap do |embedded|
-            embexpr_open = 1
-
-            until tokens.empty?
-              _, type, str = tokens.shift
-              case type
-              when :on_embexpr_beg
-                embexpr_open += 1
-              when :on_embexpr_end
-                embexpr_open -= 1
-                break if embexpr_open == 0
-              end
-
-              embedded << str
+            when Prism::EmbeddedVariableNode
+              exps << [:dynamic, in_source_encoding(part.variable.slice, code)]
+            else
+              # Prism allows more part types than a string literal can currently hold. Dropping
+              # one would silently lose content, so fail instead of rendering something wrong.
+              raise(Haml::InternalError, "Unexpected #{part.class} in string literal: #{code}")
             end
           end
         end
       end
 
-      def on_dynamic(code)
-        return [:dynamic, code] unless string_literal?(code)
-        return [:dynamic, code] if code.include?("\n")
-
-        temple = [:multi]
-        StringSplitter.compile(code).each do |type, content|
-          case type
-          when :static
-            temple << [:static, content]
-          when :dynamic
-            temple << on_dynamic(content)
-          end
-        end
-        temple
+      # The source between `#{` and `}`, kept verbatim. Slicing `code` rather than
+      # using the statements' own source preserves the whitespace around them.
+      def embedded_source(part, code)
+        from = part.opening_loc.end_offset
+        code.byteslice(from, part.closing_loc.start_offset - from)
       end
 
-      private
-
-      def string_literal?(code)
-        return false if SyntaxChecker.syntax_error?(code)
-
-        type, instructions = Ripper.sexp(code)
-        return false if type != :program
-        return false if instructions.size > 1
+      # Prism tags what it returns with the encoding it parsed `code` under (UTF-8 for
+      # a BINARY source), while the fragments Haml slices out of the source itself keep
+      # the source encoding. The compiled template mixes both kinds, so anything taken
+      # from Prism has to come back to the source encoding, or joining the fragments
+      # raises Encoding::CompatibilityError once both sides hold non-ASCII bytes.
+      def in_source_encoding(string, code)
+        return string if string.encoding == code.encoding
 
-        type, _ = instructions.first
-        type == :string_literal
+        string.dup.force_encoding(code.encoding)
       end
+    end
 
-      class SyntaxChecker < Ripper
-        class ParseError < StandardError; end
-
-        def self.syntax_error?(code)
-          self.new(code).parse
-          false
-        rescue ParseError
-          true
-        end
+    def on_dynamic(code)
+      return [:dynamic, code] if code.include?("\n")
 
-        private
+      node = RubyExpression.string_literal_node(code)
+      return [:dynamic, code] if node.nil?
 
-        def on_parse_error(*)
-          raise ParseError
+      temple = [:multi]
+      StringSplitter.compile(code, node: node).each do |type, content|
+        case type
+        when :static
+          temple << [:static, content]
+        when :dynamic
+          temple << on_dynamic(content)
         end
       end
-    else
-      # Do nothing if ripper is unavailable
-      def call(ast)
-        ast
-      end
+      temple
     end
   end
 end

data/lib/haml/template.rb

@@ -16,5 +16,5 @@ module Haml
       "extend Haml::Helpers; #{super}"
     end
   end
-  Template.send(:extend, TemplateExtension)
+  Template.extend TemplateExtension
 end

data/lib/haml/temple_line_counter.rb

@@ -19,13 +19,27 @@ module Haml
         arg.count("\n") + cases.map do |cond, e|
           (cond == :else ? 0 : cond.count("\n")) + count_lines(e)
         end.reduce(:+)
-      when :escape
+      when :escape, :fescape
         count_lines(args[1])
+      when :html
+        count_html_lines(args)
       when :newline
         1
       else
         raise UnexpectedExpression.new("[HAML BUG] Unexpected Temple expression '#{type}' is given!")
       end
     end
+
+    def self.count_html_lines(args)
+      case args[0]
+      when :attrs
+        args.drop(1).sum { |a| count_lines(a) }
+      when :attr
+        count_lines(args[2])
+      else
+        raise UnexpectedExpression.new("[HAML BUG] Unexpected Temple expression 'html #{args[0]}' is given!")
+      end
+    end
+    private_class_method :count_html_lines
   end
 end