haml 7.3.0 → 7.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 381430279e5ca21c3d03f899c686f972609e54633c8e530609511e852597b7ef
-  data.tar.gz: a108268dccda6dc0e3ca014e484a7bcc0f50c973771a0057a086d75de2847495
+  metadata.gz: 6436b78efc92438b9bd1a948dd0672bc8d3798c654fa2846d310386ef0552084
+  data.tar.gz: b8c2657b36c209e49c0e58c95470c45cc8213e30b1eb17458e0550db37bd616e
 SHA512:
-  metadata.gz: b5f8d7bcc23e52be3fd4ed443ec3fdf171773f9c9aaacd1705d92ec09a3ef372adf13a8715ca4b4d039aa2a07fc2fdbb7e59eef30a22f55be2823bb0c7e23f0d
-  data.tar.gz: 954f3aa91fa3f49499d05dd1503056dd65a439519c5b2bcc300a6cf3dfbacf686717c8aa1e10ddb1ca7fa779504d92561a33e8a7d729b8013967cde102636fca
+  metadata.gz: 0530e52beb2fd093dfeb7d55c107b92a84f91e854bd38ed2e919349fcb820043cd3dd6e328ba4b2b24aa1b64ee8fcf2faf4a36dae1da60cb1b0a2b6abef7f531
+  data.tar.gz: b5a94fe5203318f1eaefa08b86e418090b8d0ef48cb94afa219512d87b116f568833cdf42e8f24a0a5e540d47b4b98dfb8a04f9e372bfe99d912d01dc41c2c24

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Haml Changelog
 
+## 7.5.0
+
+* Compile a multi-line attribute hash statically https://github.com/haml/haml/pull/1222
+  * Errors from multi-line attributes will be reported on the tag's line instead of the expression that raised the error.
+
+## 7.4.1
+
+* Build the preserve regex once per tag list instead of per call https://github.com/haml/haml/pull/1220
+
+## 7.4.0
+
+* `Haml::BOOLEAN_ATTRIBUTES` is changed from Array to Set https://github.com/haml/haml/pull/1216
+* A guard on `respond_to?(:html_safe?)` is removed from `Haml::Util.escape_html_safe` https://github.com/haml/haml/pull/1217
+* Keep interpolated text in the source encoding, fixing the `Encoding::CompatibilityError` remaining on an ASCII-8BIT
+  template source that mixes plain text and interpolation with non-ASCII characters https://github.com/haml/haml/issues/1218
+
+## 7.3.1
+
+* Keep Prism-derived fragments in the source encoding, fixing `Encoding::CompatibilityError` on an ASCII-8BIT
+  template source with non-ASCII characters (regression in 7.3.0) https://github.com/haml/haml/issues/1218
+
 ## 7.3.0
 
 * Replace Ripper with Prism https://github.com/haml/haml/pull/1214

data/REFERENCE.md

@@ -320,7 +320,7 @@ or using `true` and `false`:
     %input(selected=true)
 
 This feature works only for attributes that are included in
-[`Haml::BOOLEAN_ATTRIBUTES`](https://github.com/haml/haml/blob/main/lib/haml/attribute_compiler.rb#L8),
+[`Haml::BOOLEAN_ATTRIBUTES`](https://github.com/haml/haml/blob/main/lib/haml/attribute_compiler.rb#L9),
 as well as `data-` and `aria-` attributes.
 
     %input{'data-hidden' => false}

data/lib/haml/attribute_builder.rb

@@ -19,10 +19,12 @@ module Haml::AttributeBuilder
           buf << build_data(escape_attrs, quote, format, *hash[key])
         when 'aria'
           buf << build_aria(escape_attrs, quote, format, *hash[key])
-        when *Haml::BOOLEAN_ATTRIBUTES, /\Adata-/, /\Aaria-/
-          build_boolean!(escape_attrs, quote, format, buf, key, hash[key])
         else
-          buf << " #{key}=#{quote}#{escape_html(escape_attrs, hash[key].to_s)}#{quote}"
+          if boolean_attribute?(key)
+            build_boolean!(escape_attrs, quote, format, buf, key, hash[key])
+          else
+            buf << " #{key}=#{quote}#{escape_html(escape_attrs, hash[key].to_s)}#{quote}"
+          end
         end
       end
       buf.join
@@ -72,6 +74,10 @@ module Haml::AttributeBuilder
 
     private
 
+    def boolean_attribute?(key)
+      Haml::BOOLEAN_ATTRIBUTES.include?(key) || key.start_with?('data-', 'aria-')
+    end
+
     def build_data_attribute(key, escape_attrs, quote, format, *hashes)
       attrs = []
       if hashes.size > 1 && hashes.all? { |h| h.is_a?(Hash) }
@@ -98,14 +104,18 @@ module Haml::AttributeBuilder
       flattened = {}
 
       attributes.each do |key, value|
+        # Recursing into a hash which contains itself would never end.
+        next if value.equal?(attributes)
+
         case value
-        when attributes
         when Hash
           flatten_attributes(value).each do |k, v|
             if k.nil?
               flattened[key] = v
             else
-              flattened["#{key}-#{k.to_s.tr('_', '-')}"] = v
+              k = k.is_a?(Symbol) ? k.name : k.to_s
+              k = k.tr('_', '-') if k.include?('_')
+              flattened["#{key}-#{k}"] = v
             end
           end
         else
@@ -122,7 +132,8 @@ module Haml::AttributeBuilder
           raise ArgumentError, "Non-hash object is given to attributes!"
         end
         hash.each do |key, value|
-          key = key.to_s
+          # Symbol#name hands back the interned String; to_s allocates one per render.
+          key = key.is_a?(Symbol) ? key.name : key.to_s
           case key
           when 'id', 'class', 'data', 'aria'
             merged[key] ||= []

data/lib/haml/attribute_compiler.rb

@@ -1,16 +1,18 @@
 # frozen_string_literal: true
+require 'set'
 require 'haml/attribute_builder'
 require 'haml/attribute_parser'
 require 'haml/ruby_expression'
+require 'haml/temple_line_counter'
 
 module Haml
-  # The list of boolean attributes. You may add custom attributes to this constant.
-  BOOLEAN_ATTRIBUTES = %w[disabled readonly multiple checked autobuffer
+  # The set of boolean attributes. You may add custom attributes to this constant.
+  BOOLEAN_ATTRIBUTES = Set.new(%w[disabled readonly multiple checked autobuffer
                        autoplay controls loop selected hidden scoped async
                        defer reversed ismap seamless muted required
                        autofocus novalidate formnovalidate open pubdate
                        itemscope allowfullscreen default inert sortable
-                       truespeed typemustmatch download]
+                       truespeed typemustmatch download])
 
   class AttributeCompiler
     def initialize(identity, options)
@@ -23,12 +25,15 @@ module Haml
     def compile(node)
       hashes = []
       return runtime_compile(node) if node.value[:object_ref] != :nil
-      [node.value[:dynamic_attributes].new, node.value[:dynamic_attributes].old].compact.each do |attribute_str|
+      dynamic_attributes = node.value[:dynamic_attributes]
+      [dynamic_attributes.new, dynamic_attributes.old].compact.each do |attribute_str|
         hash = AttributeParser.parse(attribute_str)
         return runtime_compile(node) if hash.nil? || hash.any? { |_key, value| value.empty? }
         hashes << hash
       end
-      static_compile(node.value[:attributes], hashes)
+      temple = static_compile(node.value[:attributes], hashes)
+      restore_newlines!(temple, dynamic_attributes)
+      temple
     end
 
     private
@@ -55,15 +60,24 @@ module Haml
           compile_class!(temple, key, values)
         when 'data', 'aria'
           compile_data!(temple, key, values)
-        when *BOOLEAN_ATTRIBUTES, /\Adata-/, /\Aaria-/
-          compile_boolean!(temple, key, values)
         else
-          compile_common!(temple, key, values)
+          if BOOLEAN_ATTRIBUTES.include?(key) || key.start_with?('data-', 'aria-')
+            compile_boolean!(temple, key, values)
+          else
+            compile_common!(temple, key, values)
+          end
         end
       end
       temple
     end
 
+    # ChildrenCompiler counts on the tag spanning as many lines as its attribute source does.
+    def restore_newlines!(temple, dynamic_attributes)
+      missing = dynamic_attributes.newline_count
+      missing -= TempleLineCounter.count_lines(temple) if missing > 0
+      missing.times { temple << [:newline] }
+    end
+
     def compile_id!(temple, key, values)
       build_code = attribute_builder(:id, values)
       if values.all? { |type, exp| type == :static || Temple::StaticAnalyzer.static?(exp) }

data/lib/haml/attribute_parser.rb

@@ -15,15 +15,11 @@ module Haml
     end
 
     # @return [Hash,nil] - keys and values are the attribute source as written, or nil if
-    #   the text is not a Hash literal whose keys are all static.
+    #   the text is not a Hash literal whose keys are all static, or if it holds a heredoc.
     def parse(text)
       exp = wrap_bracket(text)
-      # A multi-line hash is left to the runtime, which keeps the [:newline] bookkeeping of
-      # the compiled code correct. Compiling it statically is a separate optimization.
-      return if exp.include?("\n")
-
       node = hash_node(exp)
-      return if node.nil?
+      return if node.nil? || contains_heredoc?(exp, node)
 
       hash = {}
       node.elements.each do |element|
@@ -32,7 +28,7 @@ module Haml
         key = static_key(element.key)
         return if key.nil?
 
-        hash[key] = value_source(element.value)
+        hash[in_source_encoding(key, exp)] = in_source_encoding(value_source(element.value), exp)
       end
       hash
     end
@@ -58,6 +54,12 @@ module Haml
       node if node.is_a?(Prism::HashNode)
     end
 
+    # A heredoc's body lies outside its node, so the value's slice would not be the value.
+    # Its opener always spells `<<`, which spares the tree walk for nearly every hash.
+    def contains_heredoc?(exp, node)
+      exp.include?('<<') && !node.breadth_first_search { |n| n.respond_to?(:heredoc?) && n.heredoc? }.nil?
+    end
+
     # The key as written between its delimiters, not unescaped: an escape has to reach the
     # attribute name as the source spelled it, like the `\0` of `{ "a\0b" => 1 }`.
     def static_key(key)
@@ -74,5 +76,15 @@ module Haml
 
       value.slice
     end
+
+    # Prism tags its slices with the encoding it parsed the source under (UTF-8 for a
+    # BINARY source), while the rest of the compiled template stays in the source
+    # encoding. Mixing the two raises Encoding::CompatibilityError once both sides hold
+    # non-ASCII bytes, so bring everything back to the source encoding.
+    def in_source_encoding(string, source)
+      return string if string.encoding == source.encoding
+
+      string.dup.force_encoding(source.encoding)
+    end
   end
 end

data/lib/haml/compiler/children_compiler.rb

@@ -40,9 +40,7 @@ module Haml
         when :script, :silent_script
           @lineno += 1
         when :tag
-          [node.value[:dynamic_attributes].new, node.value[:dynamic_attributes].old].compact.each do |attribute_hash|
-            @lineno += attribute_hash.count("\n")
-          end
+          @lineno += node.value[:dynamic_attributes].newline_count
           @lineno += 1 if node.children.empty? && node.value[:parse]
         end
 

data/lib/haml/compiler/script_compiler.rb

@@ -1,18 +1,15 @@
 # frozen_string_literal: true
 require 'temple/static_analyzer'
+require 'haml/helpers'
+require 'haml/preserver'
 require 'haml/ruby_expression'
 require 'haml/string_splitter'
 
 module Haml
   class Compiler
     class ScriptCompiler
-      def self.find_and_preserve(input, tags)
-        tags = tags.map { |tag| Regexp.escape(tag) }.join('|')
-        re = /<(#{tags})([^>]*)>(.*?)(<\/\1>)/im
-        input.to_s.gsub(re) do |s|
-          s =~ re # Can't rely on $1, etc. existing since Rails' SafeBuffer#gsub is incompatible
-          "<#{$1}#{$2}>#{Haml::Helpers.preserve($3)}</#{$1}>"
-        end
+      def self.find_and_preserve(input, tags = ::Haml::Preserver::DEFAULT_TAGS)
+        ::Haml::Preserver.find_and_preserve(input, tags) { |content| ::Haml::Helpers.preserve(content) }
       end
 
       def initialize(identity, options)
@@ -69,7 +66,7 @@ module Haml
         if node.value[:escape_html]
           str = Haml::Util.escape_html(str)
         elsif node.value[:preserve]
-          str = ScriptCompiler.find_and_preserve(str, %w(textarea pre code))
+          str = ScriptCompiler.find_and_preserve(str)
         end
         [:multi, [:static, str], [:newline]]
       end
@@ -104,7 +101,7 @@ module Haml
       end
 
       def find_and_preserve(code)
-        %Q[::Haml::Compiler::ScriptCompiler.find_and_preserve(#{code}, %w(textarea pre code))]
+        %Q[::Haml::Compiler::ScriptCompiler.find_and_preserve(#{code})]
       end
 
       def escape_html(temple)

data/lib/haml/helpers.rb

@@ -3,7 +3,7 @@ module Haml
   module Helpers
     def self.preserve(input)
       s = input.to_s.chomp("\n")
-      s.gsub!(/\n/, '
')
+      s.gsub!("\n", '
')
       s.delete!("\r")
       s
     end

data/lib/haml/parser.rb

@@ -247,6 +247,12 @@ module Haml
         [new, stripped_old].compact.join(', ')
       end
 
+      # The lines a tag spans beyond its first because of these attributes, which is what
+      # the compiled tag has to span too for the line numbers after it to hold.
+      def newline_count
+        (new ? new.count("\n") : 0) + (old ? old.count("\n") : 0)
+      end
+
       private
 
       # For `%foo{ { foo: 1 }, bar: 2 }`, :old is "{ { foo: 1 }, bar: 2 }" and this method returns " { foo: 1 }, bar: 2 " for last argument.
@@ -444,7 +450,8 @@ module Haml
         end
       else
         if Util.contains_interpolation?(value)
-          value = Util.unescape_interpolation(value, escape_html)
+          value = Util.unescape_interpolation(value)
+          escape_interpolation = true if escape_html
           parse = true
           escape_html = false
         end

data/lib/haml/preserver.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+module Haml
+  # find_and_preserve, shared by compiled templates and RailsHelpers.
+  #
+  # @api private
+  module Preserver
+    DEFAULT_TAGS = %w[textarea pre code].freeze
+
+    def self.build_regex(tags)
+      # An empty entry would add an alternative matching `<>`.
+      pattern = tags.reject(&:empty?).map { |tag| Regexp.escape(tag) }.join('|')
+      /<(#{pattern})([^>]*)>(.*?)(<\/\1>)/im
+    end
+    private_class_method :build_regex
+
+    DEFAULT_REGEX = build_regex(DEFAULT_TAGS)
+    private_constant :DEFAULT_REGEX
+
+    # Past this many lists the merge below turns quadratic, so the cache starts over.
+    MAX_REGEXES = 64
+    private_constant :MAX_REGEXES
+
+    INITIAL_REGEXES = { DEFAULT_TAGS => DEFAULT_REGEX }.freeze
+    private_constant :INITIAL_REGEXES
+
+    @regexes = INITIAL_REGEXES
+
+    # The cache is replaced, never mutated, so a racing writer at worst rebuilds a regex.
+    def self.regex(tags)
+      return DEFAULT_REGEX if tags.equal?(DEFAULT_TAGS)
+
+      cache = @regexes
+      cache[tags] || begin
+        regex = build_regex(tags)
+        # Copied so a caller mutating its own list, or a String in it, cannot strand the entry.
+        key = tags.map { |tag| tag.dup.freeze }.freeze
+        cache = INITIAL_REGEXES if cache.size >= MAX_REGEXES
+        @regexes = cache.merge(key => regex).freeze
+        regex
+      end
+    end
+
+    def self.find_and_preserve(input, tags)
+      re = regex(tags)
+      input.to_s.gsub(re) do |s|
+        s =~ re # Can't rely on $1, etc. existing since Rails' SafeBuffer#gsub is incompatible
+        name, attributes, content = $1, $2, $3
+        "<#{name}#{attributes}>#{yield(content)}</#{name}>"
+      end
+    end
+  end
+end

data/lib/haml/rails_helpers.rb

@@ -1,5 +1,6 @@
-# frozen_string_literal: false
+# frozen_string_literal: true
 require 'haml/helpers'
+require 'haml/preserver'
 
 # There are only helpers that depend on ActionView internals.
 module Haml
@@ -7,21 +8,12 @@ module Haml
     include Helpers
     extend self
 
-    DEFAULT_PRESERVE_TAGS = %w[textarea pre code].freeze
+    DEFAULT_PRESERVE_TAGS = Preserver::DEFAULT_TAGS
 
     def find_and_preserve(input = nil, tags = DEFAULT_PRESERVE_TAGS, &block)
       return find_and_preserve(capture_haml(&block), input || tags) if block
 
-      tags = tags.each_with_object('') do |t, s|
-        s << '|' unless s.empty?
-        s << Regexp.escape(t)
-      end
-
-      re = /<(#{tags})([^>]*)>(.*?)(<\/\1>)/im
-      input.to_s.gsub(re) do |s|
-        s =~ re # Can't rely on $1, etc. existing since Rails' SafeBuffer#gsub is incompatible
-        "<#{$1}#{$2}>#{preserve($3)}</#{$1}>"
-      end
+      Preserver.find_and_preserve(input, tags) { |content| preserve(content) }
     end
 
     def preserve(input = nil, &block)

data/lib/haml/string_splitter.rb

@@ -10,7 +10,7 @@ module Haml
       def compile(code, node: RubyExpression.string_literal_node(code))
         case node
         when Prism::StringNode
-          node.unescaped.empty? ? [] : [[:static, node.unescaped]]
+          node.unescaped.empty? ? [] : [[:static, in_source_encoding(node.unescaped, code)]]
         when Prism::InterpolatedStringNode
           compile_parts(node.parts, code)
         else
@@ -33,12 +33,12 @@ module Haml
             case part
             when Prism::StringNode
               content = part.unescaped
-              exps << [:static, content] unless content.empty?
+              exps << [:static, in_source_encoding(content, code)] unless content.empty?
             when Prism::EmbeddedStatementsNode
               embedded = embedded_source(part, code)
               exps << [:dynamic, embedded] unless embedded.empty?
             when Prism::EmbeddedVariableNode
-              exps << [:dynamic, part.variable.slice]
+              exps << [:dynamic, in_source_encoding(part.variable.slice, code)]
             else
               # Prism allows more part types than a string literal can currently hold. Dropping
               # one would silently lose content, so fail instead of rendering something wrong.
@@ -54,6 +54,17 @@ module Haml
         from = part.opening_loc.end_offset
         code.byteslice(from, part.closing_loc.start_offset - from)
       end
+
+      # Prism tags what it returns with the encoding it parsed `code` under (UTF-8 for
+      # a BINARY source), while the fragments Haml slices out of the source itself keep
+      # the source encoding. The compiled template mixes both kinds, so anything taken
+      # from Prism has to come back to the source encoding, or joining the fragments
+      # raises Encoding::CompatibilityError once both sides hold non-ASCII bytes.
+      def in_source_encoding(string, code)
+        return string if string.encoding == code.encoding
+
+        string.dup.force_encoding(code.encoding)
+      end
     end
 
     def on_dynamic(code)

data/lib/haml/template.rb

@@ -16,5 +16,5 @@ module Haml
       "extend Haml::Helpers; #{super}"
     end
   end
-  Template.send(:extend, TemplateExtension)
+  Template.extend TemplateExtension
 end

data/lib/haml/temple_line_counter.rb

@@ -19,13 +19,27 @@ module Haml
         arg.count("\n") + cases.map do |cond, e|
           (cond == :else ? 0 : cond.count("\n")) + count_lines(e)
         end.reduce(:+)
-      when :escape
+      when :escape, :fescape
         count_lines(args[1])
+      when :html
+        count_html_lines(args)
       when :newline
         1
       else
         raise UnexpectedExpression.new("[HAML BUG] Unexpected Temple expression '#{type}' is given!")
       end
     end
+
+    def self.count_html_lines(args)
+      case args[0]
+      when :attrs
+        args.drop(1).sum { |a| count_lines(a) }
+      when :attr
+        count_lines(args[2])
+      else
+        raise UnexpectedExpression.new("[HAML BUG] Unexpected Temple expression 'html #{args[0]}' is given!")
+      end
+    end
+    private_class_method :count_html_lines
   end
 end

data/lib/haml/util.rb

@@ -5,7 +5,6 @@ begin
 rescue LoadError
   require 'erb'
 end
-require 'set'
 require 'stringio'
 require 'strscan'
 
@@ -26,10 +25,9 @@ module Haml
       end
     end
 
-    # TODO: Remove unescape_interpolation's workaround and get rid of `respond_to?`.
     def self.escape_html_safe(html)
       html = html.to_s
-      (html.respond_to?(:html_safe?) && html.html_safe?) ? html : escape_html(html)
+      html.html_safe? ? html : escape_html(html)
     end
 
     # Silence all output to STDERR within a block.
@@ -202,7 +200,7 @@ MSG
       /#[\{$@]/ === str
     end
 
-    def unescape_interpolation(str, escape_html = nil)
+    def unescape_interpolation(str)
       res = ''.dup
       rest = Haml::Util.handle_interpolation str.dump do |scan|
         escapes = (scan[2].size - 1) / 2
@@ -218,12 +216,17 @@ MSG
           end
           content = eval("\"#{interpolated}\"")
           content = "#{char}#{content}" if char == '@' || char == '$'
-          content = "Haml::Util.escape_html_safe((#{content}).to_s)" if escape_html
 
           res << "\#{#{content}}"
         end
       end
-      res + rest
+      # The result is rebuilt from `str.dump` fragments and eval'd pieces, so it can
+      # come out tagged with an encoding that differs from `str`'s (e.g. UTF-8 for a
+      # BINARY source). The compiled template joins this string literal's fragments
+      # with fragments sliced out of the source itself, and mixed encodings raise
+      # Encoding::CompatibilityError, so bring it back to the source encoding.
+      # See haml/haml#1218.
+      (res + rest).force_encoding(str.encoding)
     end
 
     private

data/lib/haml/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Haml
-  VERSION = '7.3.0'
+  VERSION = '7.5.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: haml
 version: !ruby/object:Gem::Version
-  version: 7.3.0
+  version: 7.5.0
 platform: ruby
 authors:
 - Natalie Weizenbaum
@@ -307,6 +307,7 @@ files:
 - lib/haml/identity.rb
 - lib/haml/object_ref.rb
 - lib/haml/parser.rb
+- lib/haml/preserver.rb
 - lib/haml/rails_helpers.rb
 - lib/haml/rails_template.rb
 - lib/haml/railtie.rb