herb 0.10.4 → 0.11.0

data/lib/herb/engine/compiler.rb

@@ -1,15 +1,14 @@
 # frozen_string_literal: false
 
+require_relative "../html/util"
+
 module Herb
   class Engine
     class Compiler < ::Herb::Visitor
-      EXPRESSION_TOKEN_TYPES = [:expr, :expr_escaped, :expr_block, :expr_block_escaped].freeze
-
-      TRAILING_WHITESPACE = /[ \t]+\z/
-      TRAILING_INDENTATION = /\n[ \t]+\z/
-      TRAILING_INDENTATION_CAPTURE = /\n([ \t]+)\z/
-      WHITESPACE_ONLY = /\A[ \t]+\z/
-      WHITESPACE_ONLY_CAPTURE = /\A([ \t]+)\z/
+      PADDING_NEWLINES = Array.new(17) { |count| ("\n" * count).freeze }.freeze #: Array[String]
+      EXPRESSION_TOKEN_TYPES = [:expr, :expr_escaped, :expr_block, :expr_block_escaped].freeze #: Array[Symbol]
+      HORIZONTAL_SPACE = [" ", "\t"].freeze #: Array[String]
+      RAW_TEXT_ELEMENTS = ["script", "style"].freeze #: Array[String]
 
       attr_reader :tokens
 
@@ -17,20 +16,36 @@ module Herb
         super()
 
         @engine = engine
+        @source_lines = options[:source]&.lines
         @escape = options.fetch(:escape) { options.fetch(:escape_html, false) }
+        @trim = options[:trim] != false
+        @literal_prefix = options[:literal_prefix] || "<%"
+        @literal_postfix = options[:literal_postfix] || "%>"
         @tokens = [] #: Array[untyped]
+        @padding_before = nil #: Hash[Integer, Integer]?
         @element_stack = [] #: Array[String]
         @context_stack = [:html_content]
         @trim_next_whitespace = false
         @last_trim_consumed_newline = false
+        @pending_trim_newline_index = nil #: Integer?
+        @pending_trim_owns_next_whitespace = false
+        @pending_span_extra_index = nil #: Integer?
         @pending_leading_whitespace = nil
         @pending_leading_whitespace_insert_index = 0
         @current_element_source = nil
       end
 
-      def generate_output
-        optimized_tokens = optimize_tokens(@tokens)
+      def optimized_tokens
+        @optimized_tokens ||= optimize_tokens(@tokens)
+      end
+
+      def static_template_text
+        return unless optimized_tokens.all? { |token| token[0] == :text }
 
+        optimized_tokens.map { |token| token[1] }.join
+      end
+
+      def generate_output
         optimized_tokens.each do |type, value, context, escaped|
           case type
           when :text
@@ -49,26 +64,30 @@ module Herb
             @engine.send(:add_expression_block, indicator_for(type), value)
           when :expr_block_end
             @engine.send(:add_expression_block_end, value, escaped: escaped)
+          when :chain
+            @engine.send(:add_expression_result, value)
           end
         end
       end
 
       def visit_document_node(node)
         visit_all(node.children)
+
+        restore_pending_leading_whitespace! if @trim_next_whitespace
+
+        settle_pending_trim_newline!(false)
       end
 
       def visit_html_element_node(node)
-        with_element_context(node) do
+        with_element_context(node) do |tag_name|
           visit(node.open_tag)
           visit_all(node.body)
 
-          tag_name = node.tag_name&.value&.downcase
-
           if node.open_tag.is_a?(Herb::AST::ERBOpenTagNode) && tag_name && node.close_tag
             if node.close_tag.is_a?(Herb::AST::ERBEndNode)
-              remove_trailing_whitespace_from_last_token! if left_trim?(node.close_tag)
+              remove_trailing_whitespace_from_last_token! if @trim && left_trim?(node.close_tag)
               add_text("</#{tag_name}>")
-              @trim_next_whitespace = true
+              @trim_next_whitespace = true if @trim
             else
               add_text("</#{tag_name}>")
             end
@@ -96,7 +115,7 @@ module Herb
       end
 
       def visit_html_attribute_node(node)
-        add_whitespace(" ")
+        add_text(" ") unless preceded_by_whitespace?
 
         visit(node.name)
 
@@ -157,13 +176,6 @@ module Herb
       end
 
       def visit_html_close_tag_node(node)
-        tag_name = node.tag_name&.value&.downcase
-
-        if @engine.content_for_head && tag_name == "head"
-          escaped_html = @engine.content_for_head.gsub("'", "\\\\'")
-          @tokens << [:expr, "'#{escaped_html}'.html_safe", current_context]
-        end
-
         add_text(node.tag_opening&.value)
         add_text(node.tag_name&.value)
         add_text(node.tag_closing&.value)
@@ -182,7 +194,7 @@ module Herb
       end
 
       def visit_whitespace_node(node)
-        add_whitespace(node.value.value)
+        add_text(node.value.value)
       end
 
       def visit_html_comment_node(node)
@@ -203,6 +215,13 @@ module Herb
         add_text(node.tag_closing.value)
       end
 
+      def visit_xml_processing_instruction_node(node)
+        add_text(node.tag_opening.value)
+        add_text(node.target.value)
+        visit_all(node.children)
+        add_text(node.tag_closing.value)
+      end
+
       def visit_cdata_node(node)
         add_text(node.tag_opening.value)
         visit_all(node.children)
@@ -215,9 +234,26 @@ module Herb
         process_erb_tag(node)
       end
 
+      def visit_erb_comment_node(node)
+        process_erb_tag(node)
+      end
+
       def visit_erb_control_node(node, &)
         if node.content
-          apply_trim(node, node.content.value.strip)
+          if node.tag_opening && erb_escaped?(node.tag_opening.value)
+            add_escaped_erb_tag(node)
+          elsif continues_into_next_node?(node)
+            @continued_head = node
+          else
+            head = @continued_head
+            @continued_head = nil
+
+            code_index = @tokens.length
+            raw = head ? head.content.value + node.content.value : node.content.value
+
+            apply_trim(head || node, raw.strip)
+            keep_line_count(node, at: code_index, raw: raw)
+          end
         end
 
         yield if block_given?
@@ -246,7 +282,7 @@ module Herb
       end
 
       def visit_erb_case_node(node)
-        visit_erb_control_with_parts(node, :conditions, :else_clause, :end_node)
+        visit_erb_control_with_parts(node, *case_parts(node))
       end
 
       def visit_erb_when_node(node)
@@ -282,7 +318,7 @@ module Herb
       end
 
       def visit_erb_case_match_node(node)
-        visit_erb_control_with_parts(node, :conditions, :else_clause, :end_node)
+        visit_erb_control_with_parts(node, *case_parts(node))
       end
 
       def visit_erb_in_node(node)
@@ -296,11 +332,12 @@ module Herb
       def visit_erb_block_node(node)
         opening = node.tag_opening.value
 
-        check_for_escaped_erb_tag!(opening)
+        return add_escaped_erb_block(node) if erb_escaped?(opening)
 
         if opening.include?("=")
           should_escape = should_escape_output?(opening)
-          code = node.content.value.strip
+          code = ::Herb::Engine::Helpers.strip_trailing_comment(node.content.value.strip)
+          code_index = @tokens.length
 
           @tokens << if should_escape
                        [:expr_block_escaped, code, current_context]
@@ -308,6 +345,8 @@ module Herb
                        [:expr_block, code, current_context]
                      end
 
+          keep_line_count(node, at: code_index)
+
           @last_trim_consumed_newline = false
           @trim_next_whitespace = true if right_trim?(node)
 
@@ -324,12 +363,22 @@ module Herb
         end
       end
 
+      def visit_erb_iteration_block_node(node)
+        visit_erb_block_node(node)
+      end
+
+      def visit_erb_render_node(node)
+        return process_erb_tag(node) unless node.end_node
+
+        visit_erb_block_node(node)
+      end
+
       def visit_erb_block_end_node(node, escaped: false)
-        remove_trailing_whitespace_from_last_token! if left_trim?(node)
+        remove_trailing_whitespace_from_last_token! if @trim && left_trim?(node)
 
-        code = node.content.value.strip
+        code = ::Herb::Engine::Helpers.strip_trailing_comment(node.content.value.strip)
 
-        if at_line_start?
+        if @trim && at_line_start?
           leading_space = extract_and_remove_leading_space!
           right_space = " \n"
 
@@ -340,6 +389,12 @@ module Herb
         end
       end
 
+      def case_parts(node)
+        parts = [:conditions, :else_clause, :end_node]
+
+        erb_escaped?(node.tag_opening.value) ? [:children, *parts] : parts
+      end
+
       def visit_erb_control_with_parts(node, *parts)
         visit_erb_control_node(node) do
           parts.each do |part|
@@ -358,13 +413,25 @@ module Herb
 
       private
 
-      def check_for_escaped_erb_tag!(opening)
-        return unless opening.start_with?("<%%")
+      def add_escaped_erb_tag(node)
+        opening = node.tag_opening.value.sub("<%%") { @literal_prefix }
+        closing = (node.tag_closing&.value || "").sub(/%>\z/) { @literal_postfix }
+
+        add_text("#{opening}#{node.content.value}#{closing}")
+      end
 
-        raise Herb::Engine::GeneratorTemplateError,
-              "This file appears to be a generator template (a template used to generate ERB files) " \
-              "rather than a standard ERB template. It contains escaped ERB tags like <%%= %> which " \
-              "produce literal ERB output in the generated file."
+      def add_escaped_erb_block(node)
+        add_escaped_erb_tag(node)
+        visit_all(node.body)
+
+        end_node = node.end_node
+        return unless end_node
+
+        if erb_escaped?(end_node.tag_opening.value)
+          add_escaped_erb_tag(end_node)
+        else
+          visit(end_node)
+        end
       end
 
       def current_context
@@ -379,7 +446,7 @@ module Herb
         @context_stack.pop
       end
 
-      #: (untyped node) { () -> untyped } -> untyped
+      #: (untyped node) { (String?) -> untyped } -> untyped
       def with_element_context(node)
         tag_name = node.tag_name&.value&.downcase
         previous_element_source = @current_element_source
@@ -393,9 +460,9 @@ module Herb
           push_context(:style_content)
         end
 
-        yield
+        yield(tag_name)
 
-        pop_context if ["script", "style"].include?(tag_name)
+        pop_context if RAW_TEXT_ELEMENTS.include?(tag_name)
 
         @element_stack.pop if tag_name
         @current_element_source = previous_element_source
@@ -404,28 +471,67 @@ module Herb
       def process_erb_tag(node, skip_comment_check: false)
         opening = node.tag_opening.value
 
-        check_for_escaped_erb_tag!(opening)
+        return add_escaped_erb_tag(node) if erb_escaped?(opening)
+
+        if !skip_comment_check && erb_omitted?(opening)
+          unless @trim
+            keep_span_line_count(node)
+
+            return
+          end
 
-        if !skip_comment_check && erb_comment?(opening)
           follows_newline = leading_space_follows_newline?
           remove_trailing_whitespace_from_last_token! if left_trim?(node)
+          swallows_newline = at_line_start?
 
-          if at_line_start?
+          if swallows_newline
             leading_space = extract_and_remove_leading_space!
             @trim_next_whitespace = true
             save_pending_leading_whitespace!(leading_space) if !leading_space.empty? && follows_newline
           end
+
+          keep_span_line_count(node, extra: swallows_newline ? 1 : 0)
+          @pending_span_extra_index = @tokens.length if swallows_newline
+
           return
         end
-        return if erb_graphql?(opening)
 
-        code = node.content.value.strip
+        code = ::Herb::Engine::Helpers.strip_trailing_comment(node.content.value.strip)
+
+        code_index = @tokens.length
 
         if erb_output?(opening)
           process_erb_output(node, opening, code)
         else
           apply_trim(node, code)
         end
+
+        absorbed = erb_output?(opening) && ::Herb::Engine::Helpers.ends_on_heredoc_terminator?(code) ? 1 : 0
+
+        keep_line_count(node, at: code_index, absorbed: absorbed)
+      end
+
+      #: (untyped, ?extra: Integer) -> void
+      def keep_span_line_count(node, extra: 0)
+        lines = node.content.value.count("\n") + extra
+
+        return unless lines.positive?
+
+        @padding_before ||= Hash.new(0)
+        @padding_before[@tokens.length] += lines
+      end
+
+      def keep_line_count(node, extra: 0, at: nil, absorbed: 0, raw: nil)
+        raw ||= node.content.value
+
+        leading = raw[0, raw.length - raw.lstrip.length].to_s.count("\n")
+        trailing = raw.count("\n") - raw.strip.count("\n") - leading + extra - absorbed
+
+        block_comment = raw.include?("=begin") || raw.include?("=end")
+
+        @padding_before ||= Hash.new(0)
+        @padding_before[at] += leading if at && leading.positive? && !block_comment
+        @padding_before[@tokens.length] += trailing if trailing.positive?
       end
 
       def add_text(text)
@@ -436,9 +542,14 @@ module Herb
           text = text.sub(/\A[ \t]*\r?\n/, "")
           @trim_next_whitespace = false
 
+          settle_pending_trim_newline!(@last_trim_consumed_newline)
+          settle_pending_span_extra!(@last_trim_consumed_newline)
           restore_pending_leading_whitespace! unless @last_trim_consumed_newline
         else
           @last_trim_consumed_newline = false
+
+          settle_pending_trim_newline!(false)
+          settle_pending_span_extra!(false)
         end
 
         @pending_leading_whitespace = nil
@@ -448,10 +559,6 @@ module Herb
         @tokens << [:text, text, current_context]
       end
 
-      def add_whitespace(whitespace)
-        @tokens << [:whitespace, whitespace, current_context]
-      end
-
       def add_code(code)
         @tokens << [:code, code, current_context]
       end
@@ -468,82 +575,98 @@ module Herb
 
       def optimize_tokens(tokens)
         return tokens if tokens.empty?
-
-        compacted = compact_whitespace_tokens(tokens)
+        return optimize_tokens_with_padding(tokens) if @padding_before
 
         optimized = [] #: Array[untyped]
-        current_text = ""
+        current_text = nil #: String?
         current_context = nil
 
-        compacted.each do |type, value, context, escaped|
+        tokens.each do |token|
+          type = token[0]
+
           if type == :text
-            current_text += value
-            current_context ||= context
+            value = token[1]
+
+            if current_text
+              current_text << value
+              current_context ||= token[2]
+            else
+              current_text = value.dup
+              current_context = token[2]
+            end
           else
-            unless current_text.empty?
+            if current_text
               optimized << [:text, current_text, current_context]
 
-              current_text = ""
+              current_text = nil
               current_context = nil
             end
 
-            optimized << [type, value, context, escaped]
+            optimized << [type, token[1], token[2], token[3]]
           end
         end
 
-        optimized << [:text, current_text, current_context] unless current_text.empty?
+        optimized << [:text, current_text, current_context] if current_text
 
         optimized
       end
 
-      def compact_whitespace_tokens(tokens)
-        return tokens if tokens.empty?
-
-        tokens.map.with_index { |token, index|
-          next token unless token[0] == :whitespace
-
-          next nil if adjacent_whitespace?(tokens, index)
-          next nil if whitespace_before_code_sequence?(tokens, index)
-
-          [:text, token[1], token[2]]
-        }.compact
+      def padding_newlines(count)
+        PADDING_NEWLINES[count] || ("\n" * count)
       end
 
-      def adjacent_whitespace?(tokens, index)
-        prev_token = index.positive? ? tokens[index - 1] : nil
-        next_token = index < tokens.length - 1 ? tokens[index + 1] : nil
-
-        trailing_whitespace?(prev_token) || leading_whitespace?(next_token)
-      end
+      def optimize_tokens_with_padding(tokens)
+        optimized = [] #: Array[untyped]
+        current_text = nil #: String?
+        current_context = nil
+        pending_padding = 0
 
-      def trailing_whitespace?(token)
-        return false unless token
+        tokens.each_with_index do |token, index|
+          pending_padding += @padding_before[index]
 
-        token[0] == :whitespace || (token[0] == :text && token[1] =~ /\s\z/)
-      end
+          if token[0] == :text
+            if pending_padding.positive?
+              if current_text
+                optimized << [:text, current_text, current_context]
+                current_text = nil
+                current_context = nil
+              end
 
-      def leading_whitespace?(token)
-        token && token[0] == :text && token[1] =~ /\A\s/
-      end
+              optimized << [:code, padding_newlines(pending_padding), nil]
+              pending_padding = 0
+            end
 
-      def whitespace_before_code_sequence?(tokens, current_index)
-        previous_token = tokens[current_index - 1] if current_index.positive?
+            if current_text
+              current_text << token[1]
+              current_context ||= token[2]
+            else
+              current_text = token[1].dup
+              current_context = token[2]
+            end
 
-        return false unless previous_token && previous_token[0] == :code
+            next
+          end
 
-        token_before_code = find_token_before_code_sequence(tokens, current_index)
+          if current_text
+            optimized << [:text, current_text, current_context]
+            current_text = nil
+            current_context = nil
+          end
 
-        return false unless token_before_code
+          if pending_padding.positive?
+            optimized << [:code, padding_newlines(pending_padding), nil]
+            pending_padding = 0
+          end
 
-        trailing_whitespace?(token_before_code)
-      end
+          optimized << [token[0], token[1], token[2], token[3]]
+        end
 
-      def find_token_before_code_sequence(tokens, whitespace_index)
-        search_index = whitespace_index - 1
+        optimized << [:text, current_text, current_context] if current_text
 
-        search_index -= 1 while search_index >= 0 && tokens[search_index][0] == :code
+        pending_padding += @padding_before[tokens.length]
+        optimized << [:code, padding_newlines(pending_padding), nil] if pending_padding.positive?
 
-        search_index >= 0 ? tokens[search_index] : nil
+        optimized
       end
 
       def process_erb_output(node, opening, code)
@@ -554,6 +677,8 @@ module Herb
           @last_trim_consumed_newline = false
         end
 
+        settle_pending_trim_newline!(false)
+
         should_escape = should_escape_output?(opening)
         add_expression_with_escaping(code, should_escape)
         @trim_next_whitespace = true if right_trim?(node)
@@ -582,6 +707,36 @@ module Herb
         end
       end
 
+      #: (String) -> Integer
+      def trailing_space_length(text)
+        count = 0
+        count += 1 while count < text.length && HORIZONTAL_SPACE.include?(text[text.length - 1 - count])
+
+        count
+      end
+
+      #: (String) -> String
+      def trailing_spaces(text)
+        text[text.length - trailing_space_length(text), text.length].to_s
+      end
+
+      #: (String) -> void
+      def remove_trailing_spaces!(text)
+        text.replace(text[0, text.length - trailing_space_length(text)].to_s)
+      end
+
+      #: (String) -> bool
+      def whitespace_only?(text)
+        !text.empty? && trailing_space_length(text) == text.length
+      end
+
+      #: (String) -> bool
+      def trailing_indentation?(text)
+        spaces = trailing_space_length(text)
+
+        spaces.positive? && text[text.length - spaces - 1] == "\n"
+      end
+
       def at_line_start?
         return true if @tokens.empty?
 
@@ -589,7 +744,7 @@ module Herb
         last_value = @tokens.last[1]
 
         if last_type == :text
-          last_value.empty? || last_value.end_with?("\n") || (last_value =~ WHITESPACE_ONLY && preceding_token_ends_with_newline?) || last_value =~ TRAILING_INDENTATION
+          last_value.empty? || last_value.end_with?("\n") || (whitespace_only?(last_value) && preceding_token_ends_with_newline?) || trailing_indentation?(last_value)
         elsif EXPRESSION_TOKEN_TYPES.include?(last_type)
           @last_trim_consumed_newline
         else
@@ -609,13 +764,30 @@ module Herb
       end
 
       def left_trim?(node)
-        node.tag_opening.value == "<%-"
+        node.tag_opening&.value == "<%-"
       end
 
       def right_trim?(node)
         node.tag_closing&.value == "-%>"
       end
 
+      def preceded_by_whitespace?
+        index = @tokens.length - 1
+        index -= 1 while index >= 0 && emits_nothing?(@tokens[index])
+
+        return false if index.negative?
+
+        token = @tokens[index]
+
+        return false unless token[0] == :text
+
+        token[1].match?(/\s\z/)
+      end
+
+      def emits_nothing?(token)
+        token[0] == :code || (token[0] == :text && token[1].empty?)
+      end
+
       def last_text_token
         return unless @tokens.last && @tokens.last[0] == :text
 
@@ -628,7 +800,7 @@ module Herb
 
         text = token[1]
 
-        return Regexp.last_match(1) if text =~ TRAILING_INDENTATION_CAPTURE || text =~ WHITESPACE_ONLY_CAPTURE
+        return trailing_spaces(text) if trailing_indentation?(text) || whitespace_only?(text)
 
         ""
       end
@@ -639,10 +811,17 @@ module Herb
 
         text = token[1]
 
-        return true if text.match?(TRAILING_INDENTATION)
-        return true if @last_trim_consumed_newline && text.match?(WHITESPACE_ONLY)
+        return true if trailing_indentation?(text)
 
-        false
+        whitespace_only?(text) && (preceding_text_ends_with_newline? || @last_trim_consumed_newline)
+      end
+
+      def preceding_text_ends_with_newline?
+        return true unless @tokens.length >= 2
+
+        preceding = @tokens[-2]
+
+        preceding[0] == :text && preceding[1].end_with?("\n")
       end
 
       def extract_and_remove_leading_space!
@@ -651,9 +830,9 @@ module Herb
 
         text = @tokens.last[1]
 
-        if text =~ TRAILING_INDENTATION
-          text.sub!(TRAILING_WHITESPACE, "")
-        elsif text =~ WHITESPACE_ONLY
+        if trailing_indentation?(text)
+          remove_trailing_spaces!(text)
+        elsif whitespace_only?(text)
           text.replace("")
         end
 
@@ -663,23 +842,77 @@ module Herb
       end
 
       def apply_trim(node, code)
+        return add_code(code) unless @trim
+
         follows_newline = leading_space_follows_newline?
         removed_whitespace = left_trim?(node) ? remove_trailing_whitespace_from_last_token! : ""
 
         if at_line_start?
           leading_space = extract_and_remove_leading_space!
           effective_leading_space = leading_space.empty? ? removed_whitespace : leading_space
-          right_space = Herb::Engine.heredoc?(code) ? "\n" : " \n"
+          right_space = if Herb::Engine::Helpers.heredoc?(code)
+                          "\n"
+                        elsif without_source?(node)
+                          " "
+                        else
+                          " \n"
+                        end
 
           @pending_leading_whitespace_insert_index = @tokens.length
           @pending_leading_whitespace = effective_leading_space if !effective_leading_space.empty? && follows_newline
           @tokens << [:code, "#{effective_leading_space}#{code}#{right_space}", current_context]
           @trim_next_whitespace = true
+
+          if right_space.end_with?(" \n")
+            @pending_trim_newline_index = @tokens.length - 1
+            @pending_trim_owns_next_whitespace = true
+          end
         else
           @tokens << [:code, code, current_context]
         end
       end
 
+      def continues_into_next_node?(node)
+        node.tag_closing.nil? && !node.content.nil?
+      end
+
+      #: (untyped) -> bool
+      def without_source?(node)
+        position = node.tag_closing&.location&.end || node.location&.end
+
+        !position.nil? && !position.line.positive?
+      end
+
+      #: (bool) -> void
+      def settle_pending_span_extra!(keep)
+        index = @pending_span_extra_index
+
+        return unless index
+
+        @pending_span_extra_index = nil
+
+        return if keep
+
+        @padding_before[index] -= 1 if @padding_before&.key?(index)
+      end
+
+      #: (bool) -> void
+      def settle_pending_trim_newline!(keep)
+        index = @pending_trim_newline_index
+
+        return unless index
+
+        owned = @pending_trim_owns_next_whitespace
+
+        @pending_trim_newline_index = nil
+        @pending_trim_owns_next_whitespace = false
+
+        return if keep
+
+        @tokens[index][1] = @tokens[index][1].chomp
+        @trim_next_whitespace = false if owned
+      end
+
       def save_pending_leading_whitespace!(whitespace)
         @pending_leading_whitespace = whitespace
         @pending_leading_whitespace_insert_index = @tokens.length
@@ -689,6 +922,11 @@ module Herb
         return unless @pending_leading_whitespace
 
         @tokens.insert(@pending_leading_whitespace_insert_index, [:text, @pending_leading_whitespace, current_context])
+
+        return unless @pending_trim_newline_index
+        return if @pending_trim_newline_index < @pending_leading_whitespace_insert_index
+
+        @pending_trim_newline_index += 1
       end
 
       def remove_trailing_whitespace_from_last_token!
@@ -696,12 +934,12 @@ module Herb
         return "" unless token
 
         text = token[1]
-        removed = text[TRAILING_WHITESPACE] || ""
+        removed = trailing_spaces(text)
 
-        if text =~ TRAILING_INDENTATION
-          text.sub!(TRAILING_WHITESPACE, "")
+        if trailing_indentation?(text)
+          remove_trailing_spaces!(text)
           token[1] = text
-        elsif text =~ WHITESPACE_ONLY
+        elsif whitespace_only?(text)
           text.replace("")
           token[1] = text
         end