w_syntax_tree-erb 0.9.5 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7625d38e4764dc486d61c0141a4e25892efa6fc993779b08885110dd55f830e6
-  data.tar.gz: 1551342a3b0550f0e59bf92c184586f0b8ea401bb2c4f55058e6fb1b7d0759f7
+  metadata.gz: a76706871104d4d2ea966b498103883d197378ec7e93fb339430b82d8e70d7ee
+  data.tar.gz: 2d78160b39db8ddf8a62aaa132c3d935870d5db8f2383250a0502d7475d1bd1f
 SHA512:
-  metadata.gz: c737cc77acef8c790b3e08a99aacc509e01e69ea67a70ecfa2684db9034bcb65c4ec7cb705fafe6e8c147a8f0a4237d015ebeedddd213e26c7a75fe123348618
-  data.tar.gz: 38f27bdf74980a3fae073b2fc5ff5e2114eacfe69886da0b819e62ac5547ae220e369de722a0284aa556dfce2f1e00e3b61364838e43438961119891fdf4f2b8
+  metadata.gz: 111165a4191e3dd16baf66ef3ab1b1607a4ea1303113a63b5bf01a49d6b0ff0949ff313fb3ba6ada0e42a04de7809294738dc6b079537590915b3b764ac4b030
+  data.tar.gz: 8ff9ea2143768e7eb7ee067688b4ed1b33f05412f8bf381782376586edc7d958f3f53c4c28042320f583408f0775bee00ecedda7df8c145598b874cf8bcf2046

data/.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npx lint-staged

data/.vscode/launch.json

@@ -0,0 +1,25 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "ruby_lsp",
+      "name": "Debug file",
+      "request": "launch",
+      "program": "ruby ${file}"
+    },
+    {
+      "type": "ruby_lsp",
+      "name": "Debug test",
+      "request": "launch",
+      "program": "ruby -Itest ${relativeFile}"
+    },
+    {
+      "type": "ruby_lsp",
+      "name": "Debug attach",
+      "request": "attach"
+    }
+  ]
+}

data/CHANGELOG.md

@@ -6,6 +6,21 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [0.10.1] - 2023-08-20
+
+### Added
+
+- Allow `DOCTYPE` to be after other tags, to work with e.g. ERB-tags on first line.
+
+## [0.10.0] - 2023-08-20
+
+- Changes how whitespace and newlines are handled.
+- Supports syntax like:
+
+```erb
+<%= part %> / <%= total %> (<%= percentage %>%)
+```
+
 ## [0.9.5] - 2023-07-02
 
 - Fixes ruby comment in ERB-tag included VoidStatement
@@ -61,5 +76,12 @@ Output:
 - Can format a lot of .html.erb-syntax and works as a plugin to syntax_tree.
 - This is still early and there are a lot of different weird syntaxes out there.
 
-[unreleased]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.9.0...HEAD
+[unreleased]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.10.1...HEAD
+[0.10.1]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.10.0...v0.10.1
+[0.10.0]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.9.5...v0.10.0
+[0.9.5]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.9.4...v0.9.5
+[0.9.4]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.9.3...v0.9.4
+[0.9.3]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.9.2...v0.9.3
+[0.9.2]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.9.1...v0.9.2
+[0.9.1]: https://github.com/davidwessman/syntax_tree-erb/compare/v0.9.0...v0.9.1
 [0.9.0]: https://github.com/davidwessman/syntax_tree-erb/compare/419727a73af94057ca0980733e69ac8b4d52fdf4...v0.9.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    w_syntax_tree-erb (0.9.5)
+    w_syntax_tree-erb (0.10.1)
       prettier_print (~> 1.2, >= 1.2.0)
       syntax_tree (~> 6.1, >= 6.1.1)
 
@@ -9,7 +9,7 @@ GEM
   remote: https://rubygems.org/
   specs:
     docile (1.4.0)
-    minitest (5.18.1)
+    minitest (5.19.0)
     prettier_print (1.2.1)
     rake (13.0.6)
     simplecov (0.22.0)

data/README.md

@@ -30,7 +30,7 @@ Currently handles
 Add this line to your application's Gemfile:
 
 ```ruby
-gem "w_syntax_tree-erb", "~> 0.9", require: false
+gem "w_syntax_tree-erb", "~> 0.10", require: false
 ```
 
 > I added the `w_` prefix to avoid conflicts if there will ever be an official `syntax_tree-erb` gem.
@@ -77,6 +77,21 @@ Dir
 puts failures
 ```
 
+## Development
+
+Setup linting:
+
+```sh
+npm run prepare
+```
+
+Install dependencies and run tests:
+
+```sh
+bundle
+bundle exec rake
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/davidwessman/syntax_tree-erb.

data/lib/syntax_tree/erb/format.rb

@@ -22,9 +22,7 @@ module SyntaxTree
       def visit_document(node)
         child_nodes = node.child_nodes.sort_by(&:location)
 
-        q.seplist(child_nodes, -> { q.breakable(force: true) }) do |child_node|
-          visit(child_node)
-        end
+        handle_child_nodes(child_nodes)
 
         q.breakable(force: true)
       end
@@ -32,31 +30,55 @@ module SyntaxTree
       def visit_block(node)
         visit(node.opening)
 
+        breakable = breakable_inside(node)
         if node.elements.any?
           q.indent do
-            q.breakable("")
-            q.seplist(
-              node.elements,
-              -> { q.breakable(force: true) }
-            ) { |child_node| visit(child_node) }
+            q.breakable if breakable
+            handle_child_nodes(node.elements)
           end
         end
 
         if node.closing
-          q.breakable("")
+          q.breakable("") if breakable
           visit(node.closing)
         end
       end
 
-      def visit_html(node)
-        # Make sure to group the tags together if there is no child nodes.
+      def visit_html_groupable(node, group)
         if node.elements.size == 0
-          q.group do
-            visit(node.opening)
-            visit(node.closing)
+          visit(node.opening)
+          visit(node.closing)
+        else
+          visit(node.opening)
+
+          with_break = breakable_inside(node)
+          q.indent do
+            if with_break
+              group ? q.breakable("") : q.breakable
+            end
+            handle_child_nodes(node.elements)
+          end
+
+          if with_break
+            group ? q.breakable("") : q.breakable
           end
+          visit(node.closing)
+        end
+      end
+
+      def visit_html(node)
+        # Make sure to group the tags together if there is no child nodes.
+        if node.elements.size == 0 ||
+             node.elements.any? { |node|
+               node.is_a?(SyntaxTree::ERB::CharData)
+             } ||
+             (
+               node.elements.size == 1 &&
+                 node.elements.first.is_a?(SyntaxTree::ERB::ErbNode)
+             )
+          q.group { visit_html_groupable(node, true) }
         else
-          visit_block(node)
+          visit_html_groupable(node, false)
         end
       end
 
@@ -146,7 +168,7 @@ module SyntaxTree
         formatter.flush
         rows = formatter.output.join.split("\n")
 
-        output_rows(formatter.output.join.split("\n"))
+        output_rows(rows)
       end
 
       def output_rows(rows)
@@ -170,9 +192,14 @@ module SyntaxTree
                 visit(child_node)
               end
             end
+
+            # Only add breakable if we have attributes
+            q.breakable(node.closing.value == "/>" ? " " : "")
+          elsif node.closing.value == "/>"
+            # Need a space before end-tag for self-closing
+            q.text(" ")
           end
 
-          q.breakable(node.closing.value == "/>" ? " " : "")
           visit(node.closing)
         end
       end
@@ -208,15 +235,20 @@ module SyntaxTree
         visit(node.token)
       end
 
+      def visit_erb_comment(node)
+        visit(node.token)
+      end
+
       # Visit a CharData node.
       def visit_char_data(node)
-        lines = node.value.value.strip.split("\n")
+        return if node.value.value.strip.empty?
 
-        if lines.size > 0
-          q.seplist(lines, -> { q.breakable(indent: false) }) do |line|
-            q.text(line)
-          end
-        end
+        q.text(node.value.value)
+      end
+
+      def visit_new_line(node)
+        q.breakable(force: :skip_parent_break)
+        q.breakable(force: :skip_parent_break) if node.count > 1
       end
 
       # Visit a Doctype node.
@@ -230,6 +262,49 @@ module SyntaxTree
         end
       end
 
+      private
+
+      def breakable_inside(node)
+        if node.is_a?(SyntaxTree::ERB::HtmlNode)
+          node.elements.first.class != SyntaxTree::ERB::CharData ||
+            node_new_line_count(node.opening) > 0
+        elsif node.is_a?(SyntaxTree::ERB::Block)
+          true
+        end
+      end
+
+      def breakable_between(node, next_node)
+        new_lines = node_new_line_count(node)
+
+        if new_lines == 1
+          q.breakable
+        elsif new_lines > 1
+          q.breakable
+          q.breakable(force: :skip_parent_break)
+        elsif next_node && !node.is_a?(SyntaxTree::ERB::CharData) &&
+              !next_node.is_a?(SyntaxTree::ERB::CharData)
+          q.breakable
+        end
+      end
+
+      def breakable_between_group(node, next_node)
+        new_lines = node_new_line_count(node)
+
+        if new_lines == 1
+          q.breakable(force: true)
+        elsif new_lines > 1
+          q.breakable(force: true)
+          q.breakable(force: true)
+        elsif next_node && !node.is_a?(SyntaxTree::ERB::CharData) &&
+              !next_node.is_a?(SyntaxTree::ERB::CharData)
+          q.breakable("")
+        end
+      end
+
+      def node_new_line_count(node)
+        node.respond_to?(:new_line) ? node.new_line&.count || 0 : 0
+      end
+
       def erb_print_width(node)
         # Set the width to maximum if we have an IfNode or IfOp,
         # we cannot format them purely with SyntaxTree because the ERB-syntax will be unparseable.
@@ -247,6 +322,60 @@ module SyntaxTree
           check_for_if_statement(child_node)
         end
       end
+
+      def handle_child_nodes(child_nodes)
+        group = []
+
+        if child_nodes.size == 1
+          visit(child_nodes.first)
+          return
+        end
+
+        child_nodes.each_with_index do |child_node, index|
+          is_last = index == child_nodes.size - 1
+
+          # Last element should not have new lines
+          node = is_last ? child_node.without_new_line : child_node
+
+          if node_should_group(node)
+            group << node
+            next
+          end
+
+          # Render all group elements before the current node
+          handle_group(group, break_after: true)
+          group = []
+
+          # Render the current node
+          visit(node)
+          next_node = child_nodes[index + 1]
+
+          breakable_between(node, next_node)
+        end
+
+        # Handle group if we have any nodes left
+        handle_group(group, break_after: false)
+      end
+
+      def handle_group(nodes, break_after:)
+        return unless nodes.any?
+
+        q.group do
+          nodes.each_with_index do |node, group_index|
+            visit(node)
+            next_node = nodes[group_index + 1]
+            next if next_node.nil?
+            breakable_between_group(node, next_node)
+          end
+        end
+
+        breakable_between_group(nodes.last, nil) if break_after
+      end
+
+      def node_should_group(node)
+        node.is_a?(SyntaxTree::ERB::CharData) ||
+          node.is_a?(SyntaxTree::ERB::ErbNode)
+      end
     end
   end
 end

data/lib/syntax_tree/erb/nodes.rb

@@ -53,6 +53,14 @@ module SyntaxTree
       def pretty_print(q)
         PrettyPrint.new(q).visit(self)
       end
+
+      def without_new_line
+        self
+      end
+
+      def skip?
+        false
+      end
     end
 
     # A Token is any kind of lexical token from the source. It has a type, a
@@ -110,6 +118,25 @@ module SyntaxTree
       end
     end
 
+    # This is a base class for a Node that can also hold an appended
+    # new line.
+    class Element < Node
+      attr_reader(:new_line, :location)
+
+      def initialize(new_line:, location:)
+        @new_line = new_line
+        @location = location
+      end
+
+      def without_new_line
+        self.class.new(**deconstruct_keys([]).merge(new_line: nil))
+      end
+
+      def deconstruct_keys(keys)
+        { new_line: new_line, location: location }
+      end
+    end
+
     # This is a base class for a block that contains:
     # - an opening
     # - optional elements
@@ -131,12 +158,22 @@ module SyntaxTree
         [opening, *elements, closing].compact
       end
 
+      def new_line
+        closing.new_line if closing.respond_to?(:new_line)
+      end
+
+      def without_new_line
+        self.class.new(
+          **deconstruct_keys([]).merge(closing: closing&.without_new_line)
+        )
+      end
+
       alias deconstruct child_nodes
 
       def deconstruct_keys(keys)
         {
           opening: opening,
-          content: content,
+          elements: elements,
           closing: closing,
           location: location
         }
@@ -151,15 +188,22 @@ module SyntaxTree
       # The opening tag of an element. It contains the opening character (<),
       # the name of the element, any optional attributes, and the closing
       # token (either > or />).
-      class OpeningTag < Node
-        attr_reader :opening, :name, :attributes, :closing, :location
-
-        def initialize(opening:, name:, attributes:, closing:, location:)
+      class OpeningTag < Element
+        attr_reader :opening, :name, :attributes, :closing
+
+        def initialize(
+          opening:,
+          name:,
+          attributes:,
+          closing:,
+          new_line:,
+          location:
+        )
+          super(new_line: new_line, location: location)
           @opening = opening
           @name = name
           @attributes = attributes
           @closing = closing
-          @location = location
         end
 
         def accept(visitor)
@@ -173,26 +217,25 @@ module SyntaxTree
         alias deconstruct child_nodes
 
         def deconstruct_keys(keys)
-          {
+          super.merge(
             opening: opening,
             name: name,
             attributes: attributes,
-            closing: closing,
-            location: location
-          }
+            closing: closing
+          )
         end
       end
 
       # The closing tag of an element. It contains the opening character (<),
       # the name of the element, and the closing character (>).
-      class ClosingTag < Node
-        attr_reader :opening, :name, :closing, :location
+      class ClosingTag < Element
+        attr_reader :opening, :name, :closing
 
-        def initialize(opening:, name:, closing:, location:)
+        def initialize(opening:, name:, closing:, location:, new_line:)
+          super(new_line: new_line, location: location)
           @opening = opening
           @name = name
           @closing = closing
-          @location = location
         end
 
         def accept(visitor)
@@ -206,19 +249,41 @@ module SyntaxTree
         alias deconstruct child_nodes
 
         def deconstruct_keys(keys)
-          { opening: opening, name: name, closing: closing, location: location }
+          super.merge(opening: opening, name: name, closing: closing)
         end
       end
 
+      def without_new_line
+        self.class.new(
+          **deconstruct_keys([]).merge(
+            opening: closing.nil? ? opening.without_new_line : opening,
+            closing: closing&.without_new_line
+          )
+        )
+      end
+
+      # The HTML-closing tag is responsible for new lines after the node.
+      def new_line
+        closing.nil? ? opening.new_line : closing&.new_line
+      end
+
       def accept(visitor)
         visitor.visit_html(self)
       end
     end
 
-    class ErbNode < Node
-      attr_reader :opening_tag, :keyword, :content, :closing_tag, :location
-
-      def initialize(opening_tag:, keyword:, content:, closing_tag:, location:)
+    class ErbNode < Element
+      attr_reader :opening_tag, :keyword, :content, :closing_tag
+
+      def initialize(
+        opening_tag:,
+        keyword:,
+        content:,
+        closing_tag:,
+        new_line:,
+        location:
+      )
+        super(new_line: new_line, location: location)
         @opening_tag = opening_tag
         # prune whitespace from keyword
         @keyword =
@@ -229,12 +294,18 @@ module SyntaxTree
               location: keyword.location
             )
           end
-        # Set content to nil if it is empty
-        content ||= []
-        content = content.map(&:value).join if content.is_a?(Array)
-        @content = ErbContent.new(value: content) unless content.strip.empty?
+
+        @content =
+          if content.is_a?(ErbContent)
+            content
+          else
+            # Set content to nil if it is empty
+            content ||= []
+            content = content.map(&:value).join if content.is_a?(Array)
+            ErbContent.new(value: content) unless content.strip.empty?
+          end
+
         @closing_tag = closing_tag
-        @location = location
       end
 
       def accept(visitor)
@@ -245,30 +316,50 @@ module SyntaxTree
         [opening_tag, keyword, content, closing_tag].compact
       end
 
+      def new_line
+        closing_tag&.new_line
+      end
+
+      def without_new_line
+        self.class.new(
+          **deconstruct_keys([]).merge(
+            closing_tag: closing_tag.without_new_line
+          )
+        )
+      end
+
       alias deconstruct child_nodes
 
       def deconstruct_keys(keys)
-        {
+        super.merge(
           opening_tag: opening_tag,
           keyword: keyword,
           content: content,
-          closing_tag: closing_tag,
-          location: location
-        }
+          closing_tag: closing_tag
+        )
       end
     end
 
     class ErbBlock < Block
+      def initialize(opening:, location:, elements: nil, closing: nil)
+        super(
+          opening: opening,
+          location: location,
+          elements: elements,
+          closing: closing
+        )
+      end
+
       def accept(visitor)
         visitor.visit_erb_block(self)
       end
     end
 
-    class ErbClose < Node
-      attr_reader :location, :closing
+    class ErbClose < Element
+      attr_reader :closing
 
-      def initialize(location:, closing:)
-        @location = location
+      def initialize(closing:, new_line:, location:)
+        super(new_line: new_line, location: location)
         @closing = closing
       end
 
@@ -283,7 +374,7 @@ module SyntaxTree
       alias deconstruct child_nodes
 
       def deconstruct_keys(keys)
-        { location: location, closing: closing }
+        super.merge(closing: closing)
       end
     end
 
@@ -442,12 +533,12 @@ module SyntaxTree
       end
     end
 
-    class HtmlComment < Node
-      attr_reader :token, :location
+    class HtmlComment < Element
+      attr_reader :token
 
-      def initialize(token:, location:)
+      def initialize(token:, new_line:, location:)
+        super(new_line: new_line, location: location)
         @token = token
-        @location = location
       end
 
       def accept(visitor)
@@ -461,18 +552,41 @@ module SyntaxTree
       alias deconstruct child_nodes
 
       def deconstruct_keys(keys)
-        { token: token, location: location }
+        super.merge(token: token)
+      end
+    end
+
+    class ErbComment < Element
+      attr_reader :token
+
+      def initialize(token:, new_line:, location:)
+        super(new_line: new_line, location: location)
+        @token = token
+      end
+
+      def accept(visitor)
+        visitor.visit_erb_comment(self)
+      end
+
+      def child_nodes
+        []
+      end
+
+      alias deconstruct child_nodes
+
+      def deconstruct_keys(keys)
+        super.merge(token: token)
       end
     end
 
     # A CharData contains either plain text or whitespace within an element.
     # It wraps a single token value.
-    class CharData < Node
-      attr_reader :value, :location
+    class CharData < Element
+      attr_reader :value
 
-      def initialize(value:, location:)
+      def initialize(value:, new_line:, location:)
+        super(new_line: new_line, location: location)
         @value = value
-        @location = location
       end
 
       def accept(visitor)
@@ -486,7 +600,34 @@ module SyntaxTree
       alias deconstruct child_nodes
 
       def deconstruct_keys(keys)
-        { value: value, location: location }
+        super.merge(value: value)
+      end
+
+      def skip?
+        value.value.strip.empty?
+      end
+    end
+
+    class NewLine < Node
+      attr_reader :count, :location
+
+      def initialize(location:, count:)
+        @location = location
+        @count = count
+      end
+
+      def accept(visitor)
+        visitor.visit_new_line(self)
+      end
+
+      def child_nodes
+        []
+      end
+
+      alias deconstruct child_nodes
+
+      def deconstruct_keys(keys)
+        { location: location, count: count }
       end
     end
 
@@ -494,14 +635,14 @@ module SyntaxTree
     # type of the document. It contains an opening declaration, the name of
     # the document type, an optional external identifier, and a closing of the
     # tag.
-    class Doctype < Node
-      attr_reader :opening, :name, :closing, :location
+    class Doctype < Element
+      attr_reader :opening, :name, :closing
 
-      def initialize(opening:, name:, closing:, location:)
+      def initialize(opening:, name:, closing:, new_line:, location:)
+        super(new_line: new_line, location: location)
         @opening = opening
         @name = name
         @closing = closing
-        @location = location
       end
 
       def accept(visitor)
@@ -515,7 +656,7 @@ module SyntaxTree
       alias deconstruct child_nodes
 
       def deconstruct_keys(keys)
-        { opening: opening, name: name, closing: closing, location: location }
+        super.merge(opening: opening, name: name, closing: closing)
       end
     end
   end

data/lib/syntax_tree/erb/parser.rb

@@ -20,16 +20,16 @@ module SyntaxTree
       def initialize(source)
         @source = source
         @tokens = make_tokens
+        @found_doctype = false
       end
 
       def parse
-        doctype = maybe { parse_doctype }
         elements = many { parse_any_tag }
 
         location =
           elements.first.location.to(elements.last.location) if elements.any?
 
-        Document.new(elements: [doctype].compact + elements, location: location)
+        Document.new(elements: elements, location: location)
       end
 
       def debug_tokens
@@ -41,10 +41,24 @@ module SyntaxTree
       private
 
       def parse_any_tag
-        atleast do
-          maybe { parse_html_comment } || maybe { parse_erb_tag } ||
-            maybe { consume(:erb_comment) } || maybe { parse_html_element } ||
-            maybe { parse_blank_line } || maybe { parse_chardata }
+        loop do
+          tag =
+            atleast do
+              maybe { parse_doctype } || maybe { parse_html_comment } ||
+                maybe { parse_erb_tag } || maybe { parse_erb_comment } ||
+                maybe { parse_html_element } || maybe { parse_new_line } ||
+                maybe { parse_chardata }
+            end
+
+          if tag.is_a?(Doctype)
+            if @found_doctype
+              raise(ParseError, "Only one doctype element is allowed")
+            else
+              @found_doctype = true
+            end
+          end
+          # Allow skipping empty CharData
+          return tag unless tag.skip?
         end
       end
 
@@ -62,10 +76,10 @@ module SyntaxTree
                 # two or more newlines should be ONE blank line
                 enum.yield :blank_line, $&, index, line
                 line += $&.count("\n")
-              when /\A(?: |\t|\n|\r\n)+/m
-                # whitespace
-                # enum.yield :whitespace, $&, index, line
-                line += $&.count("\n")
+              when /\A\n/
+                # newlines
+                enum.yield :new_line, $&, index, line
+                line += 1
               when /\A<!--(.|\r?\n)*?-->/m
                 # comments
                 # <!-- this is a comment -->
@@ -97,8 +111,11 @@ module SyntaxTree
                 # <
                 enum.yield :open, $&, index, line
                 state << :inside
-              when /\A[^<]+/
-                # plain text content
+              when /\A(?: |\t|\r)+/m
+                # whitespace
+                enum.yield :whitespace, $&, index, line
+              when /\A(?!\s+$)[^<\n]+/
+                # plain text content, but do not allow only white space
                 # abc
                 enum.yield :text, $&, index, line
               else
@@ -178,7 +195,7 @@ module SyntaxTree
                 # the beginning of an ERB tag
                 # <%
                 enum.yield :erb_open, $&, index, line
-                state << :erb
+                state << :erb_start
               when /\A[^<']+/
                 # plain text content
                 # abc
@@ -201,7 +218,7 @@ module SyntaxTree
                 # the beginning of an ERB tag
                 # <%
                 enum.yield :erb_open, $&, index, line
-                state << :erb
+                state << :erb_start
               when /\A[^<"]+/
                 # plain text content
                 # abc
@@ -251,7 +268,7 @@ module SyntaxTree
                 # the beginning of an ERB tag
                 # <%
                 enum.yield :erb_open, $&, index, line
-                state << :erb
+                state << :erb_start
               when /\A"/
                 # the beginning of a string
                 enum.yield :string_open_double_quote, $&, index, line
@@ -357,12 +374,18 @@ module SyntaxTree
             maybe { consume(:close) } || maybe { consume(:slash_close) }
           end
 
+        new_line = maybe { parse_new_line }
+
+        # Parse any whitespace after new lines
+        maybe { consume(:whitespace) }
+
         HtmlNode::OpeningTag.new(
           opening: opening,
           name: name,
           attributes: attributes,
           closing: closing,
-          location: opening.location.to(closing.location)
+          location: opening.location.to(closing.location),
+          new_line: new_line
         )
       end
 
@@ -371,11 +394,14 @@ module SyntaxTree
         name = consume(:name)
         closing = consume(:close)
 
+        new_line = maybe { parse_new_line }
+
         HtmlNode::ClosingTag.new(
           opening: opening,
           name: name,
           closing: closing,
-          location: opening.location.to(closing.location)
+          location: opening.location.to(closing.location),
+          new_line: new_line
         )
       end
 
@@ -450,6 +476,9 @@ module SyntaxTree
       end
 
       def parse_erb_if(erb_node)
+        # Skip any leading whitespace
+        maybe { consume(:whitespace) }
+
         elements =
           maybe { parse_until_erb(classes: [ErbElsif, ErbElse, ErbEnd]) } || []
 
@@ -513,11 +542,14 @@ module SyntaxTree
       end
 
       def parse_erb_end(erb_node)
+        new_line = maybe { parse_new_line }
+
         ErbEnd.new(
           opening_tag: erb_node.opening_tag,
           keyword: erb_node.keyword,
           content: nil,
           closing_tag: erb_node.closing_tag,
+          new_line: new_line,
           location: erb_node.location
         )
       end
@@ -540,12 +572,15 @@ module SyntaxTree
           )
         end
 
+        new_line = maybe { parse_new_line }
+
         erb_node =
           ErbNode.new(
             opening_tag: opening_tag,
             keyword: keyword,
             content: content,
             closing_tag: closing_tag,
+            new_line: new_line,
             location: opening_tag.location.to(closing_tag.location)
           )
 
@@ -602,6 +637,7 @@ module SyntaxTree
               maybe { parse_erb_do_close } || maybe { parse_erb_close } ||
                 maybe { consume(:erb_code) }
             end
+
           items << result
 
           break if result.is_a?(ErbClose)
@@ -610,22 +646,46 @@ module SyntaxTree
         items
       end
 
-      def parse_blank_line
-        blank_line = consume(:blank_line)
+      # This method is called at the end of most tags, it fixes:
+      # 1. Parsing any new lines after the tag
+      # 2. Parsing any whitespace after the new lines
+      # The whitespace is just consumed
+      def parse_new_line
+        line_break =
+          atleast do
+            maybe { consume(:blank_line) } || maybe { consume(:new_line) }
+          end
+
+        maybe { consume(:whitespace) }
 
-        CharData.new(value: blank_line, location: blank_line.location)
+        NewLine.new(
+          location: line_break.location,
+          count: line_break.value.count("\n")
+        )
       end
 
       def parse_erb_close
         closing = consume(:erb_close)
 
-        ErbClose.new(location: closing.location, closing: closing)
+        new_line = maybe { parse_new_line }
+
+        ErbClose.new(
+          location: closing.location,
+          new_line: new_line,
+          closing: closing
+        )
       end
 
       def parse_erb_do_close
         closing = consume(:erb_do_close)
 
-        ErbDoClose.new(location: closing.location, closing: closing)
+        new_line = maybe { parse_new_line }
+
+        ErbDoClose.new(
+          location: closing.location,
+          new_line: new_line,
+          closing: closing
+        )
       end
 
       def parse_html_string
@@ -715,7 +775,15 @@ module SyntaxTree
             values.first
           end
 
-        CharData.new(value: token, location: token.location) if token
+        new_line = maybe { parse_new_line }
+
+        if token&.value
+          CharData.new(
+            value: token,
+            location: token.location,
+            new_line: new_line
+          )
+        end
       end
 
       def parse_doctype
@@ -723,10 +791,13 @@ module SyntaxTree
         name = consume(:name)
         closing = consume(:close)
 
+        new_line = maybe { parse_new_line }
+
         Doctype.new(
           opening: opening,
           name: name,
           closing: closing,
+          new_line: new_line,
           location: opening.location.to(closing.location)
         )
       end
@@ -734,7 +805,25 @@ module SyntaxTree
       def parse_html_comment
         comment = consume(:html_comment)
 
-        HtmlComment.new(token: comment, location: comment.location)
+        new_line = maybe { parse_new_line }
+
+        HtmlComment.new(
+          token: comment,
+          new_line: new_line,
+          location: comment.location
+        )
+      end
+
+      def parse_erb_comment
+        comment = consume(:erb_comment)
+
+        new_line = maybe { parse_new_line }
+
+        ErbComment.new(
+          token: comment,
+          new_line: new_line,
+          location: comment.location
+        )
       end
     end
   end

data/lib/syntax_tree/erb/pretty_print.rb

@@ -108,7 +108,7 @@ module SyntaxTree
       end
 
       def visit_erb_end(node)
-        q.pp("erb_end")
+        q.pp("(erb_end)")
       end
 
       # Visit an ErbContent node.
@@ -131,6 +131,10 @@ module SyntaxTree
         visit_node("char_data", node)
       end
 
+      def visit_new_line(node)
+        node.count > 1 ? q.text("(new_line blank)") : q.text("(new_line)")
+      end
+
       def visit_erb_close(node)
         visit(node.closing)
       end
@@ -148,6 +152,10 @@ module SyntaxTree
         visit_node("html_comment", node)
       end
 
+      def visit_erb_comment(node)
+        visit_node("erb_comment", node)
+      end
+
       private
 
       # A generic visit node function for how we pretty print nodes.

data/lib/syntax_tree/erb/version.rb

@@ -2,6 +2,6 @@
 
 module SyntaxTree
   module ERB
-    VERSION = "0.9.5"
+    VERSION = "0.10.1"
   end
 end

data/lib/syntax_tree/erb.rb

@@ -26,5 +26,6 @@ module SyntaxTree
     end
   end
 
+  register_handler(".html.erb", ERB)
   register_handler(".erb", ERB)
 end

data/package.json

@@ -0,0 +1,10 @@
+{
+  "scripts": {
+    "prepare": "husky install"
+  },
+  "lint-staged": {
+    "lib/**/*.rb": [
+      "bundle exec stree write"
+    ]
+  }
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: w_syntax_tree-erb
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.10.1
 platform: ruby
 authors:
 - Kevin Newton
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-02 00:00:00.000000000 Z
+date: 2023-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: prettier_print
@@ -121,6 +121,8 @@ files:
 - ".github/workflows/auto-merge.yml"
 - ".github/workflows/main.yml"
 - ".gitignore"
+- ".husky/pre-commit"
+- ".vscode/launch.json"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
@@ -135,6 +137,7 @@ files:
 - lib/syntax_tree/erb/pretty_print.rb
 - lib/syntax_tree/erb/version.rb
 - lib/syntax_tree/erb/visitor.rb
+- package.json
 - syntax_tree-erb.gemspec
 homepage: https://github.com/davidwessman/syntax_tree-erb
 licenses:
@@ -156,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: Syntax Tree support for ERB