syntax_tree 2.4.0 → 2.6.0

data/lib/syntax_tree/node.rb

@@ -123,7 +123,7 @@ module SyntaxTree
     end
 
     def construct_keys
-      PP.format(+"") { |q| Visitor::MatchVisitor.new(q).visit(self) }
+      PrettierPrint.format(+"") { |q| Visitor::MatchVisitor.new(q).visit(self) }
     end
   end
 
@@ -597,10 +597,30 @@ module SyntaxTree
         q.indent do
           q.breakable("")
           q.format(arguments)
+          q.if_break { q.text(",") } if q.trailing_comma? && trailing_comma?
         end
         q.breakable("")
       end
     end
+
+    private
+
+    def trailing_comma?
+      case arguments
+      in Args[parts: [*, ArgBlock]]
+        # If the last argument is a block, then we can't put a trailing comma
+        # after it without resulting in a syntax error.
+        false
+      in Args[parts: [Command | CommandCall]]
+        # If the only argument is a command or command call, then a trailing
+        # comma would be parsed as part of that expression instead of on this
+        # one, so we don't want to add a trailing comma.
+        false
+      else
+        # Otherwise, we should be okay to add a trailing comma.
+        true
+      end
+    end
   end
 
   # Args represents a list of arguments being passed to a method call or array
@@ -859,6 +879,7 @@ module SyntaxTree
             end
 
             q.seplist(contents.parts, separator) { |part| q.format(part) }
+            q.if_break { q.text(",") } if q.trailing_comma?
           end
           q.breakable("")
         end
@@ -954,6 +975,7 @@ module SyntaxTree
           q.indent do
             q.breakable("")
             q.format(contents)
+            q.if_break { q.text(",") } if q.trailing_comma?
           end
         end
 
@@ -1388,7 +1410,7 @@ module SyntaxTree
   module HashKeyFormatter
     # Formats the keys of a hash literal using labels.
     class Labels
-      LABEL = /^[@$_A-Za-z]([_A-Za-z0-9]*)?([!_=?A-Za-z0-9])?$/
+      LABEL = /\A[A-Za-z_](\w*[\w!?])?\z/
 
       def format_key(q, key)
         case key
@@ -1666,52 +1688,6 @@ module SyntaxTree
     end
   end
 
-  # This module will remove any breakables from the list of contents so that no
-  # newlines are present in the output.
-  module RemoveBreaks
-    class << self
-      def call(doc)
-        marker = Object.new
-        stack = [doc]
-
-        while stack.any?
-          doc = stack.pop
-
-          if doc == marker
-            stack.pop
-            next
-          end
-
-          stack += [doc, marker]
-
-          case doc
-          when PrettyPrint::Align, PrettyPrint::Indent, PrettyPrint::Group
-            doc.contents.map! { |child| remove_breaks(child) }
-            stack += doc.contents.reverse
-          when PrettyPrint::IfBreak
-            doc.flat_contents.map! { |child| remove_breaks(child) }
-            stack += doc.flat_contents.reverse
-          end
-        end
-      end
-
-      private
-
-      def remove_breaks(doc)
-        case doc
-        when PrettyPrint::Breakable
-          text = PrettyPrint::Text.new
-          text.add(object: doc.force? ? "; " : doc.separator, width: doc.width)
-          text
-        when PrettyPrint::IfBreak
-          PrettyPrint::Align.new(indent: 0, contents: doc.flat_contents)
-        else
-          doc
-        end
-      end
-    end
-  end
-
   # BlockVar represents the parameters being declared for a block. Effectively
   # this node is everything contained within the pipes. This includes all of the
   # various parameter types, as well as block-local variable declarations.
@@ -1752,8 +1728,7 @@ module SyntaxTree
 
     def format(q)
       q.group(0, "|", "|") do
-        doc = q.format(params)
-        RemoveBreaks.call(doc)
+        q.remove_breaks(q.format(params))
 
         if locals.any?
           q.text("; ")
@@ -3077,8 +3052,20 @@ module SyntaxTree
         doc =
           q.nest(0) do
             q.format(receiver)
-            q.format(CallOperatorFormatter.new(operator), stackable: false)
-            q.format(message)
+
+            # If there are leading comments on the message then we know we have
+            # a newline in the source that is forcing these things apart. In
+            # this case we will have to use a trailing operator.
+            if message.comments.any?(&:leading?)
+              q.format(CallOperatorFormatter.new(operator), stackable: false)
+              q.indent do
+                q.breakable("")
+                q.format(message)
+              end
+            else
+              q.format(CallOperatorFormatter.new(operator), stackable: false)
+              q.format(message)
+            end
           end
 
         case arguments
@@ -3096,31 +3083,6 @@ module SyntaxTree
 
     private
 
-    # This is a somewhat naive method that is attempting to sum up the width of
-    # the doc nodes that make up the given doc node. This is used to align
-    # content.
-    def doc_width(parent)
-      queue = [parent]
-      width = 0
-
-      until queue.empty?
-        doc = queue.shift
-
-        case doc
-        when PrettyPrint::Text
-          width += doc.width
-        when PrettyPrint::Indent, PrettyPrint::Align, PrettyPrint::Group
-          queue = doc.contents + queue
-        when PrettyPrint::IfBreak
-          queue = doc.break_contents + queue
-        when PrettyPrint::Breakable
-          width = 0
-        end
-      end
-
-      width
-    end
-
     def argument_alignment(q, doc)
       # Very special handling case for rspec matchers. In general with rspec
       # matchers you expect to see something like:
@@ -3138,7 +3100,7 @@ module SyntaxTree
       if %w[to not_to to_not].include?(message.value)
         0
       else
-        width = doc_width(doc) + 1
+        width = q.last_position(doc) + 1
         width > (q.maxwidth / 2) ? 0 : width
       end
     end
@@ -3904,7 +3866,7 @@ module SyntaxTree
     # quotes, then single quotes would deactivate it.)
     def self.locked?(node)
       node.parts.any? do |part|
-        part.is_a?(TStringContent) && part.value.match?(/\\|#[@${]/)
+        !part.is_a?(TStringContent) || part.value.match?(/\\|#[@${]/)
       end
     end
 
@@ -4823,6 +4785,7 @@ module SyntaxTree
         q.indent do
           q.breakable
           q.seplist(assocs) { |assoc| q.format(assoc) }
+          q.if_break { q.text(",") } if q.trailing_comma?
         end
         q.breakable
       end
@@ -4891,17 +4854,9 @@ module SyntaxTree
     end
 
     def format(q)
-      # This is a very specific behavior that should probably be included in the
-      # prettyprint module. It's when you want to force a newline, but don't
-      # want to force the break parent.
-      breakable = -> do
-        q.target << PrettyPrint::Breakable.new(
-          " ",
-          1,
-          indent: false,
-          force: true
-        )
-      end
+      # This is a very specific behavior where you want to force a newline, but
+      # don't want to force the break parent.
+      breakable = -> { q.breakable(indent: false, force: :skip_break_parent) }
 
       q.group do
         q.format(beginning)
@@ -5064,13 +5019,16 @@ module SyntaxTree
       parts = keywords.map { |(key, value)| KeywordFormatter.new(key, value) }
       parts << KeywordRestFormatter.new(keyword_rest) if keyword_rest
 
+      nested = PATTERNS.include?(q.parent.class)
       contents = -> do
         q.group { q.seplist(parts) { |part| q.format(part, stackable: false) } }
 
         # If there isn't a constant, and there's a blank keyword_rest, then we
         # have an plain ** that needs to have a `then` after it in order to
         # parse correctly on the next parse.
-        q.text(" then") if !constant && keyword_rest && keyword_rest.value.nil?
+        if !constant && keyword_rest && keyword_rest.value.nil? && !nested
+          q.text(" then")
+        end
       end
 
       # If there is a constant, we're going to format to have the constant name
@@ -5097,7 +5055,7 @@ module SyntaxTree
 
       # If there's only one pair, then we'll just print the contents provided
       # we're not inside another pattern.
-      if !PATTERNS.include?(q.parent.class) && parts.size == 1
+      if !nested && parts.size == 1
         contents.call
         return
       end
@@ -5194,6 +5152,8 @@ module SyntaxTree
           case node
           in predicate: Assign | Command | CommandCall | MAssign | OpAssign
             false
+          in predicate: Not[parentheses: false]
+            false
           in {
                statements: { body: [truthy] },
                consequent: Else[statements: { body: [falsy] }]
@@ -5322,9 +5282,8 @@ module SyntaxTree
                 # force it into the output but we _don't_ want to explicitly
                 # break the parent. If a break-parent shows up in the tree, then
                 # it's going to force it all the way up to the tree, which is
-                # going to negate the ternary. Maybe this should be an option in
-                # prettyprint? As in force: :no_break_parent or something.
-                q.target << PrettyPrint::Breakable.new(" ", 1, force: true)
+                # going to negate the ternary.
+                q.breakable(force: :skip_break_parent)
                 q.format(node.consequent.statements)
               end
             end
@@ -5953,7 +5912,7 @@ module SyntaxTree
   #     ->(value) { value * 2 }
   #
   class Lambda < Node
-    # [Params | Paren] the parameter declaration for this lambda
+    # [LambdaVar | Paren] the parameter declaration for this lambda
     attr_reader :params
 
     # [BodyStmt | Statements] the expressions to be executed in this lambda
@@ -6008,24 +5967,100 @@ module SyntaxTree
                 node.is_a?(Command) || node.is_a?(CommandCall)
               end
 
-            q.text(force_parens ? "{" : "do")
-            q.indent do
+            if force_parens
+              q.text("{")
+
+              unless statements.empty?
+                q.indent do
+                  q.breakable
+                  q.format(statements)
+                end
+                q.breakable
+              end
+
+              q.text("}")
+            else
+              q.text("do")
+
+              unless statements.empty?
+                q.indent do
+                  q.breakable
+                  q.format(statements)
+                end
+              end
+
               q.breakable
-              q.format(statements)
+              q.text("end")
             end
-
-            q.breakable
-            q.text(force_parens ? "}" : "end")
           end
           .if_flat do
-            q.text("{ ")
-            q.format(statements)
-            q.text(" }")
+            q.text("{")
+
+            unless statements.empty?
+              q.text(" ")
+              q.format(statements)
+              q.text(" ")
+            end
+
+            q.text("}")
           end
       end
     end
   end
 
+  # LambdaVar represents the parameters being declared for a lambda. Effectively
+  # this node is everything contained within the parentheses. This includes all
+  # of the various parameter types, as well as block-local variable
+  # declarations.
+  #
+  #     -> (positional, optional = value, keyword:, &block; local) do
+  #     end
+  #
+  class LambdaVar < Node
+    # [Params] the parameters being declared with the block
+    attr_reader :params
+
+    # [Array[ Ident ]] the list of block-local variable declarations
+    attr_reader :locals
+
+    # [Array[ Comment | EmbDoc ]] the comments attached to this node
+    attr_reader :comments
+
+    def initialize(params:, locals:, location:, comments: [])
+      @params = params
+      @locals = locals
+      @location = location
+      @comments = comments
+    end
+
+    def accept(visitor)
+      visitor.visit_lambda_var(self)
+    end
+
+    def child_nodes
+      [params, *locals]
+    end
+
+    alias deconstruct child_nodes
+
+    def deconstruct_keys(_keys)
+      { params: params, locals: locals, location: location, comments: comments }
+    end
+
+    def empty?
+      params.empty? && locals.empty?
+    end
+
+    def format(q)
+      q.format(params)
+
+      if locals.any?
+        q.text("; ")
+        q.seplist(locals, -> { q.text(", ") }) { |local| q.format(local) }
+      end
+    end
+  end
+
   # LBrace represents the use of a left brace, i.e., {.
   class LBrace < Node
     # [String] the left brace
@@ -8311,8 +8346,7 @@ module SyntaxTree
         # same line in the source, then we're going to leave them in place and
         # assume that's the way the developer wanted this expression
         # represented.
-        doc = q.group(0, '#{', "}") { q.format(statements) }
-        RemoveBreaks.call(doc)
+        q.remove_breaks(q.group(0, '#{', "}") { q.format(statements) })
       else
         q.group do
           q.text('#{')

data/lib/syntax_tree/parser.rb

@@ -1671,9 +1671,15 @@ module SyntaxTree
     #     (nil | VarField) keyword_rest
     #   ) -> HshPtn
     def on_hshptn(constant, keywords, keyword_rest)
-      # Create an artificial VarField if we find an extra ** on the end
-      if !keyword_rest && (token = find_token(Op, "**", consume: false))
+      if keyword_rest
+        # We're doing this to delete the token from the list so that it doesn't
+        # confuse future patterns by thinking they have an extra ** on the end.
+        find_token(Op, "**")
+      elsif (token = find_token(Op, "**", consume: false))
         tokens.delete(token)
+
+        # Create an artificial VarField if we find an extra ** on the end. This
+        # means the formatting will be a little more consistent.
         keyword_rest = VarField.new(value: nil, location: token.location)
       end
 
@@ -1934,6 +1940,41 @@ module SyntaxTree
             token.location.start_char > beginning.location.start_char
         end
 
+      # We need to do some special mapping here. Since ripper doesn't support
+      # capturing lambda var until 3.2, we need to normalize all of that here.
+      params =
+        case params
+        in Paren[contents: Params]
+          # In this case we've gotten to the <3.2 parentheses wrapping a set of
+          # parameters case. Here we need to manually scan for lambda locals.
+          range = (params.location.start_char + 1)...params.location.end_char
+          locals = lambda_locals(source[range])
+
+          location = params.contents.location
+          location = location.to(locals.last.location) if locals.any?
+
+          Paren.new(
+            lparen: params.lparen,
+            contents:
+              LambdaVar.new(
+                params: params.contents,
+                locals: locals,
+                location: location
+              ),
+            location: params.location,
+            comments: params.comments
+          )
+        in Params
+          # In this case we've gotten to the <3.2 plain set of parameters. In
+          # this case there cannot be lambda locals, so we will wrap the
+          # parameters into a lambda var that has no locals.
+          LambdaVar.new(params: params, locals: [], location: params.location)
+        in LambdaVar
+          # In this case we've gotten to 3.2+ lambda var. In this case we don't
+          # need to do anything and can just the value as given.
+          params
+        end
+
       if braces
         opening = find_token(TLamBeg)
         closing = find_token(RBrace)
@@ -1956,6 +1997,83 @@ module SyntaxTree
       )
     end
 
+    # :call-seq:
+    #   on_lambda_var: (Params params, Array[ Ident ] locals) -> LambdaVar
+    def on_lambda_var(params, locals)
+      location = params.location
+      location = location.to(locals.last.location) if locals.any?
+
+      LambdaVar.new(params: params, locals: locals || [], location: location)
+    end
+
+    # Ripper doesn't support capturing lambda local variables until 3.2. To
+    # mitigate this, we have to parse that code for ourselves. We use the range
+    # from the parentheses to find where we _should_ be looking. Then we check
+    # if the resulting tokens match a pattern that we determine means that the
+    # declaration has block-local variables. Once it does, we parse those out
+    # and convert them into Ident nodes.
+    def lambda_locals(source)
+      tokens = Ripper.lex(source)
+
+      # First, check that we have a semi-colon. If we do, then we can start to
+      # parse the tokens _after_ the semicolon.
+      index = tokens.rindex { |token| token[1] == :on_semicolon }
+      return [] unless index
+
+      # Next, map over the tokens and convert them into Ident nodes. Bail out
+      # midway through if we encounter a token we didn't expect. Basically we're
+      # making our own mini-parser here. To do that we'll walk through a small
+      # state machine:
+      #
+      #     ┌────────┐               ┌────────┐                ┌─────────┐
+      #     │        │               │        │                │┌───────┐│
+      # ──> │  item  │ ─── ident ──> │  next  │ ─── rparen ──> ││ final ││
+      #     │        │ <── comma ─── │        │                │└───────┘│
+      #     └────────┘               └────────┘                └─────────┘
+      #        │  ^                     │  ^
+      #        └──┘                     └──┘
+      #    ignored_nl, sp              nl, sp
+      #
+      state = :item
+      transitions = {
+        item: {
+          on_ignored_nl: :item,
+          on_sp: :item,
+          on_ident: :next
+        },
+        next: {
+          on_nl: :next,
+          on_sp: :next,
+          on_comma: :item,
+          on_rparen: :final
+        },
+        final: {
+        }
+      }
+
+      tokens[(index + 1)..].each_with_object([]) do |token, locals|
+        (lineno, column), type, value, = token
+
+        # Make the state transition for the parser. If there isn't a transition
+        # from the current state to a new state for this type, then we're in a
+        # pattern that isn't actually locals. In that case we can return [].
+        state = transitions[state].fetch(type) { return [] }
+
+        # If we hit an identifier, then add it to our list.
+        next if type != :on_ident
+
+        location =
+          Location.token(
+            line: lineno,
+            char: line_counts[lineno - 1][column],
+            column: column,
+            size: value.size
+          )
+
+        locals << Ident.new(value: value, location: location)
+      end
+    end
+
     # :call-seq:
     #   on_lbrace: (String value) -> LBrace
     def on_lbrace(value)

data/lib/syntax_tree/plugin/single_quotes.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "syntax_tree/formatter/single_quotes"
+SyntaxTree::Formatter.prepend(SyntaxTree::Formatter::SingleQuotes)

data/lib/syntax_tree/plugin/trailing_comma.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "syntax_tree/formatter/trailing_comma"
+SyntaxTree::Formatter.prepend(SyntaxTree::Formatter::TrailingComma)

data/lib/syntax_tree/rake/check_task.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "rake"
+require "rake/tasklib"
+
+require "syntax_tree"
+require "syntax_tree/cli"
+
+module SyntaxTree
+  module Rake
+    # A Rake task that runs check on a set of source files.
+    #
+    # Example:
+    #
+    #   require 'syntax_tree/rake/check_task'
+    #
+    #   SyntaxTree::Rake::CheckTask.new do |t|
+    #     t.source_files = '{app,config,lib}/**/*.rb'
+    #   end
+    #
+    # This will create task that can be run with:
+    #
+    #   rake stree_check
+    #
+    class CheckTask < ::Rake::TaskLib
+      # Name of the task.
+      # Defaults to :"stree:check".
+      attr_accessor :name
+
+      # Glob pattern to match source files.
+      # Defaults to 'lib/**/*.rb'.
+      attr_accessor :source_files
+
+      # The set of plugins to require.
+      # Defaults to [].
+      attr_accessor :plugins
+
+      def initialize(
+        name = :"stree:check",
+        source_files = ::Rake::FileList["lib/**/*.rb"],
+        plugins = []
+      )
+        @name = name
+        @source_files = source_files
+        @plugins = plugins
+
+        yield self if block_given?
+        define_task
+      end
+
+      private
+
+      def define_task
+        desc "Runs `stree check` over source files"
+        task(name) { run_task }
+      end
+
+      def run_task
+        arguments = ["check"]
+        arguments << "--plugins=#{plugins.join(",")}" if plugins.any?
+
+        SyntaxTree::CLI.run(arguments + Array(source_files))
+      end
+    end
+  end
+end

data/lib/syntax_tree/rake/write_task.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "rake"
+require "rake/tasklib"
+
+require "syntax_tree"
+require "syntax_tree/cli"
+
+module SyntaxTree
+  module Rake
+    # A Rake task that runs format on a set of source files.
+    #
+    # Example:
+    #
+    #   require 'syntax_tree/rake/write_task'
+    #
+    #   SyntaxTree::Rake::WriteTask.new do |t|
+    #     t.source_files = '{app,config,lib}/**/*.rb'
+    #   end
+    #
+    # This will create task that can be run with:
+    #
+    #   rake stree_write
+    #
+    class WriteTask < ::Rake::TaskLib
+      # Name of the task.
+      # Defaults to :"stree:write".
+      attr_accessor :name
+
+      # Glob pattern to match source files.
+      # Defaults to 'lib/**/*.rb'.
+      attr_accessor :source_files
+
+      # The set of plugins to require.
+      # Defaults to [].
+      attr_accessor :plugins
+
+      def initialize(
+        name = :"stree:write",
+        source_files = ::Rake::FileList["lib/**/*.rb"],
+        plugins = []
+      )
+        @name = name
+        @source_files = source_files
+        @plugins = plugins
+
+        yield self if block_given?
+        define_task
+      end
+
+      private
+
+      def define_task
+        desc "Runs `stree write` over source files"
+        task(name) { run_task }
+      end
+
+      def run_task
+        arguments = ["write"]
+        arguments << "--plugins=#{plugins.join(",")}" if plugins.any?
+
+        SyntaxTree::CLI.run(arguments + Array(source_files))
+      end
+    end
+  end
+end

data/lib/syntax_tree/rake_tasks.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative "rake/check_task"
+require_relative "rake/write_task"

data/lib/syntax_tree/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SyntaxTree
-  VERSION = "2.4.0"
+  VERSION = "2.6.0"
 end