syntax_tree 2.7.1 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e90bde4f4aa60ce6784fd2c03234853bc281b7012cdb496d3f2bd97a1d825a9
-  data.tar.gz: 24c4a344934acc5fb0b8b7d8931f689f98185f00fd9b71726f5aeab2499dea98
+  metadata.gz: ad3c35843b6e3148499ac001d05a9c5d613d69debff5fdc411dff89a9adc5ca4
+  data.tar.gz: 570ae99edf8b17b5d205142872621b0551632d29ac4671ee14fed8415cf9cfe4
 SHA512:
-  metadata.gz: 70d191270dbe2efd07300169410e4e0674b7abd8ec1ed4d95e87f7bb986b790195f8366f27b77f38eeeb94de1a08ce3467d64fbb126910922857366389ce66c0
-  data.tar.gz: c3994813de2fa53eb132dce7e4d176e58a83071232bcdefe3ded77a3fdd35ed52639d99989878b4cf5ca3d13c0a5bc14cbdfb95555b9705e2dd08c88d2abf360
+  metadata.gz: 7ec0e9d8f5f4f828ba0d451d49bd7fa9f5576570f7427c91973503e795fe068e21a61dac4f1880be12dbd551855eb0b8d27f359a9a682b964f3183aaf687e1da
+  data.tar.gz: b8ae5c0132ebc445d060b1d5cef477253c724ea2c7a22a536d4ea78b3fb2d905c560e1f044f8843b92f79e0dbc1a434cda2a38e76b2732b426719150785a59af

data/CHANGELOG.md

@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [2.8.0] - 2022-06-21
+
+### Added
+
+- [#95](https://github.com/ruby-syntax-tree/syntax_tree/pull/95) - The `HeredocEnd` node has been added which effectively results in the ability to determine the location of the ending of a heredoc from source.
+- [#99](https://github.com/ruby-syntax-tree/syntax_tree/pull/99) - The LSP now allows you to pass the same configuration options as the other CLI commands which allows formatting to be modified in the VSCode extension.
+- [#100](https://github.com/ruby-syntax-tree/syntax_tree/pull/100) - The LSP now explicitly responds to the shutdown request so that VSCode never deadlocks.
+
+### Changed
+
+- [#96](https://github.com/ruby-syntax-tree/syntax_tree/pull/96) - The CLI now runs in parallel by default. There is a worker created for each processor on the running machine (as determined by `Etc.nprocessors`).
+- [#97](https://github.com/ruby-syntax-tree/syntax_tree/pull/97) - Syntax Tree now handles the case where `DidYouMean` is not available for whatever reason, as well as handles the newer `detailed_message` API for errors.
+
 ## [2.7.1] - 2022-05-25
 
 ### Added
@@ -259,7 +272,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...HEAD
+[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...HEAD
+[2.8.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...v2.8.0
 [2.7.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...v2.7.1
 [2.7.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...v2.7.0
 [2.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.5.0...v2.6.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    syntax_tree (2.7.1)
+    syntax_tree (2.8.0)
       prettier_print
 
 GEM
@@ -9,25 +9,25 @@ GEM
   specs:
     ast (2.4.2)
     docile (1.4.0)
-    minitest (5.15.0)
+    minitest (5.16.0)
     parallel (1.22.1)
     parser (3.1.2.0)
       ast (~> 2.4.1)
     prettier_print (0.1.0)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.4.0)
+    regexp_parser (2.5.0)
     rexml (3.2.5)
-    rubocop (1.29.1)
+    rubocop (1.30.1)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.17.0, < 2.0)
+      rubocop-ast (>= 1.18.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.17.0)
+    rubocop-ast (1.18.0)
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     simplecov (0.21.2)

data/lib/syntax_tree/basic_visitor.rb

@@ -33,7 +33,11 @@ module SyntaxTree
           ).correct(visit_method)
       end
 
-      DidYouMean.correct_error(VisitMethodError, self)
+      # In some setups with Ruby you can turn off DidYouMean, so we're going to
+      # respect that setting here.
+      if defined?(DidYouMean) && DidYouMean.method_defined?(:correct_error)
+        DidYouMean.correct_error(VisitMethodError, self)
+      end
     end
 
     class << self

data/lib/syntax_tree/cli.rb

@@ -34,9 +34,41 @@ module SyntaxTree
       end
     end
 
+    # An item of work that corresponds to a file to be processed.
+    class FileItem
+      attr_reader :filepath
+
+      def initialize(filepath)
+        @filepath = filepath
+      end
+
+      def handler
+        HANDLERS[File.extname(filepath)]
+      end
+
+      def source
+        handler.read(filepath)
+      end
+    end
+
+    # An item of work that corresponds to the stdin content.
+    class STDINItem
+      def handler
+        HANDLERS[".rb"]
+      end
+
+      def filepath
+        :stdin
+      end
+
+      def source
+        $stdin.read
+      end
+    end
+
     # The parent action class for the CLI that implements the basics.
     class Action
-      def run(handler, filepath, source)
+      def run(item)
       end
 
       def success
@@ -48,8 +80,8 @@ module SyntaxTree
 
     # An action of the CLI that prints out the AST for the given source.
     class AST < Action
-      def run(handler, _filepath, source)
-        pp handler.parse(source)
+      def run(item)
+        pp item.handler.parse(item.source)
       end
     end
 
@@ -59,10 +91,11 @@ module SyntaxTree
       class UnformattedError < StandardError
       end
 
-      def run(handler, filepath, source)
-        raise UnformattedError if source != handler.format(source)
+      def run(item)
+        source = item.source
+        raise UnformattedError if source != item.handler.format(source)
       rescue StandardError
-        warn("[#{Color.yellow("warn")}] #{filepath}")
+        warn("[#{Color.yellow("warn")}] #{item.filepath}")
         raise
       end
 
@@ -81,9 +114,11 @@ module SyntaxTree
       class NonIdempotentFormatError < StandardError
       end
 
-      def run(handler, filepath, source)
-        warning = "[#{Color.yellow("warn")}] #{filepath}"
-        formatted = handler.format(source)
+      def run(item)
+        handler = item.handler
+
+        warning = "[#{Color.yellow("warn")}] #{item.filepath}"
+        formatted = handler.format(item.source)
 
         raise NonIdempotentFormatError if formatted != handler.format(formatted)
       rescue StandardError
@@ -102,25 +137,27 @@ module SyntaxTree
 
     # An action of the CLI that prints out the doc tree IR for the given source.
     class Doc < Action
-      def run(handler, _filepath, source)
+      def run(item)
+        source = item.source
+
         formatter = Formatter.new(source, [])
-        handler.parse(source).format(formatter)
+        item.handler.parse(source).format(formatter)
         pp formatter.groups.first
       end
     end
 
     # An action of the CLI that formats the input source and prints it out.
     class Format < Action
-      def run(handler, _filepath, source)
-        puts handler.format(source)
+      def run(item)
+        puts item.handler.format(item.source)
       end
     end
 
     # An action of the CLI that converts the source into its equivalent JSON
     # representation.
     class Json < Action
-      def run(handler, _filepath, source)
-        object = Visitor::JSONVisitor.new.visit(handler.parse(source))
+      def run(item)
+        object = Visitor::JSONVisitor.new.visit(item.handler.parse(item.source))
         puts JSON.pretty_generate(object)
       end
     end
@@ -128,27 +165,28 @@ module SyntaxTree
     # An action of the CLI that outputs a pattern-matching Ruby expression that
     # would match the input given.
     class Match < Action
-      def run(handler, _filepath, source)
-        puts handler.parse(source).construct_keys
+      def run(item)
+        puts item.handler.parse(item.source).construct_keys
       end
     end
 
     # An action of the CLI that formats the input source and writes the
     # formatted output back to the file.
     class Write < Action
-      def run(handler, filepath, source)
-        print filepath
+      def run(item)
+        filepath = item.filepath
         start = Time.now
 
-        formatted = handler.format(source)
+        source = item.source
+        formatted = item.handler.format(source)
         File.write(filepath, formatted) if filepath != :stdin
 
         color = source == formatted ? Color.gray(filepath) : filepath
         delta = ((Time.now - start) * 1000).round
 
-        puts "\r#{color} #{delta}ms"
+        puts "#{color} #{delta}ms"
       rescue StandardError
-        puts "\r#{filepath}"
+        puts filepath
         raise
       end
     end
@@ -180,7 +218,7 @@ module SyntaxTree
       #{Color.bold("stree help")}
         Display this help message
 
-      #{Color.bold("stree lsp")}
+      #{Color.bold("stree lsp [OPTIONS]")}
         Run syntax tree in language server mode
 
       #{Color.bold("stree version")}
@@ -201,6 +239,20 @@ module SyntaxTree
       def run(argv)
         name, *arguments = argv
 
+        # If there are any plugins specified on the command line, then load them
+        # by requiring them here. We do this by transforming something like
+        #
+        #     stree format --plugins=haml template.haml
+        #
+        # into
+        #
+        #     require "syntax_tree/haml"
+        #
+        if arguments.first&.start_with?("--plugins=")
+          plugins = arguments.shift[/^--plugins=(.*)$/, 1]
+          plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" }
+        end
+
         case name
         when "help"
           puts HELP
@@ -244,38 +296,41 @@ module SyntaxTree
           return 1
         end
 
-        # If there are any plugins specified on the command line, then load them
-        # by requiring them here. We do this by transforming something like
-        #
-        #     stree format --plugins=haml template.haml
-        #
-        # into
-        #
-        #     require "syntax_tree/haml"
-        #
-        if arguments.first&.start_with?("--plugins=")
-          plugins = arguments.shift[/^--plugins=(.*)$/, 1]
-          plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" }
-        end
+        # We're going to build up a queue of items to process.
+        queue = Queue.new
 
-        # Track whether or not there are any errors from any of the files that
-        # we take action on so that we can properly clean up and exit.
-        errored = false
-
-        each_file(arguments) do |handler, filepath, source|
-          action.run(handler, filepath, source)
-        rescue Parser::ParseError => error
-          warn("Error: #{error.message}")
-          highlight_error(error, source)
-          errored = true
-        rescue Check::UnformattedError, Debug::NonIdempotentFormatError
-          errored = true
-        rescue StandardError => error
-          warn(error.message)
-          warn(error.backtrace)
-          errored = true
+        # If we're reading from stdin, then we'll just add the stdin object to
+        # the queue. Otherwise, we'll add each of the filepaths to the queue.
+        if $stdin.tty? || arguments.any?
+          arguments.each do |pattern|
+            Dir
+              .glob(pattern)
+              .each do |filepath|
+                queue << FileItem.new(filepath) if File.file?(filepath)
+              end
+          end
+        else
+          queue << STDINItem.new
         end
 
+        # At the end, we're going to return whether or not this worker ever
+        # encountered an error.
+        errored =
+          with_workers(queue) do |item|
+            action.run(item)
+            false
+          rescue Parser::ParseError => error
+            warn("Error: #{error.message}")
+            highlight_error(error, item.source)
+            true
+          rescue Check::UnformattedError, Debug::NonIdempotentFormatError
+            true
+          rescue StandardError => error
+            warn(error.message)
+            warn(error.backtrace)
+            true
+          end
+
         if errored
           action.failure
           1
@@ -287,22 +342,33 @@ module SyntaxTree
 
       private
 
-      def each_file(arguments)
-        if $stdin.tty? || arguments.any?
-          arguments.each do |pattern|
-            Dir
-              .glob(pattern)
-              .each do |filepath|
-                next unless File.file?(filepath)
-
-                handler = HANDLERS[File.extname(filepath)]
-                source = handler.read(filepath)
-                yield handler, filepath, source
-              end
+      def with_workers(queue)
+        # If the queue is just 1 item, then we're not going to bother going
+        # through the whole ceremony of parallelizing the work.
+        return yield queue.shift if queue.size == 1
+
+        workers =
+          Etc.nprocessors.times.map do
+            Thread.new do
+              # Propagate errors in the worker threads up to the parent thread.
+              Thread.current.abort_on_exception = true
+
+              # Track whether or not there are any errors from any of the files
+              # that we take action on so that we can properly clean up and
+              # exit.
+              errored = false
+
+              # While there is still work left to do, shift off the queue and
+              # process the item.
+              (errored ||= yield queue.shift) until queue.empty?
+
+              # At the end, we're going to return whether or not this worker
+              # ever encountered an error.
+              errored
+            end
           end
-        else
-          yield HANDLERS[".rb"], :stdin, $stdin.read
-        end
+
+        workers.inject(false) { |accum, thread| accum || thread.value }
       end
 
       # Highlights a snippet from a source and parse error.

data/lib/syntax_tree/language_server.rb

@@ -36,8 +36,9 @@ module SyntaxTree
           write(id: id, result: { capabilities: capabilities })
         in method: "initialized"
           # ignored
-        in method: "shutdown"
+        in method: "shutdown" # tolerate missing ID to be a good citizen
           store.clear
+          write(id: request[:id], result: {})
           return
         in {
              method: "textDocument/didChange",

data/lib/syntax_tree/node.rb

@@ -4813,7 +4813,7 @@ module SyntaxTree
     # [HeredocBeg] the opening of the heredoc
     attr_reader :beginning
 
-    # [String] the ending of the heredoc
+    # [HeredocEnd] the ending of the heredoc
     attr_reader :ending
 
     # [Integer] how far to dedent the heredoc
@@ -4847,7 +4847,7 @@ module SyntaxTree
     end
 
     def child_nodes
-      [beginning, *parts]
+      [beginning, *parts, ending]
     end
 
     alias deconstruct child_nodes
@@ -4883,7 +4883,7 @@ module SyntaxTree
               end
             end
 
-            q.text(ending)
+            q.format(ending)
           end
         end
       end
@@ -4929,6 +4929,45 @@ module SyntaxTree
     end
   end
 
+  # HeredocEnd represents the closing declaration of a heredoc.
+  #
+  #     <<~DOC
+  #       contents
+  #     DOC
+  #
+  # In the example above the HeredocEnd node represents the closing DOC.
+  class HeredocEnd < Node
+    # [String] the closing declaration of the heredoc
+    attr_reader :value
+
+    # [Array[ Comment | EmbDoc ]] the comments attached to this node
+    attr_reader :comments
+
+    def initialize(value:, location:, comments: [])
+      @value = value
+      @location = location
+      @comments = comments
+    end
+
+    def accept(visitor)
+      visitor.visit_heredoc_end(self)
+    end
+
+    def child_nodes
+      []
+    end
+
+    alias deconstruct child_nodes
+
+    def deconstruct_keys(_keys)
+      { value: value, location: location, comments: comments }
+    end
+
+    def format(q)
+      q.text(value)
+    end
+  end
+
   # HshPtn represents matching against a hash pattern using the Ruby 2.7+
   # pattern matching syntax.
   #

data/lib/syntax_tree/parser.rb

@@ -1640,9 +1640,19 @@ module SyntaxTree
     def on_heredoc_end(value)
       heredoc = @heredocs[-1]
 
+      location =
+      Location.token(
+        line: lineno,
+        char: char_pos,
+        column: current_column,
+        size: value.size + 1
+      )
+
+      heredoc_end = HeredocEnd.new(value: value.chomp, location: location)
+
       @heredocs[-1] = Heredoc.new(
         beginning: heredoc.beginning,
-        ending: value.chomp,
+        ending: heredoc_end,
         dedent: heredoc.dedent,
         parts: heredoc.parts,
         location:

data/lib/syntax_tree/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SyntaxTree
-  VERSION = "2.7.1"
+  VERSION = "2.8.0"
 end

data/lib/syntax_tree/visitor/field_visitor.rb

@@ -497,6 +497,10 @@ module SyntaxTree
         visit_token(node, "heredoc_beg")
       end
 
+      def visit_heredoc_end(node)
+        visit_token(node, "heredoc_end")
+      end
+
       def visit_hshptn(node)
         node(node, "hshptn") do
           field("constant", node.constant) if node.constant

data/lib/syntax_tree/visitor.rb

@@ -194,6 +194,9 @@ module SyntaxTree
     # Visit a HeredocBeg node.
     alias visit_heredoc_beg visit_child_nodes
 
+    # Visit a HeredocEnd node.
+    alias visit_heredoc_end visit_child_nodes
+
     # Visit a HshPtn node.
     alias visit_hshptn visit_child_nodes
 

data/lib/syntax_tree.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "etc"
 require "json"
 require "pp"
 require "prettier_print"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: syntax_tree
 version: !ruby/object:Gem::Version
-  version: 2.7.1
+  version: 2.8.0
 platform: ruby
 authors:
 - Kevin Newton
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-25 00:00:00.000000000 Z
+date: 2022-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: prettier_print
@@ -160,7 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.0.dev
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: A parser based on ripper