syntax_tree 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92ff69f19d5433566404bce6888eae08a442861821612c986489db05679ecd22
-  data.tar.gz: e05511cf866fa1e0cfde540f0a0bc84a88414b128583a37a68c73658ff667b16
+  metadata.gz: 52707e3169fedfce017259222e4740b2e1032d210dfb628d454134eef578a7b6
+  data.tar.gz: aed3d5b3e6de7ed1e156c5b104007db6a01dab65e0ee80913afa2c1b5c63a5c0
 SHA512:
-  metadata.gz: 3d98f8b9a997ef9278ad2ea9c1600ee5abc52c646d157b0180f6cf6956ebde2ccc6d9ea917436231645a7d4bf58ce1627fbae393b89e81b683157bc5263d65e0
-  data.tar.gz: 924ccb54dec3055dca88aef5efe839d2c979abedfdf96f5e0996e18390b3e246b676cdb1c8aba6d51edfed42219bff3c80b4bfcead0eb3f6b926e585c96f0a1b
+  metadata.gz: b81b4753a5e765712bd31081dedca6e50c67d04a7935a717fc56d278424154e6ed95b648ac1cbdd59b500ee44c2d8d89f57f52e3851f9615e378d8c9c2c6abe1
+  data.tar.gz: 32c95446f19190b8e73d9c3ce83ff4bd5573f5787f844cf695551a2d32c0c159af8c514714c4648a99be4e66189910bde7f01557ea973f6e17aaba66ace7fce2

data/.rubocop.yml

@@ -44,7 +44,7 @@ Style/ExplicitBlockArgument:
   Enabled: false
 
 Style/FormatString:
-  EnforcedStyle: percent
+  Enabled: false
 
 Style/GuardClause:
   Enabled: false

data/CHANGELOG.md

@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [3.4.0] - 2022-08-19
+
+### Added
+
+- [#127](https://github.com/ruby-syntax-tree/syntax_tree/pull/127) - Allow the language server to handle other file extensions if it is activated for those extensions.
+- [#133](https://github.com/ruby-syntax-tree/syntax_tree/pull/133) - Add documentation on supporting vim and neovim.
+
+### Changed
+
+- [#132](https://github.com/ruby-syntax-tree/syntax_tree/pull/132) - Provide better error messages when end quotes and end keywords are missing from tokens.
+- [#134](https://github.com/ruby-syntax-tree/syntax_tree/pull/134) - Ensure the correct `end` keyword is getting removed by `begin..rescue` clauses.
+- [#137](https://github.com/ruby-syntax-tree/syntax_tree/pull/137) - Better support regular expressions with no ending token.
+
 ## [3.3.0] - 2022-08-02
 
 ### Added
@@ -319,7 +332,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.3.0...HEAD
+[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.4.0...HEAD
+[3.4.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.3.0...v3.4.0
 [3.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.1...v3.3.0
 [3.2.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.2.0...v3.2.1
 [3.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v3.1.0...v3.2.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    syntax_tree (3.3.0)
+    syntax_tree (3.4.0)
       prettier_print
 
 GEM
@@ -12,24 +12,24 @@ GEM
     json (2.6.2)
     minitest (5.16.2)
     parallel (1.22.1)
-    parser (3.1.2.0)
+    parser (3.1.2.1)
       ast (~> 2.4.1)
     prettier_print (0.1.0)
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.5.0)
     rexml (3.2.5)
-    rubocop (1.32.0)
+    rubocop (1.35.0)
       json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 3.1.0.0)
+      parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.19.1, < 2.0)
+      rubocop-ast (>= 1.20.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.19.1)
+    rubocop-ast (1.21.0)
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     simplecov (0.21.2)

data/README.md

@@ -20,6 +20,7 @@ It is built with only standard library dependencies. It additionally ships with
   - [match](#match)
   - [write](#write)
   - [Configuration](#configuration)
+  - [Globbing](#globbing)
 - [Library](#library)
   - [SyntaxTree.read(filepath)](#syntaxtreereadfilepath)
   - [SyntaxTree.parse(source)](#syntaxtreeparsesource)
@@ -44,7 +45,7 @@ It is built with only standard library dependencies. It additionally ships with
 - [Integration](#integration)
   - [Rake](#rake)
   - [RuboCop](#rubocop)
-  - [VSCode](#vscode)
+  - [Editors](#editors)
 - [Contributing](#contributing)
 - [License](#license)
 
@@ -245,6 +246,24 @@ This should be a text file with each argument on a separate line.
 
 If this file is present, it will _always_ be used for CLI commands. You can also pass options from the command line as in the examples above. The options in the `.streerc` file are passed to the CLI first, then the arguments from the command line. In the case of exclusive options (e.g. `--print-width`), this means that the command line options override what's in the config file. In the case of options that can take multiple inputs (e.g. `--plugins`), the effect is additive. That is, the plugins passed from the command line will be loaded _in addition to_ the plugins in the config file.
 
+### Globbing
+
+When running commands with `stree`, it's common to pass in lists of files. For example:
+
+```sh
+stree write 'lib/*.rb' 'test/*.rb'
+```
+
+The commands in the CLI accept any number of arguments. This means you _could_ pass `**/*.rb` (note the lack of quotes). This would make your shell expand out the file paths listed according to its own rules. (For example, [here](https://www.gnu.org/software/bash/manual/html_node/Filename-Expansion.html) are the rules for GNU bash.)
+
+However, it's recommended to instead use quotes, which means that Ruby is responsible for performing the file path expansion instead. This ensures a consistent experience across different environments and shells. The globs must follow the Ruby-specific globbing syntax as specified in the documentation for [Dir](https://ruby-doc.org/core-3.1.1/Dir.html#method-c-glob).
+
+Baked into this syntax is the ability to provide exceptions to file name patterns as well. For example, if you are in a Rails app and want to exclude files named `schema.rb` but write all other Ruby files, you can use the following syntax:
+
+```shell
+stree write "**/{[!schema]*,*}.rb"
+```
+
 ## Library
 
 Syntax Tree can be used as a library to access the syntax tree underlying Ruby source code.
@@ -548,9 +567,11 @@ inherit_gem:
   syntax_tree: config/rubocop.yml
 ```
 
-### VSCode
+### Editors
 
-To integrate Syntax Tree into VSCode, you should use the official VSCode extension [ruby-syntax-tree/vscode-syntax-tree](https://github.com/ruby-syntax-tree/vscode-syntax-tree).
+* [Neovim](https://neovim.io/) - [neovim/nvim-lspconfig](https://github.com/neovim/nvim-lspconfig).
+* [Vim](https://www.vim.org/) - [dense-analysis/ale](https://github.com/dense-analysis/ale).
+* [VSCode](https://code.visualstudio.com/) - [ruby-syntax-tree/vscode-syntax-tree](https://github.com/ruby-syntax-tree/vscode-syntax-tree).
 
 ## Contributing
 

data/lib/syntax_tree/cli.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
+require "optparse"
+
 module SyntaxTree
   # Syntax Tree ships with the `stree` CLI, which can be used to inspect and
   # manipulate Ruby code. This module is responsible for powering that CLI.
   module CLI
-    CONFIG_FILE = ".streerc"
-
     # A utility wrapper around colored strings in the output.
     class Color
       attr_reader :value, :code
@@ -70,6 +70,12 @@ module SyntaxTree
 
     # The parent action class for the CLI that implements the basics.
     class Action
+      attr_reader :options
+
+      def initialize(options)
+        @options = options
+      end
+
       def run(item)
       end
 
@@ -93,15 +99,9 @@ module SyntaxTree
       class UnformattedError < StandardError
       end
 
-      attr_reader :print_width
-
-      def initialize(print_width:)
-        @print_width = print_width
-      end
-
       def run(item)
         source = item.source
-        if source != item.handler.format(source, print_width)
+        if source != item.handler.format(source, options.print_width)
           raise UnformattedError
         end
       rescue StandardError
@@ -124,19 +124,13 @@ module SyntaxTree
       class NonIdempotentFormatError < StandardError
       end
 
-      attr_reader :print_width
-
-      def initialize(print_width:)
-        @print_width = print_width
-      end
-
       def run(item)
         handler = item.handler
 
         warning = "[#{Color.yellow("warn")}] #{item.filepath}"
-        formatted = handler.format(item.source, print_width)
+        formatted = handler.format(item.source, options.print_width)
 
-        if formatted != handler.format(formatted, print_width)
+        if formatted != handler.format(formatted, options.print_width)
           raise NonIdempotentFormatError
         end
       rescue StandardError
@@ -166,14 +160,8 @@ module SyntaxTree
 
     # An action of the CLI that formats the input source and prints it out.
     class Format < Action
-      attr_reader :print_width
-
-      def initialize(print_width:)
-        @print_width = print_width
-      end
-
       def run(item)
-        puts item.handler.format(item.source, print_width)
+        puts item.handler.format(item.source, options.print_width)
       end
     end
 
@@ -197,18 +185,12 @@ module SyntaxTree
     # An action of the CLI that formats the input source and writes the
     # formatted output back to the file.
     class Write < Action
-      attr_reader :print_width
-
-      def initialize(print_width:)
-        @print_width = print_width
-      end
-
       def run(item)
         filepath = item.filepath
         start = Time.now
 
         source = item.source
-        formatted = item.handler.format(source, print_width)
+        formatted = item.handler.format(source, options.print_width)
         File.write(filepath, formatted) if filepath != :stdin
 
         color = source == formatted ? Color.gray(filepath) : filepath
@@ -264,74 +246,114 @@ module SyntaxTree
         The maximum line width to use when formatting.
     HELP
 
+    # This represents all of the options that can be passed to the CLI. It is
+    # responsible for parsing the list and then returning the file paths at the
+    # end.
+    class Options
+      attr_reader :print_width
+
+      def initialize(print_width: DEFAULT_PRINT_WIDTH)
+        @print_width = print_width
+      end
+
+      def parse(arguments)
+        parser.parse(arguments)
+      end
+
+      private
+
+      def parser
+        OptionParser.new do |opts|
+          # If there are any plugins specified on the command line, then load
+          # them by requiring them here. We do this by transforming something
+          # like
+          #
+          #     stree format --plugins=haml template.haml
+          #
+          # into
+          #
+          #     require "syntax_tree/haml"
+          #
+          opts.on("--plugins=PLUGINS") do |plugins|
+            plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" }
+          end
+
+          # If there is a print width specified on the command line, then
+          # parse that out here and use it when formatting.
+          opts.on("--print-width=NUMBER", Integer) do |print_width|
+            @print_width = print_width
+          end
+        end
+      end
+    end
+
+    # We allow a minimal configuration file to act as additional command line
+    # arguments to the CLI. Each line of the config file should be a new
+    # argument, as in:
+    #
+    #     --plugins=plugin/single_quote
+    #     --print-width=100
+    #
+    # When invoking the CLI, we will read this config file and then parse it if
+    # it exists in the current working directory.
+    class ConfigFile
+      FILENAME = ".streerc"
+
+      attr_reader :filepath
+
+      def initialize
+        @filepath = File.join(Dir.pwd, FILENAME)
+      end
+
+      def exists?
+        File.readable?(filepath)
+      end
+
+      def arguments
+        exists? ? File.readlines(filepath, chomp: true) : []
+      end
+    end
+
     class << self
       # Run the CLI over the given array of strings that make up the arguments
       # passed to the invocation.
       def run(argv)
         name, *arguments = argv
-        print_width = DEFAULT_PRINT_WIDTH
-
-        config_file = File.join(Dir.pwd, CONFIG_FILE)
-        if File.readable?(config_file)
-          arguments.unshift(*File.readlines(config_file, chomp: true))
-        end
 
-        while arguments.first&.start_with?("--")
-          case (argument = arguments.shift)
-          when /^--plugins=(.+)$/
-            # If there are any plugins specified on the command line, then load
-            # them by requiring them here. We do this by transforming something
-            # like
-            #
-            #     stree format --plugins=haml template.haml
-            #
-            # into
-            #
-            #     require "syntax_tree/haml"
-            #
-            $1.split(",").each { |plugin| require "syntax_tree/#{plugin}" }
-          when /^--print-width=(\d+)$/
-            # If there is a print width specified on the command line, then
-            # parse that out here and use it when formatting.
-            print_width = Integer($1)
-          else
-            warn("Unknown CLI option: #{argument}")
-            warn(HELP)
-            return 1
-          end
-        end
+        config_file = ConfigFile.new
+        arguments.unshift(*config_file.arguments)
 
-        case name
-        when "help"
-          puts HELP
-          return 0
-        when "lsp"
-          require "syntax_tree/language_server"
-          LanguageServer.new(print_width: print_width).run
-          return 0
-        when "version"
-          puts SyntaxTree::VERSION
-          return 0
-        end
+        options = Options.new
+        options.parse(arguments)
 
         action =
           case name
           when "a", "ast"
-            AST.new
+            AST.new(options)
           when "c", "check"
-            Check.new(print_width: print_width)
+            Check.new(options)
           when "debug"
-            Debug.new(print_width: print_width)
+            Debug.new(options)
           when "doc"
-            Doc.new
+            Doc.new(options)
+          when "help"
+            puts HELP
+            return 0
           when "j", "json"
-            Json.new
+            Json.new(options)
+          when "lsp"
+            require "syntax_tree/language_server"
+            LanguageServer.new(print_width: options.print_width).run
+            return 0
           when "m", "match"
-            Match.new
+            Match.new(options)
           when "f", "format"
-            Format.new(print_width: print_width)
+            Format.new(options)
+          when "version"
+            puts SyntaxTree::VERSION
+            return 0
           when "w", "write"
-            Write.new(print_width: print_width)
+            Write.new(options)
           else
             warn(HELP)
             return 1

data/lib/syntax_tree/language_server.rb

@@ -56,7 +56,7 @@ module SyntaxTree
           store.delete(uri)
         in { method: "textDocument/formatting", id:, params: { textDocument: { uri: } } }
           contents = store[uri]
-          write(id: id, result: contents ? [format(store[uri])] : nil)
+          write(id: id, result: contents ? [format(store[uri], uri.split(".").last)] : nil)
         in { method: "textDocument/inlayHint", id:, params: { textDocument: { uri: } } }
           contents = store[uri]
           write(id: id, result: contents ? inlay_hints(store[uri]) : nil)
@@ -86,7 +86,9 @@ module SyntaxTree
       }
     end
 
-    def format(source)
+    def format(source, extension)
+      text = SyntaxTree::HANDLERS[".#{extension}"].format(source, print_width)
+
       {
         range: {
           start: {
@@ -98,7 +100,7 @@ module SyntaxTree
             character: 0
           }
         },
-        newText: SyntaxTree.format(source, print_width)
+        newText: text
       }
     end
 
@@ -117,5 +119,9 @@ module SyntaxTree
       output.print("Content-Length: #{response.bytesize}\r\n\r\n#{response}")
       output.flush
     end
+
+    def log(message)
+      write(method: "window/logMessage", params: { type: 4, message: message })
+    end
   end
 end

data/lib/syntax_tree/parser.rb

@@ -57,6 +57,26 @@ module SyntaxTree
       end
     end
 
+    # This represents all of the tokens coming back from the lexer. It is
+    # replacing a simple array because it keeps track of the last deleted token
+    # from the list for better error messages.
+    class TokenList < SimpleDelegator
+      attr_reader :last_deleted
+
+      def initialize(object)
+        super
+        @last_deleted = nil
+      end
+
+      def delete(value)
+        @last_deleted = super || @last_deleted
+      end
+
+      def delete_at(index)
+        @last_deleted = super
+      end
+    end
+
     # [String] the source being parsed
     attr_reader :source
 
@@ -124,7 +144,7 @@ module SyntaxTree
       # Most of the time, when a parser event consumes one of these events, it
       # will be deleted from the list. So ideally, this list stays pretty short
       # over the course of parsing a source string.
-      @tokens = []
+      @tokens = TokenList.new([])
 
       # Here we're going to build up a list of SingleByteString or
       # MultiByteString objects. They're each going to represent a string in the
@@ -174,6 +194,33 @@ module SyntaxTree
       line[column].to_i - line.start
     end
 
+    # Returns the current location that is being looked at for the parser for
+    # the purpose of locating the error.
+    def find_token_error(location)
+      if location
+        # If we explicitly passed a location into this find_token_error method,
+        # that means that's the source of the error, so we'll use that
+        # information for our error object.
+        lineno = location.start_line
+        [lineno, location.start_char - line_counts[lineno - 1].start]
+      elsif lineno && column
+        # If there is a line number associated with the current ripper state,
+        # then we'll use that information to generate the error.
+        [lineno, column]
+      elsif (location = tokens.last_deleted&.location)
+        # If we've already deleted a token from the list of tokens that we are
+        # consuming, then we'll fall back to that token's location.
+        lineno = location.start_line
+        [lineno, location.start_char - line_counts[lineno - 1].start]
+      else
+        # Finally, it's possible that when we hit this error the parsing thread
+        # for ripper has died. In that case, lineno and column both return nil.
+        # So we're just going to set it to line 1, column 0 in the hopes that
+        # that makes any sense.
+        [1, 0]
+      end
+    end
+
     # As we build up a list of tokens, we'll periodically need to go backwards
     # and find the ones that we've already hit in order to determine the
     # location information for nodes that use them. For example, if you have a
@@ -201,14 +248,7 @@ module SyntaxTree
         unless index
           token = value == :any ? type.name.split("::", 2).last : value
           message = "Cannot find expected #{token}"
-
-          if location
-            lineno = location.start_line
-            column = location.start_char - line_counts[lineno - 1].start
-            raise ParseError.new(message, lineno, column)
-          else
-            raise ParseError.new(message, lineno, column)
-          end
+          raise ParseError.new(message, *find_token_error(location))
         end
 
         tokens.delete_at(index)
@@ -677,8 +717,7 @@ module SyntaxTree
       else
         keyword = find_token(Kw, "begin")
         end_location =
-          if bodystmt.rescue_clause || bodystmt.ensure_clause ||
-               bodystmt.else_clause
+          if bodystmt.else_clause
             bodystmt.location
           else
             find_token(Kw, "end").location
@@ -2798,14 +2837,21 @@ module SyntaxTree
     # :call-seq:
     #   on_regexp_literal: (
     #     RegexpContent regexp_content,
-    #     RegexpEnd ending
+    #     (nil | RegexpEnd) ending
     #   ) -> RegexpLiteral
     def on_regexp_literal(regexp_content, ending)
+      location = regexp_content.location
+
+      if ending.nil?
+        message = "Cannot find expected regular expression ending"
+        raise ParseError.new(message, *find_token_error(location))
+      end
+
       RegexpLiteral.new(
         beginning: regexp_content.beginning,
         ending: ending.value,
         parts: regexp_content.parts,
-        location: regexp_content.location.to(ending.location)
+        location: location.to(ending.location)
       )
     end
 

data/lib/syntax_tree/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SyntaxTree
-  VERSION = "3.3.0"
+  VERSION = "3.4.0"
 end

data/lib/syntax_tree.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "delegate"
 require "etc"
 require "json"
 require "pp"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: syntax_tree
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Kevin Newton
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-02 00:00:00.000000000 Z
+date: 2022-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: prettier_print