syntax_tree 6.0.1 → 6.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b1e39c546fb96aea864e8190b8a8ff338543afd644d2334edad5b98663d418f2
-  data.tar.gz: 7633b68e7b42fcd782fc1c89395d93679e72828fe1a9214e741ad00718543c6b
+  metadata.gz: 7515a827509d352259e6cae476930093fe382fcc35c248fba3ab716019e69ca9
+  data.tar.gz: add2625387abe5616f90d28843f4fcebac5b68222f9a6258cbfb04fbd41b4772
 SHA512:
-  metadata.gz: a0990a769cbc448e549d14542610ccfbf09536ada139cc10701871a506a1b160a8ce751712a94a8f003d3b2c1828a6b5d4bd5384efe63f0f82a21c0762b689d5
-  data.tar.gz: a08b4057370c230bfd713ff9a717ea4f2a02be4f8610af51b18143ff9b2b9e542541b77c049488c500033751b50adc279d5e3a8beb0568f5e2bb165793c612d4
+  metadata.gz: ea0a666bd65180facaab89acd7b7bd87d3f5208c69a0fe80b91d1d25d57cdc2c281ac301070da8d73c7d119574a7f7de30c0facb0d7131f895bbc8fa7fe582c5
+  data.tar.gz: 4c9df489dcb876280a219a8e213a800f0544a0aaa02a303d8fc3ba948e07e195dbfb21da8f8f04825abc4bf61fb9ec92141d5a00cc7b3d53b1d6103d58d10be3

data/.gitignore

@@ -4,6 +4,7 @@
 /coverage/
 /pkg/
 /rdocs/
+/sorbet/
 /spec/reports/
 /tmp/
 /vendor/

data/.rubocop.yml

@@ -7,7 +7,7 @@ AllCops:
   SuggestExtensions: false
   TargetRubyVersion: 2.7
   Exclude:
-  - '{.git,.github,bin,coverage,pkg,spec,test/fixtures,vendor,tmp}/**/*'
+  - '{.git,.github,bin,coverage,pkg,sorbet,spec,test/fixtures,vendor,tmp}/**/*'
   - test.rb
 
 Gemspec/DevelopmentDependencies:

data/.ruby-version

@@ -0,0 +1 @@
+3.2.0

data/CHANGELOG.md

@@ -6,6 +6,31 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [6.1.0] - 2023-03-20
+
+### Added
+
+- The `stree ctags` command for generating ctags like `universal-ctags` or `ripper-tags` would.
+- The `definedivar` YARV instruction has been added to match CRuby's implementation.
+- We now generate better Sorbet RBI files for the nodes in the tree and the visitors.
+- `SyntaxTree::Reflection.nodes` now includes the visitor method.
+
+### Changed
+
+- We now explicitly require `pp` in environments that need it.
+
+## [6.0.2] - 2023-03-03
+
+### Added
+
+- The `WithScope` visitor mixin will now additionally report local variables defined through regular expression named captures.
+- The `WithScope` visitor mixin now properly handles destructured splat arguments in required positions.
+
+### Changed
+
+- Fixed the AST output by adding blocks to `Command` and `CommandCall` nodes in the `FieldVisitor`.
+- Fixed the location of lambda local variables (e.g., `->(; a) {}`).
+
 ## [6.0.1] - 2023-02-26
 
 ### Added
@@ -572,7 +597,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.1...HEAD
+[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.2...HEAD
+[6.0.2]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.1...v6.0.2
 [6.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v6.0.0...v6.0.1
 [6.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.3.0...v6.0.0
 [5.3.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...v5.3.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    syntax_tree (6.0.1)
+    syntax_tree (6.1.0)
       prettier_print (>= 1.2.0)
 
 GEM
@@ -10,16 +10,16 @@ GEM
     ast (2.4.2)
     docile (1.4.0)
     json (2.6.3)
-    minitest (5.17.0)
+    minitest (5.18.0)
     parallel (1.22.1)
-    parser (3.2.1.0)
+    parser (3.2.1.1)
       ast (~> 2.4.1)
-    prettier_print (1.2.0)
+    prettier_print (1.2.1)
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.7.0)
     rexml (3.2.5)
-    rubocop (1.46.0)
+    rubocop (1.48.1)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.2.0.0)
@@ -29,9 +29,9 @@ GEM
       rubocop-ast (>= 1.26.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.26.0)
+    rubocop-ast (1.27.0)
       parser (>= 3.2.1.0)
-    ruby-progressbar (1.11.0)
+    ruby-progressbar (1.13.0)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)

data/README.md

@@ -15,6 +15,7 @@ It is built with only standard library dependencies. It additionally ships with
 - [CLI](#cli)
   - [ast](#ast)
   - [check](#check)
+  - [ctags](#ctags)
   - [expr](#expr)
   - [format](#format)
   - [json](#json)
@@ -139,6 +140,33 @@ To change the print width that you are checking against, specify the `--print-wi
 stree check --print-width=100 path/to/file.rb
 ```
 
+### ctags
+
+This command will output to stdout a set of tags suitable for usage with [ctags](https://github.com/universal-ctags/ctags).
+
+```sh
+stree ctags path/to/file.rb
+```
+
+For a file containing the following Ruby code:
+
+```ruby
+class Foo
+end
+
+class Bar < Foo
+end
+```
+
+you will receive:
+
+```
+!_TAG_FILE_FORMAT	2	/extended format; --format=1 will not append ;" to lines/
+!_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
+Bar	test.rb	/^class Bar < Foo$/;"	c	inherits:Foo
+Foo	test.rb	/^class Foo$/;"	c
+```
+
 ### expr
 
 This command will output a Ruby case-match expression that would match correctly against the first expression of the input.
@@ -788,6 +816,7 @@ inherit_gem:
 * [Neovim](https://neovim.io/) - [neovim/nvim-lspconfig](https://github.com/neovim/nvim-lspconfig).
 * [Vim](https://www.vim.org/) - [dense-analysis/ale](https://github.com/dense-analysis/ale).
 * [VSCode](https://code.visualstudio.com/) - [ruby-syntax-tree/vscode-syntax-tree](https://github.com/ruby-syntax-tree/vscode-syntax-tree).
+* [Emacs](https://www.gnu.org/software/emacs/) - [emacs-format-all-the-code](https://github.com/lassik/emacs-format-all-the-code).
 
 ## Contributing
 

data/lib/syntax_tree/cli.rb

@@ -154,6 +154,92 @@ module SyntaxTree
       end
     end
 
+    # An action of the CLI that generates ctags for the given source.
+    class CTags < Action
+      attr_reader :entries
+
+      def initialize(options)
+        super(options)
+        @entries = []
+      end
+
+      def run(item)
+        lines = item.source.lines(chomp: true)
+
+        SyntaxTree
+          .index(item.source)
+          .each do |entry|
+            line = lines[entry.location.line - 1]
+            pattern = "/^#{line.gsub("\\", "\\\\\\\\").gsub("/", "\\/")}$/;\""
+
+            entries << case entry
+            when SyntaxTree::Index::ModuleDefinition
+              parts = [entry.name, item.filepath, pattern, "m"]
+
+              if entry.nesting != [[entry.name]]
+                parts << "class:#{entry.nesting.flatten.tap(&:pop).join(".")}"
+              end
+
+              parts.join("\t")
+            when SyntaxTree::Index::ClassDefinition
+              parts = [entry.name, item.filepath, pattern, "c"]
+
+              if entry.nesting != [[entry.name]]
+                parts << "class:#{entry.nesting.flatten.tap(&:pop).join(".")}"
+              end
+
+              unless entry.superclass.empty?
+                inherits = entry.superclass.join(".").delete_prefix(".")
+                parts << "inherits:#{inherits}"
+              end
+
+              parts.join("\t")
+            when SyntaxTree::Index::MethodDefinition
+              parts = [entry.name, item.filepath, pattern, "f"]
+
+              unless entry.nesting.empty?
+                parts << "class:#{entry.nesting.flatten.join(".")}"
+              end
+
+              parts.join("\t")
+            when SyntaxTree::Index::SingletonMethodDefinition
+              parts = [entry.name, item.filepath, pattern, "F"]
+
+              unless entry.nesting.empty?
+                parts << "class:#{entry.nesting.flatten.join(".")}"
+              end
+
+              parts.join("\t")
+            when SyntaxTree::Index::AliasMethodDefinition
+              parts = [entry.name, item.filepath, pattern, "a"]
+
+              unless entry.nesting.empty?
+                parts << "class:#{entry.nesting.flatten.join(".")}"
+              end
+
+              parts.join("\t")
+            when SyntaxTree::Index::ConstantDefinition
+              parts = [entry.name, item.filepath, pattern, "C"]
+
+              unless entry.nesting.empty?
+                parts << "class:#{entry.nesting.flatten.join(".")}"
+              end
+
+              parts.join("\t")
+            end
+          end
+      end
+
+      def success
+        puts(<<~HEADER)
+          !_TAG_FILE_FORMAT	2	/extended format; --format=1 will not append ;" to lines/
+          !_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
+        HEADER
+
+        entries.sort.each { |entry| puts(entry) }
+      end
+    end
+
     # An action of the CLI that formats the source twice to check if the first
     # format is not idempotent.
     class Debug < Action
@@ -327,6 +413,9 @@ module SyntaxTree
       #{Color.bold("stree check [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")}
         Check that the given files are formatted as syntax tree would format them
 
+      #{Color.bold("stree ctags [-e SCRIPT] FILE")}
+        Print out a ctags-compatible index of the given files
+
       #{Color.bold("stree debug [--plugins=...] [--print-width=NUMBER] [-e SCRIPT] FILE")}
         Check that the given files can be formatted idempotently
 
@@ -488,6 +577,8 @@ module SyntaxTree
             AST.new(options)
           when "c", "check"
             Check.new(options)
+          when "ctags"
+            CTags.new(options)
           when "debug"
             Debug.new(options)
           when "doc"

data/lib/syntax_tree/field_visitor.rb

@@ -263,6 +263,7 @@ module SyntaxTree
         node(node, "command") do
           field("message", node.message)
           field("arguments", node.arguments)
+          field("block", node.block) if node.block
           comments(node)
         end
       end
@@ -273,6 +274,7 @@ module SyntaxTree
           field("operator", node.operator)
           field("message", node.message)
           field("arguments", node.arguments) if node.arguments
+          field("block", node.block) if node.block
           comments(node)
         end
       end

data/lib/syntax_tree/index.rb

@@ -31,6 +31,18 @@ module SyntaxTree
       end
     end
 
+    # This entry represents a constant assignment.
+    class ConstantDefinition
+      attr_reader :nesting, :name, :location, :comments
+
+      def initialize(nesting, name, location, comments)
+        @nesting = nesting
+        @name = name
+        @location = location
+        @comments = comments
+      end
+    end
+
     # This entry represents a module definition using the module keyword.
     class ModuleDefinition
       attr_reader :nesting, :name, :location, :comments
@@ -68,6 +80,19 @@ module SyntaxTree
       end
     end
 
+    # This entry represents a method definition that was created using the alias
+    # keyword.
+    class AliasMethodDefinition
+      attr_reader :nesting, :name, :location, :comments
+
+      def initialize(nesting, name, location, comments)
+        @nesting = nesting
+        @name = name
+        @location = location
+        @comments = comments
+      end
+    end
+
     # When you're using the instruction sequence backend, this class is used to
     # lazily parse comments out of the source code.
     class FileComments
@@ -178,7 +203,14 @@ module SyntaxTree
       end
 
       def find_constant_path(insns, index)
-        index -= 1 while insns[index].is_a?(Integer)
+        index -= 1 while index >= 0 &&
+          (
+            insns[index].is_a?(Integer) ||
+              (
+                insns[index].is_a?(Array) &&
+                  %i[swap topn].include?(insns[index][0])
+              )
+          )
         insn = insns[index]
 
         if insn.is_a?(Array) && insn[0] == :opt_getconstant_path
@@ -207,11 +239,43 @@ module SyntaxTree
         end
       end
 
+      def find_attr_arguments(insns, index)
+        orig_argc = insns[index][1][:orig_argc]
+        names = []
+
+        current = index - 1
+        while current >= 0 && names.length < orig_argc
+          if insns[current].is_a?(Array) && insns[current][0] == :putobject
+            names.unshift(insns[current][1])
+          end
+
+          current -= 1
+        end
+
+        names if insns[current] == [:putself] && names.length == orig_argc
+      end
+
+      def method_definition(nesting, name, location, file_comments)
+        comments = EntryComments.new(file_comments, location)
+
+        if nesting.last == [:singletonclass]
+          SingletonMethodDefinition.new(
+            nesting[0...-1],
+            name,
+            location,
+            comments
+          )
+        else
+          MethodDefinition.new(nesting, name, location, comments)
+        end
+      end
+
       def index_iseq(iseq, file_comments)
         results = []
         queue = [[iseq, []]]
 
         while (current_iseq, current_nesting = queue.shift)
+          file = current_iseq[5]
           line = current_iseq[8]
           insns = current_iseq[13]
 
@@ -246,8 +310,8 @@ module SyntaxTree
                   find_constant_path(insns, index - 1)
 
                 if superclass.empty?
-                  raise NotImplementedError,
-                        "superclass with non constant path on line #{line}"
+                  warn("#{file}:#{line}: superclass with non constant path")
+                  next
                 end
               end
 
@@ -265,8 +329,10 @@ module SyntaxTree
                 # defined on self. We could, but it would require more
                 # emulation.
                 if insns[index - 2] != [:putself]
-                  raise NotImplementedError,
-                        "singleton class with non-self receiver"
+                  warn(
+                    "#{file}:#{line}: singleton class with non-self receiver"
+                  )
+                  next
                 end
               elsif flags & VM_DEFINECLASS_TYPE_MODULE > 0
                 location = location_for(class_iseq)
@@ -290,16 +356,16 @@ module SyntaxTree
               queue << [class_iseq, next_nesting]
             when :definemethod
               location = location_for(insn[2])
-              results << MethodDefinition.new(
+              results << method_definition(
                 current_nesting,
                 insn[1],
                 location,
-                EntryComments.new(file_comments, location)
+                file_comments
               )
             when :definesmethod
-              if current_iseq[13][index - 1] != [:putself]
-                raise NotImplementedError,
-                      "singleton method with non-self receiver"
+              if insns[index - 1] != [:putself]
+                warn("#{file}:#{line}: singleton method with non-self receiver")
+                next
               end
 
               location = location_for(insn[2])
@@ -309,6 +375,69 @@ module SyntaxTree
                 location,
                 EntryComments.new(file_comments, location)
               )
+            when :setconstant
+              next_nesting = current_nesting.dup
+              name = insn[1]
+
+              _, nesting = find_constant_path(insns, index - 1)
+              next_nesting << nesting if nesting.any?
+
+              location = Location.new(line, :unknown)
+              results << ConstantDefinition.new(
+                next_nesting,
+                name,
+                location,
+                EntryComments.new(file_comments, location)
+              )
+            when :opt_send_without_block, :send
+              case insn[1][:mid]
+              when :attr_reader, :attr_writer, :attr_accessor
+                attr_names = find_attr_arguments(insns, index)
+                next unless attr_names
+
+                location = Location.new(line, :unknown)
+                attr_names.each do |attr_name|
+                  if insn[1][:mid] != :attr_writer
+                    results << method_definition(
+                      current_nesting,
+                      attr_name,
+                      location,
+                      file_comments
+                    )
+                  end
+
+                  if insn[1][:mid] != :attr_reader
+                    results << method_definition(
+                      current_nesting,
+                      :"#{attr_name}=",
+                      location,
+                      file_comments
+                    )
+                  end
+                end
+              when :"core#set_method_alias"
+                # Now we have to validate that the alias is happening with a
+                # non-interpolated value. To do this we'll match the specific
+                # pattern we're expecting.
+                values =
+                  insns[(index - 4)...index].map do |previous|
+                    previous.is_a?(Array) ? previous[0] : previous
+                  end
+                if values !=
+                     %i[putspecialobject putspecialobject putobject putobject]
+                  next
+                end
+
+                # Now that we know it's in the structure we want it, we can use
+                # the values of the putobject to determine the alias.
+                location = Location.new(line, :unknown)
+                results << AliasMethodDefinition.new(
+                  current_nesting,
+                  insns[index - 2][1],
+                  location,
+                  EntryComments.new(file_comments, location)
+                )
+              end
             end
           end
         end
@@ -321,6 +450,20 @@ module SyntaxTree
     # It is not as fast as using the instruction sequences directly, but is
     # supported on all runtimes.
     class ParserBackend
+      class ConstantNameVisitor < Visitor
+        def visit_const_ref(node)
+          [node.constant.value.to_sym]
+        end
+
+        def visit_const_path_ref(node)
+          visit(node.parent) << node.constant.value.to_sym
+        end
+
+        def visit_var_ref(node)
+          [node.value.value.to_sym]
+        end
+      end
+
       class IndexVisitor < Visitor
         attr_reader :results, :nesting, :statements
 
@@ -331,8 +474,46 @@ module SyntaxTree
         end
 
         visit_methods do
+          def visit_alias(node)
+            if node.left.is_a?(SymbolLiteral) && node.right.is_a?(SymbolLiteral)
+              location =
+                Location.new(
+                  node.location.start_line,
+                  node.location.start_column
+                )
+
+              results << AliasMethodDefinition.new(
+                nesting.dup,
+                node.left.value.value.to_sym,
+                location,
+                comments_for(node)
+              )
+            end
+
+            super
+          end
+
+          def visit_assign(node)
+            if node.target.is_a?(VarField) && node.target.value.is_a?(Const)
+              location =
+                Location.new(
+                  node.location.start_line,
+                  node.location.start_column
+                )
+
+              results << ConstantDefinition.new(
+                nesting.dup,
+                node.target.value.value.to_sym,
+                location,
+                comments_for(node)
+              )
+            end
+
+            super
+          end
+
           def visit_class(node)
-            names = visit(node.constant)
+            names = node.constant.accept(ConstantNameVisitor.new)
             nesting << names
 
             location =
@@ -340,7 +521,7 @@ module SyntaxTree
 
             superclass =
               if node.superclass
-                visited = visit(node.superclass)
+                visited = node.superclass.accept(ConstantNameVisitor.new)
 
                 if visited == [[]]
                   raise NotImplementedError, "superclass with non constant path"
@@ -363,12 +544,41 @@ module SyntaxTree
             nesting.pop
           end
 
-          def visit_const_ref(node)
-            [node.constant.value.to_sym]
-          end
+          def visit_command(node)
+            case node.message.value
+            when "attr_reader", "attr_writer", "attr_accessor"
+              comments = comments_for(node)
+              location =
+                Location.new(
+                  node.location.start_line,
+                  node.location.start_column
+                )
+
+              node.arguments.parts.each do |argument|
+                next unless argument.is_a?(SymbolLiteral)
+                name = argument.value.value.to_sym
+
+                if node.message.value != "attr_writer"
+                  results << MethodDefinition.new(
+                    nesting.dup,
+                    name,
+                    location,
+                    comments
+                  )
+                end
+
+                if node.message.value != "attr_reader"
+                  results << MethodDefinition.new(
+                    nesting.dup,
+                    :"#{name}=",
+                    location,
+                    comments
+                  )
+                end
+              end
+            end
 
-          def visit_const_path_ref(node)
-            visit(node.parent) << node.constant.value.to_sym
+            super
           end
 
           def visit_def(node)
@@ -391,10 +601,12 @@ module SyntaxTree
                 comments_for(node)
               )
             end
+
+            super
           end
 
           def visit_module(node)
-            names = visit(node.constant)
+            names = node.constant.accept(ConstantNameVisitor.new)
             nesting << names
 
             location =
@@ -420,10 +632,6 @@ module SyntaxTree
             @statements = node
             super
           end
-
-          def visit_var_ref(node)
-            [node.value.value.to_sym]
-          end
         end
 
         private
@@ -433,8 +641,10 @@ module SyntaxTree
 
           body = statements.body
           line = node.location.start_line - 1
-          index = body.index(node) - 1
+          index = body.index(node)
+          return comments if index.nil?
 
+          index -= 1
           while index >= 0 && body[index].is_a?(Comment) &&
                   (line - body[index].location.start_line < 2)
             comments.unshift(body[index].value)