masamune-ast 1.2.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f599b770d8e12444bbb2c2bb6aad28d8d0b9673e99708d9d713af4186f96fb4f
-  data.tar.gz: 723f6ab17c2152f9ab92e6da6f0130bc1ab1aa1fd0174ee00a1d9bb7206df182
+  metadata.gz: caf2d78e9516f9431e751db61028a2dd90354b2dce71eecc116a848b7b171344
+  data.tar.gz: c556d620be50a699a6757981ce692929298baf92cd9ebbaa86256f7277e28d9e
 SHA512:
-  metadata.gz: 622cfe88710212e12d4045fe39837dd9767b56b64abb62872f544b37d6ac286f85dfb9324ef709bb32ab4219e280eff7f4fc6a966ddb379b79b5a5b3b9c06dc3
-  data.tar.gz: 7db8c29128764c9c36f79a98156c68d4bc008a183ceeed351405a6377606721e33523b267362b0cf1f3a19119db3e1606d494869ec28f340680a8812a4e3868b
+  metadata.gz: 62015efa9a2e971af08db707acccf0df1761fc4a6b89fa14ca3ee830732d02ad3c06434c14c9b153006a938217f01e80370ec1dda11fa108f954ec8b47575b5a
+  data.tar.gz: 125c5527076511907993aee66d47e66caee9c416c33d44f63585e31676c214cbe959f5bf35165a748199fca2c87d3fda181f0f55a51331edd45c382998796178

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [2.0.0] - 2023-10-13
+
+- Migrate to Prism by @gazayas in #60
+- Return array of Prism nodes for `Masamune::AbstractSyntaxTree` search methods.
+- Refactor NodeHelper into inline extensions by @kaspth in #61
+- Remove functionality to replace source code by Node name (the `type` keyword now only accepts Symbols such as `:variable` and `:method_call`).
+
 ## [1.0.0] - 2023-05-01
 
 - Change `MasamuneAst` module to `Masamune`.

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    masamune-ast (1.2.1)
+    masamune-ast (2.0.0)
       activesupport
+      prism (>= 0.13.0)
 
 GEM
   remote: https://rubygems.org/
@@ -18,6 +19,7 @@ GEM
       concurrent-ruby (~> 1.0)
     method_source (1.0.0)
     minitest (5.18.0)
+    prism (0.13.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -26,6 +28,7 @@ GEM
       concurrent-ruby (~> 1.0)
 
 PLATFORMS
+  arm64-darwin-22
   x86_64-darwin-21
   x86_64-linux
 

data/README.md

@@ -1,6 +1,6 @@
 # Masamune
 
-## A Ruby source code analyzer based on Ripper’s Abstract Syntax Tree generator.
+## A covenience wrapper around Prism, a Ruby source code parser 
 
 ## Installation
 
@@ -23,25 +23,21 @@ require "masamune"
 Isolate and replace variables, methods, or strings in your Ruby source code according to the specific tokens allotted when the code is intially parsed:
 ```ruby
 code = <<~CODE
-  10.times do |n|
-    puts n
-  end
-
-  def n
-    "n"
+  :hello
+  hello = "hello"
+  def hello
+    puts hello
   end
 CODE
 
 msmn = Masamune::AbstractSyntaxTree.new(code)
-msmn.replace(type: :variable, old_token: "n", new_token: "foo")
+msmn.replace(type: :variable, old_token_value: "hello", new_token_value: "greeting")
 
 # This will produce the following code in string form.
-10.times do |foo|
-  puts foo
-end
-
-def n
-  "n"
+:hello
+greeting = "hello"
+def hello
+  puts greeting
 end
 ```
 
@@ -56,22 +52,20 @@ CODE
 
 msmn = Masamune::AbstractSyntaxTree.new(code)
 
+# Returns an array of Prism nodes
 msmn.variables
-#=> [{:line_number=>1, :index_on_line=>0, :token=>"java"},
-#=> {:line_number=>2, :index_on_line=>0, :token=>"javascript"},
-#=> {:line_number=>2, :index_on_line=>13, :token=>"java"},
-#=> {:line_number=>3, :index_on_line=>5, :token=>"java"},
-#=> {:line_number=>3, :index_on_line=>25, :token=>"javascript"}]
 
+msmn.variables.first.token_value
+#=> "java"
+msmn.variables.first.token_location
+#=> (1,0)-(1,4)
+
+# Returns an array of Prism nodes
 msmn.strings
-#=> [{:line_number=>1, :index_on_line=>8, :token=>"java"},
-#=> {:line_number=>2, :index_on_line=>21, :token=>"script"},
-#=> {:line_number=>3, :index_on_line=>13, :token=>" is not "}]
 
-msmn.variables(name: "java")
-#=> [{:line_number=>1, :index_on_line=>0, :token=>"java"},
-#=> {:line_number=>2, :index_on_line=>13, :token=>"java"},
-#=> {:line_number=>3, :index_on_line=>5, :token=>"java"}]
+last_java_node = msmn.variables(token_value: "java").last
+last_java_node.token_location
+#=> (3,5)-(3,9)
 
 code = <<CODE
 ary = [1, 2, 3]
@@ -87,68 +81,22 @@ CODE
 
 msmn = Masamune::AbstractSyntaxTree.new(code)
 
-msmn.all_methods
-#=> [{:line_number=>2, :index_on_line=>4, :token=>"sum"},
-#=> {:line_number=>2, :index_on_line=>8, :token=>"times"},
-#=> {:line_number=>3, :index_on_line=>2, :token=>"puts"},
-#=> {:line_number=>6, :index_on_line=>4, :token=>"foo"},
-#=> {:line_number=>8, :index_on_line=>0, :token=>"foo"},
-#=> {:line_number=>9, :index_on_line=>0, :token=>"foo"}]
-
-msmn.method_calls
-#=> [{:line_number=>2, :index_on_line=>4, :token=>"sum"},
-#=> {:line_number=>2, :index_on_line=>8, :token=>"times"},
-#=> {:line_number=>3, :index_on_line=>2, :token=>"puts"},
-#=> {:line_number=>8, :index_on_line=>0, :token=>"foo"},
-#=> {:line_number=>9, :index_on_line=>0, :token=>"foo"}]
-
-msmn.method_definitions
-#=> [{:line_number=>6, :index_on_line=>4, :token=>"foo"}]
-```
+msmn.method_definitions.size
+#=> 1
 
-You can also return the node classes themselves and get the data from there:
-```ruby
-code = <<~CODE
-  "ruby"
-  "rails"
-CODE
+msmn.method_calls.size
+#=> 5
 
-msmn = Masamune::AbstractSyntaxTree.new(code)
-msmn.strings(result_type: :nodes)
-#=> [#<Masamune::AbstractSyntaxTree::StringContent:0x00007ff2d987c020
-#=>   @ast_id=406820,
-#=>   @contents=[:string_content, [:@tstring_content, "ruby", [1, 1]]],
-#=>   @data_nodes=
-#=>    [#<Masamune::AbstractSyntaxTree::DataNode:0x00007ff2d9883fc8
-#=>      @ast_id=406820,
-#=>      @contents=[:@tstring_content, "ruby", [1, 1]],
-#=>      @data_nodes=nil,
-#=>      @index_on_line=1,
-#=>      @line_number=1,
-#=>      @parent=#<Masamune::AbstractSyntaxTree::StringContent:0x00007ff2d987c020 ...>,
-#=>      @token="ruby",
-#=>      @type=:@tstring_content>]>,
-#=>  #<Masamune::AbstractSyntaxTree::StringContent:0x00007ff2d9883190
-#=>   @ast_id=406820,
-#=>   @contents=[:string_content, [:@tstring_content, "rails", [2, 1]]],
-#=>   @data_nodes=
-#=>    [#<Masamune::AbstractSyntaxTree::DataNode:0x00007ff2d9883168
-#=>      @ast_id=406820,
-#=>      @contents=[:@tstring_content, "rails", [2, 1]],
-#=>      @data_nodes=nil,
-#=>      @index_on_line=1,
-#=>      @line_number=2,
-#=>      @parent=#<Masamune::AbstractSyntaxTree::StringContent:0x00007ff2d9883190 ...>,
-#=>      @token="rails",
-#=>      @type=:@tstring_content>]>]
+msmn.all_methods.size
+#=> 6
 ```
 
-In some cases, it can be easier to look at the given lex nodes to analyze your source code since you can easily see the index and the line position it's on:
+In some cases, it can be easier to look at the given lex nodes to analyze your source code:
 ```ruby
 msmn.lex_nodes
-=> [#<Masamune::LexNode:0x00007fd61810cac0 @ast_id=1200, @index=0, @position=[1, 0], @state=CMDARG, @token="java", @type=:ident>,
- #<Masamune::LexNode:0x00007fd61810c930 @ast_id=1200, @index=1, @position=[1, 4], @state=CMDARG, @token=" ", @type=:sp>,
- #<Masamune::LexNode:0x00007fd61810c7c8 @ast_id=1200, @index=2, @position=[1, 5], @state=BEG, @token="=", @type=:op>,
+=> [#<Masamune::LexNode:0x00007fd61810cac0 @ast_id=1200, @index=0, @location=[1, 0], @state=CMDARG, @token="java", @type=:ident>,
+ #<Masamune::LexNode:0x00007fd61810c930 @ast_id=1200, @index=1, @location=[1, 4], @state=CMDARG, @token=" ", @type=:sp>,
+ #<Masamune::LexNode:0x00007fd61810c7c8 @ast_id=1200, @index=2, @location=[1, 5], @state=BEG, @token="=", @type=:op>,
 …
 ]
 

data/lib/masamune/abstract_syntax_tree/prism/node_extensions.rb

@@ -0,0 +1,82 @@
+module Prism
+  class Node
+    def comment? = false
+
+    def self.node_predicate(name, body = -> { true })
+      define_method(name, &body)
+      Node.define_method(name) { false } unless Node.respond_to?(name, false)
+    end
+
+    def line_number
+      token_location.start_line
+    end
+
+    # #location provides helpful information for the source code of a node as a whole, but in
+    # Masamune we generally want the token value itself, so we primarily get the token value's location.
+    def token_location
+      location
+    end
+
+    # The source code of Prism nodes can be retrieved by calling #slice.
+    # However, the output tends to vary from node to node, and other methods
+    # like `name`, `message`, etc. are available in Prism nodes which means you
+    # have to know exactly which method to call the get the results you want.
+    #
+    # The `token` method simplifies all of this, and just gives you the
+    # variable, method name, etc. in String form by just calling `token_value`.
+    def token_value
+      content
+    end
+  end
+
+  class DefNode
+    node_predicate :method?
+    node_predicate :method_definition?
+    def token_location = name_loc
+    def token_value = name.to_s
+  end
+
+  class CallNode
+    def method? = true
+    node_predicate :method_call?
+    def token_location = message_loc
+    def token_value = message
+  end
+
+  class LocalVariableWriteNode
+    node_predicate :variable?
+    def token_location = name_loc
+    def token_value = name.to_s
+  end
+
+  class LocalVariableReadNode
+    def variable? = true
+    def token_value = name.to_s
+  end
+
+  class RequiredParameterNode
+    def variable? = true
+    def token_value = slice
+  end
+
+  class StringNode
+    node_predicate :string?
+    def token_location = content_loc
+    def token_value = content
+  end
+
+  class SymbolNode
+    node_predicate :symbol?
+    node_predicate :symbol_literal?, -> { closing_loc.nil? }
+    node_predicate :symbol_string?, -> { closing_loc.present? }
+
+    def token_location = value_loc
+    def token_value = value
+  end
+
+  class Comment
+    def comment? = true
+    def token_location = location
+    def token_value = location.slice
+  end
+end

data/lib/masamune/abstract_syntax_tree/visitors/block_parameters_visitor.rb

@@ -0,0 +1,18 @@
+module Masamune
+  class AbstractSyntaxTree
+    class BlockParametersVisitors < Prism::Visitor
+      attr_reader :results
+
+      def initialize
+        @results = []
+      end
+
+      # Since block parameters nodes can house multiple variables in one node,
+      # we don't have to check for a token_value.
+      def visit_block_parameters_node(node)
+        results << node
+        super
+      end
+    end
+  end
+end

data/lib/masamune/abstract_syntax_tree/visitors/method_calls_visitor.rb

@@ -0,0 +1,19 @@
+module Masamune
+  class AbstractSyntaxTree
+    class MethodCallsVisitor < Prism::Visitor
+      attr_reader :token_value, :results
+
+      def initialize(token_value)
+        @token_value = token_value
+        @results = []
+      end
+
+      def visit_call_node(node)
+        if node.method_call?
+          results << node if token_value.nil? || token_value == node.name
+        end
+        super
+      end
+    end
+  end
+end

data/lib/masamune/abstract_syntax_tree/visitors/method_definitions_visitor.rb

@@ -0,0 +1,17 @@
+module Masamune
+  class AbstractSyntaxTree
+    class MethodDefinitionsVisitor < Prism::Visitor
+      attr_reader :token_value, :results
+
+      def initialize(token_value)
+        @token_value = token_value
+        @results = []
+      end
+
+      def visit_def_node(node)
+        results << node if token_value.nil? || token_value == node.name.to_s
+        super
+      end
+    end
+  end
+end

data/lib/masamune/abstract_syntax_tree/visitors/parameters_visitor.rb

@@ -0,0 +1,18 @@
+module Masamune
+  class AbstractSyntaxTree
+    class ParametersVisitor < Prism::Visitor
+      attr_reader :results
+
+      def initialize
+        @results = []
+      end
+
+      # Since parameters nodes can house multiple variables in one node,
+      # we don't have to check for a token_value.
+      def visit_parameters_node(node)
+        results << node
+        super
+      end
+    end
+  end
+end

data/lib/masamune/abstract_syntax_tree/visitors/strings_visitor.rb

@@ -0,0 +1,17 @@
+module Masamune
+  class AbstractSyntaxTree
+    class StringsVisitor < Prism::Visitor
+      attr_reader :token_value, :results
+
+      def initialize(token_value)
+        @token_value = token_value
+        @results = []
+      end
+
+      def visit_string_node(node)
+        results << node if token_value.nil? || token_value == node.content
+        super
+      end
+    end
+  end
+end

data/lib/masamune/abstract_syntax_tree/visitors/symbols_visitor.rb

@@ -0,0 +1,17 @@
+module Masamune
+  class AbstractSyntaxTree
+    class SymbolsVisitor < Prism::Visitor
+      attr_reader :token_value, :results
+
+      def initialize(token_value)
+        @token_value = token_value
+        @results = []
+      end
+
+      def visit_symbol_node(node)
+        results << node if token_value.nil? || token_value == node.value
+        super
+      end
+    end
+  end
+end

data/lib/masamune/abstract_syntax_tree/visitors/variables_visitor.rb

@@ -0,0 +1,34 @@
+module Masamune
+  class AbstractSyntaxTree
+    class VariablesVisitor < Prism::Visitor
+      attr_reader :token_value, :results
+
+      def initialize(token_value)
+        @token_value = token_value
+        @results = []
+      end
+
+      def visit_local_variable_write_node(node)
+        results << node if token_value.nil? || token_value == node.name.to_s
+        super
+      end
+
+      def visit_local_variable_read_node(node)
+        results << node if token_value.nil? || token_value == node.name.to_s
+        super
+      end
+
+      def visit_required_parameter_node(node)
+        results << node if token_value.nil? || token_value == node.name.to_s
+        super
+      end
+
+      def visit_call_node(node)
+        if node.variable_call?
+          results << node if token_value.nil? || token_value == node.name
+        end
+        super
+      end
+    end
+  end
+end

data/lib/masamune/abstract_syntax_tree.rb

@@ -1,162 +1,91 @@
+require_relative "abstract_syntax_tree/prism/node_extensions"
+
 module Masamune
   class AbstractSyntaxTree
-    attr_reader :code, :tree, :comment_list
-    attr_accessor :node_list, :data_node_list, :lex_nodes
+    attr_reader :code, :prism, :ripper
+    attr_accessor :lex_nodes
 
     def initialize(code)
       @code = code
-      @tree = Ripper.sexp(code)
+      @prism = Prism.parse(code)
+      @ripper = Ripper.sexp(code)
       raw_lex_nodes = Ripper.lex(code)
       @lex_nodes = raw_lex_nodes.map do |lex_node|
         LexNode.new(raw_lex_nodes.index(lex_node), lex_node, self.__id__)
       end
-
-      @node_list = []
-      @data_node_list = []
-      @comment_list = []
-      register_nodes(@tree)
-
-      # Refer to Masamune::AbstractSyntaxTree::Comment
-      # to see why we register these separately.
-      register_comments
     end
 
-    def register_nodes(tree_node = self.tree)
-      if tree_node.is_a?(Array)
-        klass = get_node_class(tree_node.first)
-        msmn_node = klass.new(tree_node, self.__id__)
-      else
-        # Create a general node if the node is a single value.
-        msmn_node = Node.new(tree_node, self.__id__)
-      end
-
-      # Register nodes and any data nodes housed within it.
-      # See Masamune::AbstractSyntaxTree::DataNode for more details on what a data node is.
-      @node_list << msmn_node
-      msmn_node.data_nodes.each { |dn| @data_node_list << dn } if msmn_node.data_nodes
-
-      # Continue down the tree until base case is reached.
-      if !msmn_node.nil? && msmn_node.contents.is_a?(Array)
-        msmn_node.contents.each { |node| register_nodes(node) }
-      end
+    def variables(token_value: nil)
+      visitor = VariablesVisitor.new(token_value)
+      @prism.value.accept(visitor)
+      visitor.results
     end
 
-    def register_comments
-      comment_lex_nodes = @lex_nodes.select {|node| node.type == :comment}.flatten
-      @comment_list << comment_lex_nodes.map {|node| Comment.new(node, self.__id__)}
-      @comment_list.flatten!
+    def strings(token_value: nil)
+      visitor = StringsVisitor.new(token_value)
+      @prism.value.accept(visitor)
+      visitor.results
     end
 
-    # TODO: A lot of these methods have the same content,
-    # so I want to come up with a way to refactor these.
-
-    # TODO: Add block_params: true to the arguments.
-    def variables(name: nil, result_type: Hash)
-      results = find_nodes([VarField, VarRef, Params], token: name, result_type: result_type)
-      order_results(results)
-    end
-
-    def strings(content: nil, result_type: Hash)
-      results = find_nodes(StringContent, token: content, result_type: result_type)
-      order_results(results)
+    def all_methods(token_value: nil)
+      order_nodes(method_definitions(token_value: token_value) + method_calls(token_value: token_value))
     end
 
-    def method_definitions(name: nil, result_type: Hash)
-      results = find_nodes(Def, token: name, result_type: result_type)
-      order_results(results)
+    def method_definitions(token_value: nil)
+      visitor = MethodDefinitionsVisitor.new(token_value)
+      @prism.value.accept(visitor)
+      visitor.results
     end
 
-    def method_calls(name: nil, result_type: Hash)
-      results = find_nodes([Vcall, Call, Command], token: name, result_type: result_type)
-      order_results(results)
+    def method_calls(token_value: nil)
+      visitor = MethodCallsVisitor.new(token_value)
+      @prism.value.accept(visitor)
+      visitor.results
     end
 
-    # TODO
-    def do_block_params
+    def symbols(token_value: nil)
+      visitor = SymbolsVisitor.new(token_value)
+      @prism.value.accept(visitor)
+      visitor.results
     end
 
-    # TODO
-    def brace_block_params
-    end
+    def symbol_literals(token_value: nil)
+      result = symbols(token_value: token_value)
 
-    def symbols(content: nil, result_type: Hash)
-      results = symbol_literals(content: content, result_type: result_type) + string_symbols(content: content, result_type: result_type)
-      order_results(results)
+      # TODO: Describe why closing_loc has to happen.
+      result.select{|node| node.closing_loc.nil?}
     end
 
-    def symbol_literals(content: nil, result_type: Hash)
-      results = find_nodes(SymbolLiteral, token: content, result_type: result_type)
-      order_results(results)
-    end
+    def string_symbols(token_value: nil)
+      result = symbols(token_value: token_value)
 
-    def string_symbols(content: nil, result_type: Hash)
-      results = find_nodes(DynaSymbol, token: content, result_type: result_type)
-      order_results(results)
+      # TODO: Describe why closing_loc has to happen.
+      result.reject{|node| node.closing_loc.nil?}
     end
 
-    def comments(content: nil, result_type: Hash)
-      results = find_nodes(Comment, token: content, result_type: result_type)
-      order_results(results)
+    # Retrieves all parameters within pipes (i.e. - |x, y, z|).
+    def block_parameters
+      visitor = BlockParametersVisitor.new
+      @prism.value.accept(visitor)
+      visitor.results
     end
 
-    def all_methods(name: nil, result_type: Hash)
-      results = method_definitions(name: name, result_type: result_type) + method_calls(name: name, result_type: result_type)
-      order_results(results)
+    def parameters(token_value: nil)
+      visitor = ParametersVisitor.new(token_value)
+      @prism.value.accept(visitor)
+      visitor.results
     end
 
-    def block_params(content: nil, result_type: Hash)
-      # TODO: do_block_params + brace_block_params
-      results = find_nodes(Params, token: content, result_type: result_type)
-      order_results(results)
+    # TODO: Search by token_value if necessary.
+    def comments
+      @prism.comments
     end
 
-    def find_nodes(token_classes, token: nil, result_type: Hash)
-      token_classes = Array(token_classes)
-
-      nodes = []
-      token_classes.each do |klass|
-        if klass == Comment
-          nodes = @comment_list.dup
-        else
-          nodes << @node_list.select {|node| node.class == klass}
-        end
-      end
-
-      # Searching for multiple classes will yield multi-dimensional arrays,
-      # so we ensure everything is flattened out before moving forward.
-      nodes.flatten!
-
-      return nodes if result_type == :nodes
-
-      if token
-        # TODO: This most likely shouldn't be `node.data_nodes.first`.
-        # There are probably more data_nodes we need to check depending on the node class.
-        nodes = nodes.select {|node| node.data_nodes.first.token == token}.flatten
-      end
-
-      final_result = []
-      nodes.each do |node|
-        # Data for symbols are housed within a nested node, so we handle those differently here.
-        # Read the comments for `get_symbol_data` in the symbol node classes for details.
-        if node.class == SymbolLiteral || node.class == DynaSymbol
-          final_result << node.get_symbol_data.line_data_and_token
-        else
-          node.data_nodes.each {|dn| final_result << dn.line_data_and_token} if node.data_nodes
-        end
-      end
-
-      # Only order the information if we're returning hashes.
-      # TODO: We might want to change the placement of order_results_by_position
-      # if the operation is being done against hashes and not data nodes.
-      # nodes.first.class.is_a?(Hash) ? DataNode.order_results_by_position(final_result) : final_result
-      final_result
-    end
-
-    def replace(type:, old_token:, new_token:)
+    def replace(type:, old_token_value:, new_token_value:)
       Slasher.replace(
         type: type,
-        old_token: old_token,
-        new_token: new_token,
+        old_token_value: old_token_value,
+        new_token_value: new_token_value,
         code: @code,
         ast: self
       )
@@ -164,18 +93,12 @@ module Masamune
 
     private
 
-    def get_node_class(type)
-      "#{self.class}::#{type.to_s.camelize}".safe_constantize || Node # Return base Node class for any not-yet-covered nodes.
-    end
-
-    # We only order results when they are a Hash.
-    # The contents from the Hash are pulled from data nodes.
-    # i.e. - {position: [4, 7], token: "project"}
-    def order_results(results)
-      if results.first.is_a?(Hash)
-        DataNode.order_results_by_position(results)
-      else
-        results
+    # Order results according to their token location.
+    def order_nodes(nodes)
+      nodes.sort do |a, b|
+        by_line = a.token_location.start_line <=> b.token_location.start_line
+        # If we're on the same line, refer to the inner index for order.
+        by_line.zero? ? a.token_location.start_column <=> b.token_location.start_column : by_line
       end
     end
   end