masamune-ast 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1131c8f9da2cb862e77f3840c9abc806443a7d4445d788fd3683cec41b1916e8
+  data.tar.gz: bb9025361d1063fd3f7776070830b4e41b7529d3ea04b11fbe9466890bca8d28
+SHA512:
+  metadata.gz: f6e571ac354ab0fae7e39fbf1d0030c1025738732d008b9c3afa9ade11c2cb41b88d67a08ac66d878dd913fae1a9cf08f79d4539f931b7a55fd43f563332d08b
+  data.tar.gz: 47065f76eebaab82601b9d577a738765e66d588f6ad300931223bd61824e0f290612a6341dd098d8bd78cdb90e5b44ce9513d6beb5280b9ebc5691b4044145ea

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-04-24
+
+- Initial release

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in masamune.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "minitest", "~> 5.0"
+
+gem "pry"

data/Gemfile.lock

@@ -0,0 +1,27 @@
+PATH
+  remote: .
+  specs:
+    masamune-ast (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.3)
+    method_source (1.0.0)
+    minitest (5.18.0)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.0.6)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  masamune-ast!
+  minitest (~> 5.0)
+  pry
+  rake (~> 13.0)
+
+BUNDLED WITH
+   2.4.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Gabriel Zayas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,66 @@
+# MasamuneAst
+
+## A Ruby source code analyzer based on Ripper’s Abstract Syntax Tree generator (`Ripper#sexp`).
+
+## Installation
+
+```ruby
+gem "masamune-ast"
+```
+
+## Usage
+
+Pinpoint variables and methods in your source code even when other tokens have the same or similar spelling:
+```ruby
+code = <<CODE
+java = "java"
+javascript = java + "script"
+puts java + " is not " + javascript
+# java
+CODE
+
+msmn = MasamuneAst::AbstractSyntaxTree.new(code)
+
+msmn.variables
+[[[1, 0], "java"], [[2, 0], "javascript"], [[2, 13], "java"]]
+
+msmn.search(:variable, "java")
+#=> [[[1, 0], "java"], [[2, 13], "java"]]
+```
+
+In some cases, it can be easier to look at the given lex nodes to analyze your source code:
+```ruby
+msmn.lex_nodes
+=> [#<Masamune::LexNode:0x00007fd61810cac0 @ast_id=1200, @index=0, @position=[1, 0], @state=CMDARG, @token="java", @type=:ident>,
+ #<Masamune::LexNode:0x00007fd61810c930 @ast_id=1200, @index=1, @position=[1, 4], @state=CMDARG, @token=" ", @type=:sp>,
+ #<Masamune::LexNode:0x00007fd61810c7c8 @ast_id=1200, @index=2, @position=[1, 5], @state=BEG, @token="=", @type=:op>,
+ #<Masamune::LexNode:0x00007fd61810c638 @ast_id=1200, @index=3, @position=[1, 6], @state=BEG, @token=" ", @type=:sp>,
+ #<Masamune::LexNode:0x00007fd61810c480 @ast_id=1200, @index=4, @position=[1, 7], @state=BEG, @token="\"", @type=:tstring_beg>,
+ #<Masamune::LexNode:0x00007fd61810c318 @ast_id=1200, @index=5, @position=[1, 8], @state=BEG, @token="java", @type=:tstring_content>,
+ #<Masamune::LexNode:0x00007fd61810c188 @ast_id=1200, @index=6, @position=[1, 12], @state=END, @token="\"", @type=:tstring_end>,
+ #<Masamune::LexNode:0x00007fd61810c020 @ast_id=1200, @index=7, @position=[1, 13], @state=BEG, @token="\n", @type=:nl>,
+ #<Masamune::LexNode:0x00007fd618113e88 @ast_id=1200, @index=8, @position=[2, 0], @state=CMDARG, @token="javascript", @type=:ident>,
+ #<Masamune::LexNode:0x00007fd618113cf8 @ast_id=1200, @index=9, @position=[2, 10], @state=CMDARG, @token=" ", @type=:sp>,
+ #<Masamune::LexNode:0x00007fd618113b68 @ast_id=1200, @index=10, @position=[2, 11], @state=BEG, @token="=", @type=:op>,
+…
+]
+
+msmn.lex_nodes[8].is_variable?
+#=> true
+
+msmn.lex_nodes[8].is_string?
+#=> false
+
+msmn.lex_nodes[8].is_method_definition?
+#=> false
+
+# etc...
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/gazayas/masamune.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+task default: :test

data/lib/masamune-ast/abstract_syntax_tree/data_node.rb

@@ -0,0 +1,11 @@
+# TODO: Not sure if I'll implement this yet.
+
+module MasamuneAst
+  class AbstractSyntaxTree
+    class DataNode << MasamuneAst::AbstractSyntaxTree::Node
+      def initialize(tree_node)
+        @parent, @contents, _ = tree_node
+      end
+    end
+  end
+end

data/lib/masamune-ast/abstract_syntax_tree/node.rb

@@ -0,0 +1,16 @@
+# TODO: Not sure if I'll implement this yet.
+
+module MasamuneAst
+  class AbstractSyntaxTree
+    class Node
+      def initialize(tree_node)
+        @contents = tree_node
+      end
+
+      # i.e. - [:@ident, "variable_name", [4, 7]]
+      def is_data_node?
+        @contents[0].is_a?(Symbol) && @contents[1].is_a?(String) && (@contents[2][0].is_a?(Integer) && @contents[2][1].is_a?(Integer))
+      end
+    end
+  end
+end

data/lib/masamune-ast/abstract_syntax_tree.rb

@@ -0,0 +1,121 @@
+module MasamuneAst
+  class AbstractSyntaxTree
+    attr_reader :data
+    attr_accessor :lex_nodes, :debug
+
+    def initialize(code)
+      @data = Ripper.sexp(code)
+      raw_lex_nodes = Ripper.lex(code)
+      @lex_nodes = raw_lex_nodes.map do |lex_node|
+        MasamuneAst::LexNode.new(raw_lex_nodes.index(lex_node), lex_node, self.__id__)
+      end
+      @debug = false
+    end
+
+    def tree_nodes
+      raise "Set `debug` to `true` to display the tree nodes" unless @debug
+      search # Perform search with no conditions.
+    end
+
+    def variables
+      search(:variable)
+    end
+
+    def all_methods
+      method_definitions + method_calls
+    end
+
+    def method_definitions
+      search(:def)
+    end
+
+    def method_calls
+      search(:method_call)
+    end
+
+    def strings
+      search(:string)
+    end
+
+    def data_nodes
+      search(:data_node)
+    end
+
+    def search(type = nil, token = nil, tree_node = self.data, result = [])
+      return unless tree_node.is_a?(Array) || (tree_node.is_a?(Array) && tree_node.empty?)
+      debug_output(tree_node) if @debug
+
+      # If the first element is an array, then we're getting all arrays so we just continue the search.
+      if tree_node.first.is_a?(Array)
+        tree_node.each { |node| search(type, token, node, result) }
+      elsif tree_node.first.is_a?(Symbol)
+        # TODO: These two if statements bother me a lot.
+        # There should be a more effiecient/smart way to do this.
+        if has_data_node?(tree_node)
+          if (type == :variable && (tree_node.first == :var_field || tree_node.first == :var_ref)) ||
+             (type == :string && tree_node.first == :string_content) ||
+             (type == :def && tree_node.first == :def) ||
+             (type == :method_call && tree_node.first == :vcall)
+
+            # TODO: AbstractSyntaxTree::DataNode.new(tree_node)
+            # For most tree nodes, the data_node is housed in the second element.
+            position, data_node_token = data_node_parts(tree_node[1])
+
+            # Gather all results if token isn't specified.
+            result << [position, data_node_token] if token == data_node_token || token.nil?
+          end
+
+          # Continue search for all necessary elements.
+          case tree_node.first
+          when :def, :command
+            tree_node.each { |node| search(type, token, node, result) }
+          end
+
+        # The data nodes in :call nodes are in a different place within the array, so we handle that here.
+        # These :call nodes represent methods and chained methods like `[1, 2, 3].sum.times`.
+        elsif (type == :method_call && tree_node.first == :call)
+          # The method inside the [:call, ...] data node is the last element in the array.
+          position, data_node_token = data_node_parts(tree_node.last)
+          result << [position, data_node_token] if token == data_node_token || token.nil?
+          # The second element is where more :call nodes are nested, so we search it.
+          search(type, token, tree_node[1], result)
+        else
+          # Simply continue the search for all other nodes.
+          tree_node.each { |node| search(type, token, node, result) }
+        end
+      end
+
+      result
+    end
+
+    private
+
+    # A data node represents an abstraction in the AST which has details about a specific command.
+    # i.e. - [:@ident, "variable_name", [4, 7]]
+    # These values are the `type`, `token`, and `position`, respectively.
+    # It is simliar to what you see in `Ripper.lex(code)` and `Masamune::Base's @lex_nodes`.
+    # Data nodes serve as a base case when recursively searching the AST.
+    #
+    # The parent node's first element houses the type of action being performed:
+    # i.e. - [:assign, [:@ident, "variable_name", [4, 7]]]
+    # `has_data_node?` is performed on a parent node.
+    def has_data_node?(node)
+      node[1].is_a?(Array) && node[1][1].is_a?(String)
+    end
+
+    def data_node_parts(tree_node)
+      _, token, position = tree_node
+      [position, token]
+    end
+
+    def is_line_position?(tree_node)
+      tree_node.size == 2 && tree_node.first.is_a?(Integer) && tree_node.last.is_a?(Integer)
+    end
+
+    def debug_output(tree_node)
+      puts "==================================" # TODO: Track the array depth and output the number here.
+      puts "=================================="
+      p tree_node
+    end
+  end
+end

data/lib/masamune-ast/lex_node.rb

@@ -0,0 +1,52 @@
+module MasamuneAst
+  # https://docs.ruby-lang.org/en/3.0/Ripper.html#method-c-lex
+  #
+  # @position: Line number and starting point on the line. i.e. - [4, 7].
+  # @type: The type of token. i.e. - :kw (is a keyword).
+  # @token: The raw string which represents the actual ruby code. i.e. - "do".
+  # @state: Ripper::Lexer::State. i.e. - CMDARG.
+  class LexNode
+    attr_accessor :position, :type, :token, :state, :ast_id
+    attr_reader :index
+
+    def initialize(index, raw_lex_node, ast_id)
+      @index = index
+      @position, @type, @token, @state = raw_lex_node
+      @type = @type.to_s.gsub(/^[a-z]*_/, "").to_sym
+
+      # Since the Abstract Syntax Tree can get very large,
+      # We just save the id and reference it with `ast` below.
+      @ast_id = ast_id
+    end
+
+    def ast
+      ObjectSpace._id2ref(@ast_id)
+    end
+
+    def is_variable?
+      return false unless is_identifier?
+      ast.search(:variable, @token).any?
+    end
+
+    def is_method?
+      return false unless is_identifier?
+      is_method_definition? || is_method_call?
+    end
+
+    def is_method_definition?
+      ast.search(:def, @token).any?
+    end
+
+    def is_method_call?
+      ast.search(:method_call, @token).any?
+    end
+
+    def is_identifier?
+      type == :ident
+    end
+
+    def is_string?
+      type == :tstring_content
+    end
+  end
+end

data/lib/masamune-ast/slicer.rb

@@ -0,0 +1,19 @@
+module MasamuneAst
+  module Slicer
+    def initialize(ast)
+      @ast = ast
+    end
+
+    # TODO
+
+    # def replace(type, token)
+    #   @ast.search(type, token)
+    # end
+
+    # def insert_inside_block
+    # end
+
+    # def append_to_block
+    # end
+  end
+end

data/lib/masamune-ast/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module MasamuneAst
+  VERSION = "0.1.0"
+end

data/lib/masamune-ast.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative "masamune-ast/version"
+require "ripper"
+require "masamune-ast/lex_node"
+require "masamune-ast/abstract_syntax_tree"
+
+require "pp"
+require "pry"
+
+module MasamuneAst
+end

data/sig/masamune.rbs

@@ -0,0 +1,4 @@
+module Masamune
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification
+name: masamune-ast
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Gabriel Zayas
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2023-04-29 00:00:00.000000000 Z
+dependencies: []
+description: A layer of abstraction on top of Ripper for handling Abstract Syntax
+  Trees in Ruby.
+email:
+- g-zayas@hotmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/masamune-ast.rb
+- lib/masamune-ast/abstract_syntax_tree.rb
+- lib/masamune-ast/abstract_syntax_tree/data_node.rb
+- lib/masamune-ast/abstract_syntax_tree/node.rb
+- lib/masamune-ast/lex_node.rb
+- lib/masamune-ast/slicer.rb
+- lib/masamune-ast/version.rb
+- sig/masamune.rbs
+homepage: https://www.github.com/gazayas/masamune
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://www.github.com/gazayas/masamune
+  source_code_uri: https://www.github.com/gazayas/masamune
+  changelog_uri: https://www.github.com/gazayas/masamune
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key: 
+specification_version: 4
+summary: MasamuneAst
+test_files: []