syntax_tree 5.3.0 → 6.0.0

data/lib/syntax_tree.rb

@@ -1,50 +1,41 @@
 # frozen_string_literal: true
 
-require "etc"
-require "fiddle"
-require "json"
-require "pp"
 require "prettier_print"
 require "ripper"
-require "stringio"
 
-require_relative "syntax_tree/formatter"
 require_relative "syntax_tree/node"
-require_relative "syntax_tree/dsl"
-require_relative "syntax_tree/version"
-
 require_relative "syntax_tree/basic_visitor"
 require_relative "syntax_tree/visitor"
-require_relative "syntax_tree/visitor/field_visitor"
-require_relative "syntax_tree/visitor/json_visitor"
-require_relative "syntax_tree/visitor/match_visitor"
-require_relative "syntax_tree/visitor/mutation_visitor"
-require_relative "syntax_tree/visitor/pretty_print_visitor"
-require_relative "syntax_tree/visitor/environment"
-require_relative "syntax_tree/visitor/with_environment"
 
+require_relative "syntax_tree/formatter"
 require_relative "syntax_tree/parser"
-require_relative "syntax_tree/pattern"
-require_relative "syntax_tree/search"
-require_relative "syntax_tree/index"
-
-require_relative "syntax_tree/yarv"
-require_relative "syntax_tree/yarv/bf"
-require_relative "syntax_tree/yarv/compiler"
-require_relative "syntax_tree/yarv/decompiler"
-require_relative "syntax_tree/yarv/disassembler"
-require_relative "syntax_tree/yarv/instruction_sequence"
-require_relative "syntax_tree/yarv/instructions"
-require_relative "syntax_tree/yarv/legacy"
-require_relative "syntax_tree/yarv/local_table"
-require_relative "syntax_tree/yarv/assembler"
-require_relative "syntax_tree/yarv/vm"
+require_relative "syntax_tree/version"
 
 # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It
 # provides the ability to generate a syntax tree from source, as well as the
 # tools necessary to inspect and manipulate that syntax tree. It can be used to
 # build formatters, linters, language servers, and more.
 module SyntaxTree
+  # Syntax Tree the library has many features that aren't always used by the
+  # CLI. Requiring those features takes time, so we autoload as many constants
+  # as possible in order to keep the CLI as fast as possible.
+
+  autoload :DSL, "syntax_tree/dsl"
+  autoload :FieldVisitor, "syntax_tree/field_visitor"
+  autoload :Index, "syntax_tree/index"
+  autoload :JSONVisitor, "syntax_tree/json_visitor"
+  autoload :LanguageServer, "syntax_tree/language_server"
+  autoload :MatchVisitor, "syntax_tree/match_visitor"
+  autoload :Mermaid, "syntax_tree/mermaid"
+  autoload :MermaidVisitor, "syntax_tree/mermaid_visitor"
+  autoload :MutationVisitor, "syntax_tree/mutation_visitor"
+  autoload :Pattern, "syntax_tree/pattern"
+  autoload :PrettyPrintVisitor, "syntax_tree/pretty_print_visitor"
+  autoload :Search, "syntax_tree/search"
+  autoload :Translation, "syntax_tree/translation"
+  autoload :WithScope, "syntax_tree/with_scope"
+  autoload :YARV, "syntax_tree/yarv"
+
   # This holds references to objects that respond to both #parse and #format
   # so that we can use them in the CLI.
   HANDLERS = {}
@@ -63,40 +54,80 @@ module SyntaxTree
   # that Syntax Tree can format arbitrary parts of a document.
   DEFAULT_INDENTATION = 0
 
-  # This is a hook provided so that plugins can register themselves as the
-  # handler for a particular file type.
-  def self.register_handler(extension, handler)
-    HANDLERS[extension] = handler
+  # Parses the given source and returns the formatted source.
+  def self.format(
+    source,
+    maxwidth = DEFAULT_PRINT_WIDTH,
+    base_indentation = DEFAULT_INDENTATION,
+    options: Formatter::Options.new
+  )
+    format_node(
+      source,
+      parse(source),
+      maxwidth,
+      base_indentation,
+      options: options
+    )
   end
 
-  # Parses the given source and returns the syntax tree.
-  def self.parse(source)
-    parser = Parser.new(source)
-    response = parser.parse
-    response unless parser.error?
+  # Parses the given file and returns the formatted source.
+  def self.format_file(
+    filepath,
+    maxwidth = DEFAULT_PRINT_WIDTH,
+    base_indentation = DEFAULT_INDENTATION,
+    options: Formatter::Options.new
+  )
+    format(read(filepath), maxwidth, base_indentation, options: options)
   end
 
-  # Parses the given source and returns the formatted source.
-  def self.format(
+  # Accepts a node in the tree and returns the formatted source.
+  def self.format_node(
     source,
+    node,
     maxwidth = DEFAULT_PRINT_WIDTH,
     base_indentation = DEFAULT_INDENTATION,
     options: Formatter::Options.new
   )
     formatter = Formatter.new(source, [], maxwidth, options: options)
-    parse(source).format(formatter)
+    node.format(formatter)
 
     formatter.flush(base_indentation)
     formatter.output.join
   end
 
+  # Indexes the given source code to return a list of all class, module, and
+  # method definitions. Used to quickly provide indexing capability for IDEs or
+  # documentation generation.
+  def self.index(source)
+    Index.index(source)
+  end
+
+  # Indexes the given file to return a list of all class, module, and method
+  # definitions. Used to quickly provide indexing capability for IDEs or
+  # documentation generation.
+  def self.index_file(filepath)
+    Index.index_file(filepath)
+  end
+
   # A convenience method for creating a new mutation visitor.
   def self.mutation
-    visitor = Visitor::MutationVisitor.new
+    visitor = MutationVisitor.new
     yield visitor
     visitor
   end
 
+  # Parses the given source and returns the syntax tree.
+  def self.parse(source)
+    parser = Parser.new(source)
+    response = parser.parse
+    response unless parser.error?
+  end
+
+  # Parses the given file and returns the syntax tree.
+  def self.parse_file(filepath)
+    parse(read(filepath))
+  end
+
   # Returns the source from the given filepath taking into account any potential
   # magic encoding comments.
   def self.read(filepath)
@@ -112,23 +143,24 @@ module SyntaxTree
     File.read(filepath, encoding: encoding)
   end
 
+  # This is a hook provided so that plugins can register themselves as the
+  # handler for a particular file type.
+  def self.register_handler(extension, handler)
+    HANDLERS[extension] = handler
+  end
+
   # Searches through the given source using the given pattern and yields each
   # node in the tree that matches the pattern to the given block.
   def self.search(source, query, &block)
-    Search.new(Pattern.new(query).compile).scan(parse(source), &block)
-  end
+    pattern = Pattern.new(query).compile
+    program = parse(source)
 
-  # Indexes the given source code to return a list of all class, module, and
-  # method definitions. Used to quickly provide indexing capability for IDEs or
-  # documentation generation.
-  def self.index(source)
-    Index.index(source)
+    Search.new(pattern).scan(program, &block)
   end
 
-  # Indexes the given file to return a list of all class, module, and method
-  # definitions. Used to quickly provide indexing capability for IDEs or
-  # documentation generation.
-  def self.index_file(filepath)
-    Index.index_file(filepath)
+  # Searches through the given file using the given pattern and yields each
+  # node in the tree that matches the pattern to the given block.
+  def self.search_file(filepath, query, &block)
+    search(read(filepath), query, &block)
   end
 end

data/tasks/sorbet.rake

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+module SyntaxTree
+  class RBI
+    include DSL
+
+    attr_reader :body, :line
+
+    def initialize
+      @body = []
+      @line = 1
+    end
+
+    def generate
+      require "syntax_tree/reflection"
+
+      body << Comment("# typed: strict", false, location)
+      @line += 2
+
+      generate_parent
+      Reflection.nodes.sort.each { |(_, node)| generate_node(node) }
+
+      Formatter.format(nil, Program(Statements(body)))
+    end
+
+    private
+
+    def generate_comments(comment)
+      comment
+        .lines(chomp: true)
+        .map { |line| Comment("# #{line}", false, location).tap { @line += 1 } }
+    end
+
+    def generate_parent
+      attribute = Reflection.nodes[:Program].attributes[:location]
+      class_location = location
+
+      node_body = generate_comments(attribute.comment)
+      node_body << sig_block { sig_returns { sig_type_for(attribute.type) } }
+      @line += 1
+
+      node_body << Command(
+        Ident("attr_reader"),
+        Args([SymbolLiteral(Ident("location"))]),
+        nil,
+        location
+      )
+      @line += 1
+
+      body << ClassDeclaration(
+        ConstPathRef(VarRef(Const("SyntaxTree")), Const("Node")),
+        nil,
+        BodyStmt(Statements(node_body), nil, nil, nil, nil),
+        class_location
+      )
+      @line += 2
+    end
+
+    def generate_node(node)
+      body.concat(generate_comments(node.comment))
+      class_location = location
+      @line += 2
+
+      body << ClassDeclaration(
+        ConstPathRef(VarRef(Const("SyntaxTree")), Const(node.name.to_s)),
+        ConstPathRef(VarRef(Const("SyntaxTree")), Const("Node")),
+        BodyStmt(Statements(generate_node_body(node)), nil, nil, nil, nil),
+        class_location
+      )
+
+      @line += 2
+    end
+
+    def generate_node_body(node)
+      node_body = []
+      node.attributes.sort.each do |(name, attribute)|
+        next if name == :location
+
+        node_body.concat(generate_comments(attribute.comment))
+        node_body << sig_block { sig_returns { sig_type_for(attribute.type) } }
+        @line += 1
+
+        node_body << Command(
+          Ident("attr_reader"),
+          Args([SymbolLiteral(Ident(attribute.name.to_s))]),
+          nil,
+          location
+        )
+        @line += 2
+      end
+
+      node_body.concat(generate_initialize(node))
+
+      node_body << sig_block do
+        CallNode(
+          sig_params do
+            BareAssocHash(
+              [Assoc(Label("visitor:"), sig_type_for(BasicVisitor))]
+            )
+          end,
+          Period("."),
+          Ident("returns"),
+          ArgParen(
+            Args(
+              [CallNode(VarRef(Const("T")), Period("."), Ident("untyped"), nil)]
+            )
+          )
+        )
+      end
+      @line += 1
+
+      node_body << generate_def_node(
+        "accept",
+        Paren(
+          LParen("("),
+          Params.new(requireds: [Ident("visitor")], location: location)
+        )
+      )
+      @line += 2
+
+      node_body << generate_child_nodes
+      @line += 1
+
+      node_body << generate_def_node("child_nodes", nil)
+      @line += 1
+
+      node_body
+    end
+
+    def generate_initialize(node)
+      parameters =
+        SyntaxTree.const_get(node.name).instance_method(:initialize).parameters
+
+      assocs =
+        parameters.map do |(_, name)|
+          Assoc(Label("#{name}:"), sig_type_for(node.attributes[name].type))
+        end
+
+      node_body = []
+      node_body << sig_block do
+        CallNode(
+          sig_params { BareAssocHash(assocs) },
+          Period("."),
+          Ident("void"),
+          nil
+        )
+      end
+      @line += 1
+
+      params = Params.new(location: location)
+      parameters.each do |(type, name)|
+        case type
+        when :req
+          params.requireds << Ident(name.to_s)
+        when :keyreq
+          params.keywords << [Label("#{name}:"), nil]
+        when :key
+          params.keywords << [
+            Label("#{name}:"),
+            CallNode(
+              VarRef(Const("T")),
+              Period("."),
+              Ident("unsafe"),
+              ArgParen(Args([VarRef(Kw("nil"))]))
+            )
+          ]
+        else
+          raise
+        end
+      end
+
+      node_body << generate_def_node("initialize", Paren(LParen("("), params))
+      @line += 2
+
+      node_body
+    end
+
+    def generate_child_nodes
+      type =
+        Reflection::Type::ArrayType.new(
+          Reflection::Type::UnionType.new([NilClass, Node])
+        )
+
+      sig_block { sig_returns { sig_type_for(type) } }
+    end
+
+    def generate_def_node(name, params)
+      DefNode(
+        nil,
+        nil,
+        Ident(name),
+        params,
+        BodyStmt(Statements([VoidStmt()]), nil, nil, nil, nil),
+        location
+      )
+    end
+
+    def sig_block
+      MethodAddBlock(
+        CallNode(nil, nil, Ident("sig"), nil),
+        BlockNode(
+          LBrace("{"),
+          nil,
+          BodyStmt(Statements([yield]), nil, nil, nil, nil)
+        ),
+        location
+      )
+    end
+
+    def sig_params
+      CallNode(nil, nil, Ident("params"), ArgParen(Args([yield])))
+    end
+
+    def sig_returns
+      CallNode(nil, nil, Ident("returns"), ArgParen(Args([yield])))
+    end
+
+    def sig_type_for(type)
+      case type
+      when Reflection::Type::ArrayType
+        ARef(
+          ConstPathRef(VarRef(Const("T")), Const("Array")),
+          sig_type_for(type.type)
+        )
+      when Reflection::Type::TupleType
+        ArrayLiteral(LBracket("["), Args(type.types.map { sig_type_for(_1) }))
+      when Reflection::Type::UnionType
+        if type.types.include?(NilClass)
+          selected = type.types.reject { _1 == NilClass }
+          subtype =
+            if selected.size == 1
+              selected.first
+            else
+              Reflection::Type::UnionType.new(selected)
+            end
+
+          CallNode(
+            VarRef(Const("T")),
+            Period("."),
+            Ident("nilable"),
+            ArgParen(Args([sig_type_for(subtype)]))
+          )
+        else
+          CallNode(
+            VarRef(Const("T")),
+            Period("."),
+            Ident("any"),
+            ArgParen(Args(type.types.map { sig_type_for(_1) }))
+          )
+        end
+      when Symbol
+        ConstRef(Const("Symbol"))
+      else
+        *parents, constant = type.name.split("::").map { Const(_1) }
+
+        if parents.empty?
+          ConstRef(constant)
+        else
+          [*parents[1..], constant].inject(
+            VarRef(parents.first)
+          ) { |accum, const| ConstPathRef(accum, const) }
+        end
+      end
+    end
+
+    def location
+      Location.fixed(line: line, char: 0, column: 0)
+    end
+  end
+end
+
+namespace :sorbet do
+  desc "Generate RBI files for Sorbet"
+  task :rbi do
+    puts SyntaxTree::RBI.new.generate
+  end
+end

data/tasks/whitequark.rake

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# This file's purpose is to extract the examples from the whitequark/parser
+# gem and generate a test file that we can use to ensure that our parser
+# generates equivalent syntax trees when translating. To do this, it runs the
+# parser's test suite but overrides the `assert_parses` method to collect the
+# examples into a hash. Then, it writes out the hash to a file that we can use
+# to generate our own tests.
+#
+# To run the test suite, it's important to note that we have to mirror both any
+# APIs provided to the test suite (for example the ParseHelper module below).
+# This is obviously relatively brittle, but it's effective for now.
+
+require "ast"
+
+module ParseHelper
+  # This object is going to collect all of the examples from the parser gem into
+  # a hash that we can use to generate our own tests.
+  COLLECTED = Hash.new { |hash, key| hash[key] = [] }
+
+  include AST::Sexp
+  ALL_VERSIONS = %w[3.1 3.2]
+
+  private
+
+  def assert_context(*)
+  end
+
+  def assert_diagnoses(*)
+  end
+
+  def assert_diagnoses_many(*)
+  end
+
+  def refute_diagnoses(*)
+  end
+
+  def with_versions(*)
+  end
+
+  def assert_parses(_ast, code, _source_maps = "", versions = ALL_VERSIONS)
+    # We're going to skip any examples that are for older Ruby versions
+    # that we do not support.
+    return if (versions & %w[3.1 3.2]).empty?
+
+    entry = caller.find { _1.include?("test_parser.rb") }
+    _, lineno, name = *entry.match(/(\d+):in `(.+)'/)
+
+    COLLECTED["#{name}:#{lineno}"] << code
+  end
+end
+
+namespace :extract do
+  desc "Extract the whitequark/parser tests"
+  task :whitequark do
+    directory = File.expand_path("../tmp/parser", __dir__)
+    unless File.directory?(directory)
+      sh "git clone --depth 1 https://github.com/whitequark/parser #{directory}"
+    end
+
+    mkdir_p "#{directory}/extract"
+    touch "#{directory}/extract/helper.rb"
+    touch "#{directory}/extract/parse_helper.rb"
+    touch "#{directory}/extract/extracted.txt"
+    $:.unshift "#{directory}/extract"
+
+    require "parser/current"
+    require "minitest/autorun"
+    require_relative "#{directory}/test/test_parser"
+
+    Minitest.after_run do
+      filepath = File.expand_path("../test/translation/parser.txt", __dir__)
+
+      File.open(filepath, "w") do |file|
+        ParseHelper::COLLECTED.sort.each do |(key, codes)|
+          if codes.length == 1
+            file.puts("!!! #{key}\n#{codes.first}")
+          else
+            codes.each_with_index do |code, index|
+              file.puts("!!! #{key}:#{index}\n#{code}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: syntax_tree
 version: !ruby/object:Gem::Version
-  version: 5.3.0
+  version: 6.0.0
 platform: ruby
 authors:
 - Kevin Newton
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-01-26 00:00:00.000000000 Z
+date: 2023-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: prettier_print
@@ -109,7 +109,6 @@ files:
 - ".github/workflows/gh-pages.yml"
 - ".github/workflows/main.yml"
 - ".gitignore"
-- ".gitmodules"
 - ".rubocop.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -121,7 +120,9 @@ files:
 - bin/bench
 - bin/console
 - bin/profile
+- bin/whitequark
 - config/rubocop.yml
+- doc/changing_structure.md
 - doc/logo.svg
 - exe/stree
 - exe/yarv
@@ -129,42 +130,53 @@ files:
 - lib/syntax_tree/basic_visitor.rb
 - lib/syntax_tree/cli.rb
 - lib/syntax_tree/dsl.rb
+- lib/syntax_tree/field_visitor.rb
 - lib/syntax_tree/formatter.rb
 - lib/syntax_tree/index.rb
+- lib/syntax_tree/json_visitor.rb
 - lib/syntax_tree/language_server.rb
-- lib/syntax_tree/language_server/inlay_hints.rb
+- lib/syntax_tree/match_visitor.rb
+- lib/syntax_tree/mermaid.rb
+- lib/syntax_tree/mermaid_visitor.rb
+- lib/syntax_tree/mutation_visitor.rb
 - lib/syntax_tree/node.rb
 - lib/syntax_tree/parser.rb
 - lib/syntax_tree/pattern.rb
 - lib/syntax_tree/plugin/disable_ternary.rb
 - lib/syntax_tree/plugin/single_quotes.rb
 - lib/syntax_tree/plugin/trailing_comma.rb
+- lib/syntax_tree/pretty_print_visitor.rb
 - lib/syntax_tree/rake/check_task.rb
 - lib/syntax_tree/rake/task.rb
 - lib/syntax_tree/rake/write_task.rb
 - lib/syntax_tree/rake_tasks.rb
+- lib/syntax_tree/reflection.rb
 - lib/syntax_tree/search.rb
+- lib/syntax_tree/translation.rb
+- lib/syntax_tree/translation/parser.rb
+- lib/syntax_tree/translation/rubocop_ast.rb
 - lib/syntax_tree/version.rb
 - lib/syntax_tree/visitor.rb
-- lib/syntax_tree/visitor/environment.rb
-- lib/syntax_tree/visitor/field_visitor.rb
-- lib/syntax_tree/visitor/json_visitor.rb
-- lib/syntax_tree/visitor/match_visitor.rb
-- lib/syntax_tree/visitor/mutation_visitor.rb
-- lib/syntax_tree/visitor/pretty_print_visitor.rb
-- lib/syntax_tree/visitor/with_environment.rb
+- lib/syntax_tree/with_scope.rb
 - lib/syntax_tree/yarv.rb
 - lib/syntax_tree/yarv/assembler.rb
+- lib/syntax_tree/yarv/basic_block.rb
 - lib/syntax_tree/yarv/bf.rb
+- lib/syntax_tree/yarv/calldata.rb
 - lib/syntax_tree/yarv/compiler.rb
+- lib/syntax_tree/yarv/control_flow_graph.rb
+- lib/syntax_tree/yarv/data_flow_graph.rb
 - lib/syntax_tree/yarv/decompiler.rb
 - lib/syntax_tree/yarv/disassembler.rb
 - lib/syntax_tree/yarv/instruction_sequence.rb
 - lib/syntax_tree/yarv/instructions.rb
 - lib/syntax_tree/yarv/legacy.rb
 - lib/syntax_tree/yarv/local_table.rb
+- lib/syntax_tree/yarv/sea_of_nodes.rb
 - lib/syntax_tree/yarv/vm.rb
 - syntax_tree.gemspec
+- tasks/sorbet.rake
+- tasks/whitequark.rake
 homepage: https://github.com/kddnewton/syntax_tree
 licenses:
 - MIT

data/.gitmodules

@@ -1,9 +0,0 @@
-[submodule "mspec"]
-	path = spec/mspec
-	url = git@github.com:ruby/mspec.git
-[submodule "spec"]
-	path = spec/ruby
-	url = git@github.com:ruby/spec.git
-[submodule "test/ruby-syntax-fixtures"]
-	path = test/ruby-syntax-fixtures
-	url = https://github.com/ruby-syntax-tree/ruby-syntax-fixtures