syntax_tree 5.1.0 → 5.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0adfab4347f7d3abda07479e427b2df975e98fb9dafa731cb8b2188da42ab18b
-  data.tar.gz: 4d730d2eaadfa61585e683f6ad444b40336787a561ac7f08c630d5aef857d3d7
+  metadata.gz: 1c5651bab4283bf0f1c451619e5c3b54c9b229d1fb5050a3732918bccb7be177
+  data.tar.gz: 2cda5e4b62a3beac16dddc8ff9298624c161bc99c3b82ba2ecc10d8d643aac4b
 SHA512:
-  metadata.gz: 890fdc12c1d6d0643231a908bb0f1a503d365fc56d1349f36dc99497eff6a30e51309ecce2cb19e10159e3dd1a1f811a8fd8c3904c94943992ff069811f85ced
-  data.tar.gz: f1e546cd8397b62f4c518103b18c712fead74c82e23ecb527909727a91f7556cddc9886c730be8a968e957b64addb79c9665869e6fcb57b1d26b1b7f13f5bfa5
+  metadata.gz: 81d331f10356569b23ada917cea2ecf5a2dd1073bc927e39ba0b6036ea4b79813ce7b1e4826f39d919559cdbb5925da8a5680d72653d64d0eb05979729356173
+  data.tar.gz: 53b320f073ef5e15efcd2e7d706e959e3ade48f18ab0f131a588a183276aeea3352cf6da0ef2ad842dc0c9bf517ac4c441ee4fbe98fa73dd156e845d5d4007d7

data/.github/dependabot.yml

@@ -4,3 +4,7 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

data/.github/workflows/auto-merge.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1.3.3
+        uses: dependabot/fetch-metadata@v1.3.5
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Enable auto-merge for Dependabot PRs

data/.github/workflows/main.yml

@@ -1,7 +1,9 @@
 name: Main
+
 on:
 - push
 - pull_request
+
 jobs:
   ci:
     strategy:
@@ -11,13 +13,14 @@ jobs:
         - '2.7.0'
         - '3.0'
         - '3.1'
+        - '3.2'
         - head
         - truffleruby-head
     name: CI
     runs-on: ubuntu-latest
     env:
       CI: true
-      TESTOPTS: --verbose
+      # TESTOPTS: --verbose
     steps:
     - uses: actions/checkout@master
     - uses: ruby/setup-ruby@v1
@@ -37,7 +40,7 @@ jobs:
     - uses: ruby/setup-ruby@v1
       with:
         bundler-cache: true
-        ruby-version: '3.1'
+        ruby-version: '3.2'
     - name: Check
       run: |
         bundle exec rake stree:check

data/.gitmodules

@@ -0,0 +1,6 @@
+[submodule "mspec"]
+	path = spec/mspec
+	url = git@github.com:ruby/mspec.git
+[submodule "spec"]
+	path = spec/ruby
+	url = git@github.com:ruby/spec.git

data/.rubocop.yml

@@ -7,7 +7,7 @@ AllCops:
   SuggestExtensions: false
   TargetRubyVersion: 2.7
   Exclude:
-  - '{.git,.github,bin,coverage,pkg,test/fixtures,vendor,tmp}/**/*'
+  - '{.git,.github,bin,coverage,pkg,spec,test/fixtures,vendor,tmp}/**/*'
   - test.rb
 
 Layout/LineLength:
@@ -43,6 +43,9 @@ Lint/NonLocalExitFromIterator:
 Lint/RedundantRequireStatement:
   Enabled: false
 
+Lint/RescueException:
+  Enabled: false
+
 Lint/SuppressedException:
   Enabled: false
 
@@ -144,3 +147,6 @@ Style/SpecialGlobalVars:
 
 Style/StructInheritance:
   Enabled: false
+
+Style/YodaExpression:
+  Enabled: false

data/CHANGELOG.md

@@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [5.2.0] - 2023-01-04
+
+### Added
+
+- An experiment in evaluating compiled instruction sequences has been added to Syntax Tree. This is subject to change, so it will not be well documented or testing at the moment. It does not impact other functionality.
+
+### Changed
+
+- Empty parentheses on method calls will now be left in place. Previously they were left in place if the method being called looked like a constant. Now they are left in place for all method calls since the method name can mirror the name of a local variable, in which case the parentheses are required.
+
 ## [5.1.0] - 2022-12-28
 
 ### Added
@@ -471,7 +481,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...HEAD
+[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.2.0...HEAD
+[5.2.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.1.0...v5.2.0
 [5.1.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.1...v5.1.0
 [5.0.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v5.0.0...v5.0.1
 [5.0.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v4.3.0...v5.0.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    syntax_tree (5.1.0)
+    syntax_tree (5.2.0)
       prettier_print (>= 1.2.0)
 
 GEM
@@ -10,26 +10,26 @@ GEM
     ast (2.4.2)
     docile (1.4.0)
     json (2.6.3)
-    minitest (5.16.3)
+    minitest (5.17.0)
     parallel (1.22.1)
-    parser (3.1.3.0)
+    parser (3.2.0.0)
       ast (~> 2.4.1)
     prettier_print (1.2.0)
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.6.1)
     rexml (3.2.5)
-    rubocop (1.41.1)
+    rubocop (1.42.0)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.23.0, < 2.0)
+      rubocop-ast (>= 1.24.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.24.0)
+    rubocop-ast (1.24.1)
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     simplecov (0.22.0)
@@ -38,7 +38,7 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    unicode-display_width (2.3.0)
+    unicode-display_width (2.4.1)
 
 PLATFORMS
   arm64-darwin-21

data/Rakefile

@@ -26,3 +26,10 @@ end
 
 SyntaxTree::Rake::CheckTask.new(&configure)
 SyntaxTree::Rake::WriteTask.new(&configure)
+
+desc "Run mspec tests using YARV emulation"
+task :spec do
+  Dir["./spec/ruby/language/**/*_spec.rb"].each do |filepath|
+    sh "exe/yarv ./spec/mspec/bin/mspec-tag #{filepath}"
+  end
+end

data/exe/yarv

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$:.unshift(File.expand_path("../lib", __dir__))
+
+require "syntax_tree"
+
+# Require these here so that we can run binding.irb without having them require
+# anything that we've already patched.
+require "irb"
+require "irb/completion"
+require "irb/color_printer"
+require "readline"
+
+# First, create an instance of our virtual machine.
+events =
+  if ENV["DEBUG"]
+    SyntaxTree::YARV::VM::STDOUTEvents.new
+  else
+    SyntaxTree::YARV::VM::NullEvents.new
+  end
+
+vm = SyntaxTree::YARV::VM.new(events)
+
+# Next, set up a bunch of aliases for methods that we're going to hook into in
+# order to set up our virtual machine.
+class << Kernel
+  alias yarv_require require
+  alias yarv_require_relative require_relative
+  alias yarv_load load
+  alias yarv_eval eval
+  alias yarv_throw throw
+  alias yarv_catch catch
+end
+
+# Next, patch the methods that we just aliased so that they use our virtual
+# machine's versions instead. This allows us to load Ruby files and have them
+# execute in our virtual machine instead of the runtime environment.
+[Kernel, Kernel.singleton_class].each do |klass|
+  klass.define_method(:require) { |filepath| vm.require(filepath) }
+
+  klass.define_method(:load) { |filepath| vm.load(filepath) }
+
+  # klass.define_method(:require_relative) do |filepath|
+  #   vm.require_relative(filepath)
+  # end
+
+  # klass.define_method(:eval) do |
+  #   source,
+  #   binding = TOPLEVEL_BINDING,
+  #   filename = "(eval)",
+  #   lineno = 1
+  # |
+  #   vm.eval(source, binding, filename, lineno)
+  # end
+
+  # klass.define_method(:throw) { |tag, value = nil| vm.throw(tag, value) }
+
+  # klass.define_method(:catch) { |tag, &block| vm.catch(tag, &block) }
+end
+
+# Finally, require the file that we want to execute.
+vm.require_resolved(ARGV.shift)

data/lib/syntax_tree/node.rb

@@ -3001,16 +3001,25 @@ module SyntaxTree
       else
         q.format(message)
 
-        if arguments.is_a?(ArgParen) && arguments.arguments.nil? &&
-             !message.is_a?(Const)
-          # If you're using an explicit set of parentheses on something that
-          # looks like a constant, then we need to match that in order to
-          # maintain valid Ruby. For example, you could do something like Foo(),
-          # on which we would need to keep the parentheses to make it look like
-          # a method call.
-        else
-          q.format(arguments)
-        end
+        # Note that this explicitly leaves parentheses in place even if they are
+        # empty. There are two reasons we would need to do this. The first is if
+        # we're calling something that looks like a constant, as in:
+        #
+        #     Foo()
+        #
+        # In this case if we remove the parentheses then this becomes a constant
+        # reference and not a method call. The second is if we're calling a
+        # method that is the same name as a local variable that is in scope, as
+        # in:
+        #
+        #     foo = foo()
+        #
+        # In this case we have to keep the parentheses or else it treats this
+        # like assigning nil to the local variable. Note that we could attempt
+        # to be smarter about this by tracking the local variables that are in
+        # scope, but for now it's simpler and more efficient to just leave the
+        # parentheses in place.
+        q.format(arguments) if arguments
       end
     end
 

data/lib/syntax_tree/parser.rb

@@ -53,7 +53,7 @@ module SyntaxTree
       # there's a BOM at the beginning of the file, which is the reason we need
       # to compare it to 0 here.
       def [](byteindex)
-        indices[byteindex < 0 ? 0 : byteindex]
+        indices[[byteindex, 0].max]
       end
     end
 

data/lib/syntax_tree/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SyntaxTree
-  VERSION = "5.1.0"
+  VERSION = "5.2.0"
 end

data/lib/syntax_tree/yarv/assembler.rb

@@ -69,7 +69,7 @@ module SyntaxTree
       end
 
       def assemble
-        iseq = InstructionSequence.new(:top, "<main>", nil, Location.default)
+        iseq = InstructionSequence.new("<main>", "<compiled>", 1, :top)
         assemble_iseq(iseq, File.readlines(filepath, chomp: true))
 
         iseq.compile!
@@ -138,7 +138,7 @@ module SyntaxTree
             name = parse_symbol(name_value)
             flags = parse_number(flags_value)
 
-            class_iseq = iseq.class_child_iseq(name.to_s, Location.default)
+            class_iseq = iseq.class_child_iseq(name.to_s, 1)
             assemble_iseq(class_iseq, body)
             iseq.defineclass(name, class_iseq, flags)
           when "defined"
@@ -153,7 +153,7 @@ module SyntaxTree
             line_index += body.length
 
             name = parse_symbol(operands)
-            method_iseq = iseq.method_child_iseq(name.to_s, Location.default)
+            method_iseq = iseq.method_child_iseq(name.to_s, 1)
             assemble_iseq(method_iseq, body)
 
             iseq.definemethod(name, method_iseq)
@@ -162,7 +162,7 @@ module SyntaxTree
             line_index += body.length
 
             name = parse_symbol(operands)
-            method_iseq = iseq.method_child_iseq(name.to_s, Location.default)
+            method_iseq = iseq.method_child_iseq(name.to_s, 1)
 
             assemble_iseq(method_iseq, body)
             iseq.definesmethod(name, method_iseq)
@@ -221,7 +221,7 @@ module SyntaxTree
                 body = parse_nested(lines[line_index..])
                 line_index += body.length
 
-                block_iseq = iseq.block_child_iseq(Location.default)
+                block_iseq = iseq.block_child_iseq(1)
                 assemble_iseq(block_iseq, body)
                 block_iseq
               end
@@ -249,7 +249,7 @@ module SyntaxTree
                 body = parse_nested(lines[line_index..])
                 line_index += body.length
 
-                block_iseq = iseq.block_child_iseq(Location.default)
+                block_iseq = iseq.block_child_iseq(1)
                 assemble_iseq(block_iseq, body)
                 block_iseq
               end
@@ -354,7 +354,7 @@ module SyntaxTree
                 body = parse_nested(lines[line_index..])
                 line_index += body.length
 
-                block_iseq = iseq.block_child_iseq(Location.default)
+                block_iseq = iseq.block_child_iseq(1)
                 assemble_iseq(block_iseq, body)
                 block_iseq
               end

data/lib/syntax_tree/yarv/bf.rb

@@ -13,7 +13,7 @@ module SyntaxTree
 
       def compile
         # Set up the top-level instruction sequence that will be returned.
-        iseq = InstructionSequence.new(:top, "<compiled>", nil, location)
+        iseq = InstructionSequence.new("<compiled>", "<compiled>", 1, :top)
 
         # Set up the $tape global variable that will hold our state.
         iseq.duphash({ 0 => 0 })
@@ -80,19 +80,6 @@ module SyntaxTree
 
       private
 
-      # This is the location of the top instruction sequence, derived from the
-      # source string.
-      def location
-        Location.new(
-          start_line: 1,
-          start_char: 0,
-          start_column: 0,
-          end_line: source.count("\n") + 1,
-          end_char: source.size,
-          end_column: source.size - (source.rindex("\n") || 0) - 1
-        )
-      end
-
       # $tape[$cursor] += value
       def change_by(iseq, value)
         iseq.getglobal(:$tape)
@@ -111,6 +98,7 @@ module SyntaxTree
         end
 
         iseq.send(YARV.calldata(:[]=, 2))
+        iseq.pop
       end
 
       # $cursor += value
@@ -138,6 +126,7 @@ module SyntaxTree
         iseq.send(YARV.calldata(:chr))
 
         iseq.send(YARV.calldata(:putc, 1))
+        iseq.pop
       end
 
       # $tape[$cursor] = $stdin.getc.ord
@@ -150,6 +139,7 @@ module SyntaxTree
         iseq.send(YARV.calldata(:ord))
 
         iseq.send(YARV.calldata(:[]=, 2))
+        iseq.pop
       end
 
       # unless $tape[$cursor] == 0
@@ -164,14 +154,21 @@ module SyntaxTree
 
         iseq.putobject(0)
         iseq.send(YARV.calldata(:==, 1))
-        iseq.branchunless(end_label)
+        iseq.branchif(end_label)
 
         [start_label, end_label]
       end
 
       # Jump back to the start of the loop.
       def loop_end(iseq, start_label, end_label)
-        iseq.jump(start_label)
+        iseq.getglobal(:$tape)
+        iseq.getglobal(:$cursor)
+        iseq.send(YARV.calldata(:[], 1))
+
+        iseq.putobject(0)
+        iseq.send(YARV.calldata(:==, 1))
+        iseq.branchunless(start_label)
+
         iseq.push(end_label)
       end
     end

data/lib/syntax_tree/yarv/compiler.rb

@@ -304,10 +304,11 @@ module SyntaxTree
       end
 
       def visit_END(node)
+        start_line = node.location.start_line
         once_iseq =
-          with_child_iseq(iseq.block_child_iseq(node.location)) do
+          with_child_iseq(iseq.block_child_iseq(start_line)) do
             postexe_iseq =
-              with_child_iseq(iseq.block_child_iseq(node.location)) do
+              with_child_iseq(iseq.block_child_iseq(start_line)) do
                 iseq.event(:RUBY_EVENT_B_CALL)
 
                 *statements, last_statement = node.statements.body
@@ -567,7 +568,7 @@ module SyntaxTree
       end
 
       def visit_block(node)
-        with_child_iseq(iseq.block_child_iseq(node.location)) do
+        with_child_iseq(iseq.block_child_iseq(node.location.start_line)) do
           iseq.event(:RUBY_EVENT_B_CALL)
           visit(node.block_var)
           visit(node.bodystmt)
@@ -751,7 +752,9 @@ module SyntaxTree
       def visit_class(node)
         name = node.constant.constant.value.to_sym
         class_iseq =
-          with_child_iseq(iseq.class_child_iseq(name, node.location)) do
+          with_child_iseq(
+            iseq.class_child_iseq(name, node.location.start_line)
+          ) do
             iseq.event(:RUBY_EVENT_CLASS)
             visit(node.bodystmt)
             iseq.event(:RUBY_EVENT_END)
@@ -818,7 +821,8 @@ module SyntaxTree
 
       def visit_def(node)
         name = node.name.value.to_sym
-        method_iseq = iseq.method_child_iseq(name.to_s, node.location)
+        method_iseq =
+          iseq.method_child_iseq(name.to_s, node.location.start_line)
 
         with_child_iseq(method_iseq) do
           visit(node.params) if node.params
@@ -939,7 +943,9 @@ module SyntaxTree
         iseq.local_table.plain(name)
 
         block_iseq =
-          with_child_iseq(iseq.block_child_iseq(node.statements.location)) do
+          with_child_iseq(
+            iseq.block_child_iseq(node.statements.location.start_line)
+          ) do
             iseq.argument_options[:lead_num] ||= 0
             iseq.argument_options[:lead_num] += 1
             iseq.argument_options[:ambiguous_param0] = true
@@ -1076,7 +1082,7 @@ module SyntaxTree
 
       def visit_lambda(node)
         lambda_iseq =
-          with_child_iseq(iseq.block_child_iseq(node.location)) do
+          with_child_iseq(iseq.block_child_iseq(node.location.start_line)) do
             iseq.event(:RUBY_EVENT_B_CALL)
             visit(node.params)
             visit(node.statements)
@@ -1127,7 +1133,9 @@ module SyntaxTree
       def visit_module(node)
         name = node.constant.constant.value.to_sym
         module_iseq =
-          with_child_iseq(iseq.module_child_iseq(name, node.location)) do
+          with_child_iseq(
+            iseq.module_child_iseq(name, node.location.start_line)
+          ) do
             iseq.event(:RUBY_EVENT_CLASS)
             visit(node.bodystmt)
             iseq.event(:RUBY_EVENT_END)
@@ -1375,10 +1383,11 @@ module SyntaxTree
 
         top_iseq =
           InstructionSequence.new(
-            :top,
             "<compiled>",
+            "<compiled>",
+            1,
+            :top,
             nil,
-            node.location,
             options
           )
 
@@ -1543,7 +1552,9 @@ module SyntaxTree
         iseq.putnil
 
         singleton_iseq =
-          with_child_iseq(iseq.singleton_class_child_iseq(node.location)) do
+          with_child_iseq(
+            iseq.singleton_class_child_iseq(node.location.start_line)
+          ) do
             iseq.event(:RUBY_EVENT_CLASS)
             visit(node.bodystmt)
             iseq.event(:RUBY_EVENT_END)
@@ -2018,7 +2029,7 @@ module SyntaxTree
           if node.constant
             iseq.dup
             visit(node.constant)
-            iseq.checkmatch(CheckMatch::TYPE_CASE)
+            iseq.checkmatch(CheckMatch::VM_CHECKMATCH_TYPE_CASE)
             iseq.branchunless(match_failure_label)
           end
 
@@ -2078,7 +2089,7 @@ module SyntaxTree
               iseq.setlocal(lookup.index, lookup.level)
             else
               visit(required)
-              iseq.checkmatch(CheckMatch::TYPE_CASE)
+              iseq.checkmatch(CheckMatch::VM_CHECKMATCH_TYPE_CASE)
               iseq.branchunless(match_failure_label)
             end
 

data/lib/syntax_tree/yarv/decompiler.rb

@@ -64,6 +64,13 @@ module SyntaxTree
             clauses[label] = clause
             clause = []
             label = insn.name
+          when BranchIf
+            body = [
+              Assign(block_label.field, node_for(insn.label.name)),
+              Next(Args([]))
+            ]
+
+            clause << UnlessNode(clause.pop, Statements(body), nil)
           when BranchUnless
             body = [
               Assign(block_label.field, node_for(insn.label.name)),
@@ -157,6 +164,8 @@ module SyntaxTree
                 )
               end
             end
+          when Pop
+            # skip
           when PutObject
             case insn.object
             when Float

data/lib/syntax_tree/yarv/disassembler.rb

@@ -4,7 +4,8 @@ module SyntaxTree
   module YARV
     class Disassembler
       attr_reader :output, :queue
-      attr_reader :current_prefix, :current_iseq
+      attr_reader :current_prefix
+      attr_accessor :current_iseq
 
       def initialize
         @output = StringIO.new
@@ -114,7 +115,7 @@ module SyntaxTree
         output << "#{current_prefix}== disasm: "
         output << "#<ISeq:#{iseq.name}@<compiled>:1 "
 
-        location = iseq.location
+        location = Location.fixed(line: iseq.line, char: 0, column: 0)
         output << "(#{location.start_line},#{location.start_column})-"
         output << "(#{location.end_line},#{location.end_column})"
         output << "> "