ruby-next-core 0.15.3 → 1.0.0.rc.1

data/lib/ruby-next/language/rewriters/base.rb

@@ -13,7 +13,7 @@ module RubyNext
 
       MSG
 
-      class Base < ::Parser::TreeRewriter
+      class Base < Abstract
         class LocalsTracker
           using(Module.new do
             refine ::Parser::AST::Node do
@@ -65,39 +65,15 @@ module RubyNext
           end
         end
 
-        class << self
-          # Returns true if the syntax is not supported
-          # by the current Ruby (performs syntax check, not version check)
-          def unsupported_syntax?
-            save_verbose, $VERBOSE = $VERBOSE, nil
-            eval_mid = Kernel.respond_to?(:eval_without_ruby_next) ? :eval_without_ruby_next : :eval
-            Kernel.send eval_mid, self::SYNTAX_PROBE, nil, __FILE__, __LINE__
-            false
-          rescue SyntaxError, StandardError
-            true
-          ensure
-            $VERBOSE = save_verbose
-          end
-
-          # Returns true if the syntax is supported
-          # by the specified version
-          def unsupported_version?(version)
-            self::MIN_SUPPORTED_VERSION > version
-          end
-
-          private
+        attr_reader :locals
 
-          def transform(source)
-            Language.transform(source, rewriters: [self], using: false)
-          end
+        def self.ast?
+          true
         end
 
-        attr_reader :locals
-
-        def initialize(context)
-          @context = context
+        def initialize(*args)
           @locals = LocalsTracker.new
-          super()
+          super
         end
 
         def s(type, *children)
@@ -145,8 +121,6 @@ module RubyNext
 
           Unparser.unparse(ast).chomp
         end
-
-        attr_reader :context
       end
     end
   end

data/lib/ruby-next/language/rewriters/edge/it_param.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module RubyNext
+  module Language
+    module Rewriters
+      class ItParam < Base
+        using RubyNext
+
+        NAME = "it-param"
+        SYNTAX_PROBE = "proc { it.keys }.call({})"
+        MIN_SUPPORTED_VERSION = Gem::Version.new("3.4.0")
+
+        def on_block(node)
+          proc_or_lambda, args, body = *node.children
+
+          return super unless block_has_it?(body)
+
+          context.track! self
+
+          new_body = s(:begin,
+            s(:lvasgn, :it, s(:lvar, :_1)),
+            body)
+
+          insert_before(body.loc.expression, "it = _1;")
+
+          process(
+            node.updated(:numblock, [
+              proc_or_lambda,
+              args,
+              new_body
+            ])
+          )
+        end
+
+        private
+
+        # It's important to check if the current block refers to `it` variable somewhere
+        # (and not within a nested block), so we don't declare numbered params
+        def block_has_it?(node)
+          # traverse node children deeply
+          tree = [node]
+
+          while (child = tree.shift)
+            return true if it?(child)
+
+            if child.is_a?(Parser::AST::Node)
+              tree.unshift(*child.children.select { |c| c.is_a?(Parser::AST::Node) && c.type != :block && c.type != :numblock })
+            end
+          end
+        end
+
+        def it?(node)
+          node.is_a?(Parser::AST::Node) && node.type == :send && node.children[0].nil? && node.children[1] == :it && node.children[2].nil?
+        end
+      end
+    end
+  end
+end

data/lib/ruby-next/language/rewriters/edge.rb

@@ -1,3 +1,15 @@
 # frozen_string_literal: true
 
 # Load edge Ruby features
+
+require "ruby-next/language/rewriters/edge/it_param"
+
+# We must add this rewriter before nubmered params rewriter to allow it to transform the source code further
+
+number_params = RubyNext::Language.rewriters.index(RubyNext::Language::Rewriters::NumberedParams)
+
+if number_params
+  RubyNext::Language.rewriters.insert(number_params, RubyNext::Language::Rewriters::ItParam)
+else
+  RubyNext::Language.rewriters << RubyNext::Language::Rewriters::ItParam
+end

data/lib/ruby-next/language/rewriters/proposed/bind_vars_pattern.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "parser/rubynext"
+RubyNext::Language.parser_class = ::Parser::RubyNext
+
 module RubyNext
   module Language
     module Rewriters

data/lib/ruby-next/language/rewriters/proposed/method_reference.rb

@@ -1,33 +1,22 @@
 # frozen_string_literal: true
 
+require "parser/rubynext"
+RubyNext::Language.parser_class = ::Parser::RubyNext
+
 module RubyNext
   module Language
     module Rewriters
-      class MethodReference < Base
+      class MethodReference < Text
         NAME = "method-reference"
         SYNTAX_PROBE = "Language.:transform"
         MIN_SUPPORTED_VERSION = Gem::Version.new(RubyNext::NEXT_VERSION)
 
-        def on_meth_ref(node)
-          context.track! self
-
-          receiver, mid = *node.children
-
-          replace(
-            node.children.first.loc.expression.end.join(
-              node.loc.expression.end
-            ),
-            ".method(:#{mid})"
-          )
+        def safe_rewrite(source)
+          source.gsub(/\.:([\w_]+)/) do |match|
+            context.track! self
 
-          node.updated(
-            :send,
-            [
-              receiver,
-              :method,
-              s(:sym, mid)
-            ]
-          )
+            ".method(:#{$1})"
+          end
         end
       end
     end

data/lib/ruby-next/language/rewriters/text.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "ruby-next/language/paco_parser"
+
+module RubyNext
+  module Language
+    module Rewriters
+      class Text < Abstract
+        using RubyNext
+
+        class Normalizer < PacoParsers::Base
+          attr_reader :store
+
+          def initialize
+            @store = []
+          end
+
+          def normalizing(source)
+            many(
+              alt(
+                ruby_comment,
+                ruby_string,
+                ruby_code
+              )
+            ).parse(source, with_callstack: true)
+              .then(&:join)
+              .then do
+                if block_given?
+                  yield _1
+                else
+                  _1
+                end
+              end
+              .then do |new_source|
+              restore(new_source)
+            end
+          end
+
+          def ruby_comment
+            parse_comments.fmap do |result|
+              store << result
+              "# A#{store.size}Я\n"
+            end
+          end
+
+          def ruby_string
+            parse_strings.fmap do |result|
+              result.each_with_object([]) do |(type, str), acc|
+                if type == :literal
+                  store << str
+                  acc << "_A#{store.size}Я_"
+                else
+                  acc << str
+                end
+                acc
+              end.join
+            end
+          end
+
+          def ruby_code
+            any_char
+          end
+
+          def restore(source)
+            source.gsub(/(?:\# |_)A(\d+)Я(?:_|\n)/m) do |*args|
+              store[$1.to_i - 1]
+            end
+          end
+
+          def parse_comments
+            memoize { PacoParsers::Comments.new.default }
+          end
+
+          def parse_strings
+            memoize { PacoParsers::StringLiterals.new.default }
+          end
+        end
+
+        # Base class for rewriting parsers which adds the #track! method
+        class PacoParser < PacoParsers::Base
+          attr_reader :rewriter, :context
+
+          def initialize(rewriter, context)
+            @rewriter = rewriter
+            @context = context
+          end
+
+          def track!
+            context.track!(rewriter)
+          end
+        end
+
+        class << self
+          def parser(&block)
+            @paco_parser = Class.new(PacoParser, &block)
+          end
+
+          def paco_parser
+            return @paco_parser if @paco_parser
+
+            superclass.paco_parser if superclass.respond_to?(:paco_parser)
+          end
+
+          def text?
+            true
+          end
+        end
+
+        # Rewrite source code by ignoring string literals and comments
+        def rewrite(source)
+          Normalizer.new.normalizing(source) do |normalized|
+            safe_rewrite(normalized)
+          end
+        end
+
+        def safe_rewrite(source)
+          source
+        end
+
+        private
+
+        def parse(source)
+          parser_class = self.class.paco_parser
+          raise "No parser defined for #{self.class}" unless parser_class
+
+          paco_parser = self.class.paco_parser.new(self, context)
+          paco_parser.parse(source)
+        end
+      end
+    end
+  end
+end

data/lib/ruby-next/language/runtime.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
-require "pathname"
+require "require-hooks/setup"
 
 require "ruby-next"
-require "ruby-next/utils"
 require "ruby-next/language"
 require "ruby-next/language/eval"
 
@@ -16,102 +15,26 @@ module RubyNext
       using RubyNext
 
       class << self
-        include Utils
-
-        def load(path, wrap: false)
-          raise "RubyNext cannot handle `load(smth, wrap: true)`" if wrap
-
-          contents = File.read(path)
+        def load(path, contents)
+          contents ||= File.read(path)
           new_contents = transform contents
 
           RubyNext.debug_source new_contents, path
 
-          evaluate(new_contents, path)
-          true
+          new_contents
         end
 
         def transform(contents, **options)
           Language.transform(contents, rewriters: Language.current_rewriters, **options)
         end
-
-        def feature_path(path)
-          path = resolve_feature_path(path)
-          return if path.nil?
-          return if File.extname(path) != ".rb"
-          return unless Language.transformable?(path)
-          path
-        end
-
-        if defined?(JRUBY_VERSION) || defined?(TruffleRuby)
-          def evaluate(code, filepath)
-            new_toplevel.eval(code, filepath)
-          end
-
-          def new_toplevel
-            # Create new "toplevel" binding to avoid lexical scope re-use
-            # (aka "leaking refinements")
-            eval "proc{binding}.call", TOPLEVEL_BINDING, __FILE__, __LINE__
-          end
-        else
-          def evaluate(code, filepath)
-            # This is workaround to solve the "leaking refinements" problem in MRI
-            RubyVM::InstructionSequence.compile(code, filepath).then do |iseq|
-              iseq.eval
-            end
-          end
-        end
       end
     end
   end
 end
 
-# Patch Kernel to hijack require/require_relative/load/eval
-module Kernel
-  module_function
-
-  alias_method :require_without_ruby_next, :require
-  def require(path)
-    realpath = RubyNext::Language::Runtime.feature_path(path)
-    return require_without_ruby_next(path) unless realpath
-
-    return false if $LOADED_FEATURES.include?(realpath)
-
-    $LOADED_FEATURES << realpath
-
-    RubyNext::Language::Runtime.load(realpath)
-
-    true
-  rescue => e
-    $LOADED_FEATURES.delete realpath
-    RubyNext.warn "RubyNext failed to require '#{path}': #{e.message}"
-    require_without_ruby_next(path)
-  end
-
-  alias_method :require_relative_without_ruby_next, :require_relative
-  def require_relative(path)
-    loc = caller_locations(1..1).first
-    from = loc.absolute_path || loc.path || File.join(Dir.pwd, "main")
-    realpath = File.absolute_path(
-      File.join(
-        File.dirname(File.absolute_path(from)),
-        path
-      )
-    )
-    require(realpath)
-  rescue => e
-    RubyNext.warn "RubyNext failed to require relative '#{path}' from #{from}: #{e.message}"
-    require_relative_without_ruby_next(path)
-  end
-
-  alias_method :load_without_ruby_next, :load
-  def load(path, wrap = false)
-    realpath = RubyNext::Language::Runtime.feature_path(path)
-
-    return load_without_ruby_next(path, wrap) unless realpath
-
-    RubyNext::Language::Runtime.load(realpath, wrap: wrap)
-  rescue => e
-    RubyNext.warn "RubyNext failed to load '#{path}': #{e.message}"
-    load_without_ruby_next(path)
-  end
+RequireHooks.source_transform(
+  patterns: RubyNext::Language.include_patterns,
+  exclude_patterns: RubyNext::Language.exclude_patterns
+) do |path, contents|
+  RubyNext::Language::Runtime.load(path, contents)
 end

data/lib/ruby-next/language/setup.rb

@@ -12,7 +12,10 @@ module RubyNext
         return if File.directory?(target_dir)
 
         Dir.chdir(root_dir) do
-          unless system("bundle exec ruby-next nextify ./#{lib_dir} -o #{target_dir} --min-version=#{RubyNext.current_ruby_version} > /dev/null 2>&1")
+          command = "bundle exec ruby-next nextify " \
+            "./#{lib_dir} -o #{target_dir} --min-version=#{RubyNext.current_ruby_version}"
+
+          unless system("#{command} > /dev/null 2>&1")
             RubyNext.warn "Traspiled files are missing in: #{target_dir}. \n" \
               "Make sure you have gem 'ruby-next' in your Gemfile to auto-transpile the required files from source on load. " \
               "Otherwise the code from #{root_dir} may not work correctly."
@@ -43,7 +46,7 @@ module RubyNext
 
         dirname = File.realpath(dirname)
 
-        return if Language.runtime? && Language.watch_dirs.include?(dirname)
+        return if Language.runtime? && Language.target_dir?(dirname)
 
         next_dirname = File.join(dirname, rbnext_dir)
 

data/lib/ruby-next/language/unparser.rb

@@ -5,6 +5,11 @@ save_verbose, $VERBOSE = $VERBOSE, nil
 require "parser/current"
 $VERBOSE = save_verbose
 
+# For backward compatibility with older Unparser for EOL Rubies
+if !Parser::Lexer.const_defined?(:ESCAPES)
+  Parser::Lexer::ESCAPES = Parser::LexerStrings::ESCAPES
+end
+
 require "unparser"
 
 # For backward compatibility with older Unparser

data/lib/ruby-next/language.rb

@@ -3,7 +3,7 @@
 gem "ruby-next-parser", ">= 2.8.0.3"
 gem "unparser", ">= 0.4.7"
 
-require "set"
+require "set" # rubocop:disable Lint/RedundantRequireStatement
 
 require "ruby-next"
 
@@ -62,8 +62,15 @@ module RubyNext
     end
 
     class << self
+      attr_reader :include_patterns
+      attr_reader :exclude_patterns
+
+      def watch_dirs
+        warn "[DEPRECATED] Use `RubyNext::Language.include_patterns` instead of `RubyNext::Language.watch_dirs`"
+        @watch_dirs
+      end
+
       attr_accessor :rewriters
-      attr_reader :watch_dirs
 
       attr_accessor :strategy
 
@@ -95,14 +102,17 @@ module RubyNext
       end
 
       def transform(source, rewriters: self.rewriters, using: RubyNext::Core.refine?, context: TransformContext.new)
+        text_rewriters, ast_rewriters = rewriters.partition(&:text?)
+
         retried = 0
-        new_source = nil
+        new_source = text_rewrite(source, rewriters: text_rewriters, using: using, context: context)
+
         begin
           new_source =
             if mode == :rewrite
-              rewrite(source, rewriters: rewriters, using: using, context: context)
+              rewrite(new_source, rewriters: ast_rewriters, using: using, context: context)
             else
-              regenerate(source, rewriters: rewriters, using: using, context: context)
+              regenerate(new_source, rewriters: ast_rewriters, using: using, context: context)
             end
         rescue Unparser::UnknownNodeError => err
           RubyNext.warn "Ruby Next fallbacks to \"rewrite\" transpiling mode since the version of Unparser you use doesn't support some syntax yet: #{err.message}.\n" \
@@ -119,8 +129,17 @@ module RubyNext
         Core.inject! new_source.dup
       end
 
+      def target_dir?(dirname)
+        # fnmatch? requires a file name, not a folder
+        fname = File.join(dirname, "x.rb")
+
+        include_patterns.any? { |pattern| File.fnmatch?(pattern, fname) } &&
+          exclude_patterns.none? { |pattern| File.fnmatch?(pattern, fname) }
+      end
+
       def transformable?(path)
-        watch_dirs.any? { |dir| path.start_with?(dir) }
+        include_patterns.any? { |pattern| File.fnmatch?(pattern, path) } &&
+          exclude_patterns.none? { |pattern| File.fnmatch?(pattern, path) }
       end
 
       # Rewriters required for the current version
@@ -165,14 +184,36 @@ module RubyNext
         end
       end
 
+      def text_rewrite(source, rewriters:, using:, context:)
+        rewriters.inject(source) do |src, rewriter|
+          rewriter.new(context).rewrite(src)
+        end.then do |new_source|
+          next source unless context.dirty?
+
+          new_source
+        end
+      end
+
       attr_writer :watch_dirs
+      attr_writer :include_patterns, :exclude_patterns
     end
 
     self.rewriters = []
-    self.watch_dirs = %w[app lib spec test].map { |path| File.join(Dir.pwd, path) }
+    self.watch_dirs = [].tap do |dirs|
+      # For backward compatibility
+      dirs.define_singleton_method(:<<) do |dir|
+        super(dir)
+        RubyNext::Language.include_patterns << File.join(dir, "*.rb")
+      end
+    end
+
+    self.include_patterns = %w[app lib spec test].map { |path| File.join(Dir.pwd, path, "*.rb") }
+    self.exclude_patterns = %w[vendor/bundle].map { |path| File.join(Dir.pwd, path, "*") }
     self.mode = ENV.fetch("RUBY_NEXT_TRANSPILE_MODE", "rewrite").to_sym
 
+    require "ruby-next/language/rewriters/abstract"
     require "ruby-next/language/rewriters/base"
+    require "ruby-next/language/rewriters/text"
 
     require "ruby-next/language/rewriters/2.1/numeric_literals"
     rewriters << Rewriters::NumericLiterals
@@ -212,7 +253,7 @@ module RubyNext
     rewriters << Rewriters::InPattern
 
     require "ruby-next/language/rewriters/3.0/endless_method"
-    RubyNext::Language.rewriters << RubyNext::Language::Rewriters::EndlessMethod
+    rewriters << RubyNext::Language::Rewriters::EndlessMethod
 
     require "ruby-next/language/rewriters/3.1/oneline_pattern_parensless"
     rewriters << Rewriters::OnelinePatternParensless
@@ -232,16 +273,19 @@ module RubyNext
     require "ruby-next/language/rewriters/3.1/shorthand_hash"
     rewriters << RubyNext::Language::Rewriters::ShorthandHash
 
+    require "ruby-next/language/rewriters/3.2/anonymous_restargs"
+    rewriters << RubyNext::Language::Rewriters::AnonymousRestArgs
+
     # Put endless range in the end, 'cause Parser fails to parse it in
     # pattern matching
     require "ruby-next/language/rewriters/2.6/endless_range"
     rewriters << Rewriters::EndlessRange
 
-    if ENV["RUBY_NEXT_EDGE"] == "1"
+    if RubyNext.edge_syntax?
       require "ruby-next/language/rewriters/edge"
     end
 
-    if ENV["RUBY_NEXT_PROPOSED"] == "1"
+    if RubyNext.proposed_syntax?
       require "ruby-next/language/rewriters/proposed"
     end
   end

data/lib/ruby-next/pry.rb

@@ -64,7 +64,7 @@ end)
 Pry::Code.singleton_class.prepend(Module.new do
   def complete_expression?(str)
     silence_stderr do
-      ::Parser::RubyNext.parse(str)
+      ::RubyNext::Language.parser_class.parse(str)
     end
 
     true

data/lib/ruby-next/rubocop.rb

@@ -4,6 +4,7 @@
 # edge features and fix some bugs with 2.7+ syntax
 
 require "parser/ruby-next/version"
+require "ruby-next/config"
 require "ruby-next/language/parser"
 
 module RuboCop
@@ -38,6 +39,7 @@ module RuboCop
       def parser_class(version)
         return super unless version == RUBY_NEXT_VERSION
 
+        require "parser/rubynext"
         Parser::RubyNext
       end
     end

data/lib/ruby-next/utils.rb

@@ -4,28 +4,6 @@ module RubyNext
   module Utils
     module_function
 
-    if $LOAD_PATH.respond_to?(:resolve_feature_path)
-      def resolve_feature_path(feature)
-        $LOAD_PATH.resolve_feature_path(feature)&.last
-      rescue LoadError
-      end
-    else
-      def resolve_feature_path(path)
-        if File.file?(relative = File.expand_path(path))
-          path = relative
-        end
-
-        path = "#{path}.rb" if File.extname(path).empty?
-
-        return path if Pathname.new(path).absolute?
-
-        $LOAD_PATH.find do |lp|
-          lpath = File.join(lp, path)
-          return File.realpath(lpath) if File.file?(lpath)
-        end
-      end
-    end
-
     def source_with_lines(source, path)
       source.lines.map.with_index do |line, i|
         "#{(i + 1).to_s.rjust(4)}:  #{line}"
@@ -36,6 +14,7 @@ module RubyNext
 
     # Returns true if modules refinement is supported in current version
     def refine_modules?
+      save_verbose, $VERBOSE = $VERBOSE, nil
       @refine_modules ||=
         begin
           # Make sure that including modules within refinements works
@@ -61,6 +40,8 @@ module RubyNext
           true
         rescue TypeError, NoMethodError
           false
+        ensure
+          $VERBOSE = save_verbose
         end
     end
   end

data/lib/ruby-next/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubyNext
-  VERSION = "0.15.3"
+  VERSION = "1.0.0.rc.1"
 end