yarp 0.6.0 → 0.8.0

data/lib/yarp.rb

@@ -7,7 +7,7 @@ module YARP
   class Source
     attr_reader :source, :offsets
 
-    def initialize(source, offsets)
+    def initialize(source, offsets = compute_offsets(source))
       @source = source
       @offsets = offsets
     end
@@ -23,6 +23,14 @@ module YARP
     def column(value)
       value - offsets[line(value) - 1]
     end
+
+    private
+
+    def compute_offsets(code)
+      offsets = [0]
+      code.b.scan("\n") { offsets << $~.end(0) }
+      offsets
+    end
   end
 
   # This represents a location in the source.
@@ -101,6 +109,8 @@ module YARP
 
   # This represents a comment that was encountered during parsing.
   class Comment
+    TYPES = [:inline, :embdoc, :__END__]
+
     attr_reader :type, :location
 
     def initialize(type, location)
@@ -279,6 +289,11 @@ module YARP
       end
     end
 
+    # Slice the location of the node from the source.
+    def slice
+      location.slice
+    end
+
     def pretty_print(q)
       q.group do
         q.text(self.class.name.split("::").last)
@@ -306,6 +321,152 @@ module YARP
   # This module is used for testing and debugging and is not meant to be used by
   # consumers of this library.
   module Debug
+    class ISeq
+      attr_reader :parts
+
+      def initialize(parts)
+        @parts = parts
+      end
+
+      def type
+        parts[0]
+      end
+
+      def local_table
+        parts[10]
+      end
+
+      def instructions
+        parts[13]
+      end
+
+      def each_child
+        instructions.each do |instruction|
+          # Only look at arrays. Other instructions are line numbers or
+          # tracepoint events.
+          next unless instruction.is_a?(Array)
+
+          instruction.each do |opnd|
+            # Only look at arrays. Other operands are literals.
+            next unless opnd.is_a?(Array)
+
+            # Only look at instruction sequences. Other operands are literals.
+            next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat"
+
+            yield ISeq.new(opnd)
+          end
+        end
+      end
+    end
+
+    # For the given source, compiles with CRuby and returns a list of all of the
+    # sets of local variables that were encountered.
+    def self.cruby_locals(source)
+      verbose = $VERBOSE
+      $VERBOSE = nil
+
+      begin
+        locals = []
+        stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
+
+        while (iseq = stack.pop)
+          if iseq.type != :once
+            names = iseq.local_table
+
+            # CRuby will push on a special local variable when there are keyword
+            # arguments. We get rid of that here.
+            names = names.grep_v(Integer)
+
+            # TODO: We don't support numbered local variables yet, so we get rid
+            # of those here.
+            names = names.grep_v(/^_\d$/)
+
+            # For some reason, CRuby occasionally pushes this special local
+            # variable when there are splat arguments. We get rid of that here.
+            names = names.grep_v(:"#arg_rest")
+
+            # Now push them onto the list of locals.
+            locals << names
+          end
+
+          iseq.each_child { |child| stack << child }
+        end
+
+        locals
+      ensure
+        $VERBOSE = verbose
+      end
+    end
+
+    # For the given source, parses with YARP and returns a list of all of the
+    # sets of local variables that were encountered.
+    def self.yarp_locals(source)
+      locals = []
+      stack = [YARP.parse(source).value]
+
+      while (node = stack.pop)
+        case node
+        when BlockNode, DefNode, LambdaNode
+          names = node.locals
+
+          params = node.parameters
+          params = params&.parameters unless node.is_a?(DefNode)
+
+          # YARP places parameters in the same order that they appear in the
+          # source. CRuby places them in the order that they need to appear
+          # according to their own internal calling convention. We mimic that
+          # order here so that we can compare properly.
+          if params
+            sorted = [
+              *params.requireds.grep(RequiredParameterNode).map(&:constant_id),
+              *params.optionals.map(&:constant_id),
+              *((params.rest.name ? params.rest.name.to_sym : :*) if params.rest && params.rest.operator != ","),
+              *params.posts.grep(RequiredParameterNode).map(&:constant_id),
+              *params.keywords.reject(&:value).map { |param| param.name.chomp(":").to_sym },
+              *params.keywords.select(&:value).map { |param| param.name.chomp(":").to_sym }
+            ]
+
+            # TODO: When we get a ... parameter, we should be pushing * and &
+            # onto the local list. We don't do that yet, so we need to add them
+            # in here.
+            if params.keyword_rest.is_a?(ForwardingParameterNode)
+              sorted.push(:*, :&, :"...")
+            end
+
+            # Recurse down the parameter tree to find any destructured
+            # parameters and add them after the other parameters.
+            param_stack = params.requireds.concat(params.posts).grep(RequiredDestructuredParameterNode).reverse
+            while (param = param_stack.pop)
+              case param
+              when RequiredDestructuredParameterNode
+                param_stack.concat(param.parameters.reverse)
+              when RequiredParameterNode
+                sorted << param.constant_id
+              when SplatNode
+                sorted << param.expression.constant_id if param.expression
+              end
+            end
+
+            names = sorted.concat(names - sorted)
+          end
+
+          locals << names
+        when ClassNode, ModuleNode, ProgramNode, SingletonClassNode
+          locals << node.locals
+        when ForNode
+          locals << []
+        when PostExecutionNode
+          locals.push([], [])
+        when InterpolatedRegularExpressionNode
+          locals << [] if node.once?
+        end
+
+        stack.concat(node.child_nodes.compact)
+      end
+
+      locals
+    end
+
     def self.newlines(source)
       YARP.parse(source).source.offsets
     end
@@ -327,4 +488,8 @@ require_relative "yarp/ripper_compat"
 require_relative "yarp/serialize"
 require_relative "yarp/pack"
 
-require "yarp/yarp"
+if RUBY_ENGINE == "ruby" and !ENV["YARP_FFI_BACKEND"]
+  require "yarp/yarp"
+else
+  require "yarp/ffi"
+end

data/src/enc/yp_unicode.c

@@ -2318,3 +2318,12 @@ yp_encoding_t yp_encoding_utf_8 = {
     .isupper_char = yp_encoding_utf_8_isupper_char,
     .multibyte = true
 };
+
+yp_encoding_t yp_encoding_utf8_mac = {
+    .name = "utf8-mac",
+    .char_width = yp_encoding_utf_8_char_width,
+    .alnum_char = yp_encoding_utf_8_alnum_char,
+    .alpha_char = yp_encoding_utf_8_alpha_char,
+    .isupper_char = yp_encoding_utf_8_isupper_char,
+    .multibyte = true
+};