yarp 0.10.0 → 0.11.0

data/lib/yarp/parse_result/comments.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module YARP
+  class ParseResult
+    # When we've parsed the source, we have both the syntax tree and the list of
+    # comments that we found in the source. This class is responsible for
+    # walking the tree and finding the nearest location to attach each comment.
+    #
+    # It does this by first finding the nearest locations to each comment.
+    # Locations can either come from nodes directly or from location fields on
+    # nodes. For example, a `ClassNode` has an overall location encompassing the
+    # entire class, but it also has a location for the `class` keyword.
+    #
+    # Once the nearest locations are found, it determines which one to attach
+    # to. If it's a trailing comment (a comment on the same line as other source
+    # code), it will favor attaching to the nearest location that occurs before
+    # the comment. Otherwise it will favor attaching to the nearest location
+    # that is after the comment.
+    class Comments
+      # A target for attaching comments that is based on a specific node's
+      # location.
+      class NodeTarget
+        attr_reader :node
+
+        def initialize(node)
+          @node = node
+        end
+
+        def start_offset
+          node.location.start_offset
+        end
+
+        def end_offset
+          node.location.end_offset
+        end
+
+        def encloses?(comment)
+          start_offset <= comment.location.start_offset &&
+            comment.location.end_offset <= end_offset
+        end
+
+        def <<(comment)
+          node.location.comments << comment
+        end
+      end
+
+      # A target for attaching comments that is based on a location field on a
+      # node. For example, the `end` token of a ClassNode.
+      class LocationTarget
+        attr_reader :location
+
+        def initialize(location)
+          @location = location
+        end
+
+        def start_offset
+          location.start_offset
+        end
+
+        def end_offset
+          location.end_offset
+        end
+
+        def encloses?(comment)
+          false
+        end
+
+        def <<(comment)
+          location.comments << comment
+        end
+      end
+
+      attr_reader :parse_result
+
+      def initialize(parse_result)
+        @parse_result = parse_result
+      end
+
+      def attach!
+        parse_result.comments.each do |comment|
+          preceding, enclosing, following = nearest_targets(parse_result.value, comment)
+          target =
+            if comment.trailing?
+              preceding || following || enclosing || NodeTarget.new(parse_result.value)
+            else
+              # If a comment exists on its own line, prefer a leading comment.
+              following || preceding || enclosing || NodeTarget.new(parse_result.value)
+            end
+
+          target << comment
+        end
+      end
+
+      private
+
+      # Responsible for finding the nearest targets to the given comment within
+      # the context of the given encapsulating node.
+      def nearest_targets(node, comment)
+        comment_start = comment.location.start_offset
+        comment_end = comment.location.end_offset
+
+        targets = []
+        node.comment_targets.map do |value|
+          case value
+          when StatementsNode
+            targets.concat(value.body.map { |node| NodeTarget.new(node) })
+          when Node
+            targets << NodeTarget.new(value)
+          when Location
+            targets << LocationTarget.new(value)
+          end
+        end
+
+        targets.sort_by!(&:start_offset)
+        preceding = nil
+        following = nil
+
+        left = 0
+        right = targets.length
+
+        # This is a custom binary search that finds the nearest nodes to the
+        # given comment. When it finds a node that completely encapsulates the
+        # comment, it recurses downward into the tree.
+        while left < right
+          middle = (left + right) / 2
+          target = targets[middle]
+
+          target_start = target.start_offset
+          target_end = target.end_offset
+
+          if target.encloses?(comment)
+            # The comment is completely contained by this target. Abandon the
+            # binary search at this level.
+            return nearest_targets(target.node, comment)
+          end
+
+          if target_end <= comment_start
+            # This target falls completely before the comment. Because we will
+            # never consider this target or any targets before it again, this
+            # target must be the closest preceding target we have encountered so
+            # far.
+            preceding = target
+            left = middle + 1
+            next
+          end
+
+          if comment_end <= target_start
+            # This target falls completely after the comment. Because we will
+            # never consider this target or any targets after it again, this
+            # target must be the closest following target we have encountered so
+            # far.
+            following = target
+            right = middle
+            next
+          end
+
+          # This should only happen if there is a bug in this parser.
+          raise "Comment location overlaps with a target location"
+        end
+
+        [preceding, NodeTarget.new(node), following]
+      end
+    end
+
+    private_constant :Comments
+
+    # Attach the list of comments to their respective locations in the tree.
+    def attach_comments!
+      Comments.new(self).attach!
+    end
+  end
+end

data/lib/yarp/parse_result/newlines.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module YARP
+  class ParseResult
+    # The :line tracepoint event gets fired whenever the Ruby VM encounters an
+    # expression on a new line. The types of expressions that can trigger this
+    # event are:
+    #
+    # * if statements
+    # * unless statements
+    # * nodes that are children of statements lists
+    #
+    # In order to keep track of the newlines, we have a list of offsets that
+    # come back from the parser. We assign these offsets to the first nodes that
+    # we find in the tree that are on those lines.
+    #
+    # Note that the logic in this file should be kept in sync with the Java
+    # MarkNewlinesVisitor, since that visitor is responsible for marking the
+    # newlines for JRuby/TruffleRuby.
+    class Newlines < Visitor
+      def initialize(newline_marked)
+        @newline_marked = newline_marked
+      end
+
+      def visit_block_node(node)
+        old_newline_marked = @newline_marked
+        @newline_marked = Array.new(old_newline_marked.size, false)
+
+        begin
+          super(node)
+        ensure
+          @newline_marked = old_newline_marked
+        end
+      end
+
+      alias_method :visit_lambda_node, :visit_block_node
+
+      def visit_if_node(node)
+        node.set_newline_flag(@newline_marked)
+        super(node)
+      end
+
+      alias_method :visit_unless_node, :visit_if_node
+
+      def visit_statements_node(node)
+        node.body.each do |child|
+          child.set_newline_flag(@newline_marked)
+        end
+        super(node)
+      end
+    end
+
+    private_constant :Newlines
+
+    # Walk the tree and mark nodes that are on a new line.
+    def mark_newlines!
+      value.accept(Newlines.new(Array.new(1 + source.offsets.size, false)))
+    end
+  end
+end

data/lib/yarp/pattern.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+module YARP
+  # A pattern is an object that wraps a Ruby pattern matching expression. The
+  # expression would normally be passed to an `in` clause within a `case`
+  # expression or a rightward assignment expression. For example, in the
+  # following snippet:
+  #
+  #     case node
+  #     in ConstantPathNode[ConstantReadNode[name: :YARP], ConstantReadNode[name: :Pattern]]
+  #     end
+  #
+  # the pattern is the `ConstantPathNode[...]` expression.
+  #
+  # The pattern gets compiled into an object that responds to #call by running
+  # the #compile method. This method itself will run back through YARP to
+  # parse the expression into a tree, then walk the tree to generate the
+  # necessary callable objects. For example, if you wanted to compile the
+  # expression above into a callable, you would:
+  #
+  #     callable = YARP::Pattern.new("ConstantPathNode[ConstantReadNode[name: :YARP], ConstantReadNode[name: :Pattern]]").compile
+  #     callable.call(node)
+  #
+  # The callable object returned by #compile is guaranteed to respond to #call
+  # with a single argument, which is the node to match against. It also is
+  # guaranteed to respond to #===, which means it itself can be used in a `case`
+  # expression, as in:
+  #
+  #     case node
+  #     when callable
+  #     end
+  #
+  # If the query given to the initializer cannot be compiled into a valid
+  # matcher (either because of a syntax error or because it is using syntax we
+  # do not yet support) then a YARP::Pattern::CompilationError will be
+  # raised.
+  class Pattern
+    # Raised when the query given to a pattern is either invalid Ruby syntax or
+    # is using syntax that we don't yet support.
+    class CompilationError < StandardError
+      def initialize(repr)
+        super(<<~ERROR)
+          YARP was unable to compile the pattern you provided into a usable
+          expression. It failed on to understand the node represented by:
+
+          #{repr}
+
+          Note that not all syntax supported by Ruby's pattern matching syntax
+          is also supported by YARP's patterns. If you're using some syntax
+          that you believe should be supported, please open an issue on
+          GitHub at https://github.com/ruby/yarp/issues/new.
+        ERROR
+      end
+    end
+
+    attr_reader :query
+
+    def initialize(query)
+      @query = query
+      @compiled = nil
+    end
+
+    def compile
+      result = YARP.parse("case nil\nin #{query}\nend")
+      compile_node(result.value.statements.body.last.conditions.last.pattern)
+    end
+
+    def scan(root)
+      return to_enum(__method__, root) unless block_given?
+
+      @compiled ||= compile
+      queue = [root]
+
+      while (node = queue.shift)
+        yield node if @compiled.call(node)
+        queue.concat(node.child_nodes.compact)
+      end
+    end
+
+    private
+
+    # Shortcut for combining two procs into one that returns true if both return
+    # true.
+    def combine_and(left, right)
+      ->(other) { left.call(other) && right.call(other) }
+    end
+
+    # Shortcut for combining two procs into one that returns true if either
+    # returns true.
+    def combine_or(left, right)
+      ->(other) { left.call(other) || right.call(other) }
+    end
+
+    # Raise an error because the given node is not supported.
+    def compile_error(node)
+      raise CompilationError, node.inspect
+    end
+
+    # in [foo, bar, baz]
+    def compile_array_pattern_node(node)
+      compile_error(node) if !node.rest.nil? || node.posts.any?
+
+      constant = node.constant
+      compiled_constant = compile_node(constant) if constant
+
+      preprocessed = node.requireds.map { |required| compile_node(required) }
+
+      compiled_requireds = ->(other) do
+        deconstructed = other.deconstruct
+
+        deconstructed.length == preprocessed.length &&
+          preprocessed
+            .zip(deconstructed)
+            .all? { |(matcher, value)| matcher.call(value) }
+      end
+
+      if compiled_constant
+        combine_and(compiled_constant, compiled_requireds)
+      else
+        compiled_requireds
+      end
+    end
+
+    # in foo | bar
+    def compile_alternation_pattern_node(node)
+      combine_or(compile_node(node.left), compile_node(node.right))
+    end
+
+    # in YARP::ConstantReadNode
+    def compile_constant_path_node(node)
+      parent = node.parent
+
+      if parent.is_a?(ConstantReadNode) && parent.slice == "YARP"
+        compile_node(node.child)
+      else
+        compile_error(node)
+      end
+    end
+
+    # in ConstantReadNode
+    # in String
+    def compile_constant_read_node(node)
+      value = node.slice
+
+      if YARP.const_defined?(value, false)
+        clazz = YARP.const_get(value)
+
+        ->(other) { clazz === other }
+      elsif Object.const_defined?(value, false)
+        clazz = Object.const_get(value)
+
+        ->(other) { clazz === other }
+      else
+        compile_error(node)
+      end
+    end
+
+    # in InstanceVariableReadNode[name: Symbol]
+    # in { name: Symbol }
+    def compile_hash_pattern_node(node)
+      compile_error(node) unless node.kwrest.nil?
+      compiled_constant = compile_node(node.constant) if node.constant
+
+      preprocessed =
+        node.assocs.to_h do |assoc|
+          [assoc.key.unescaped.to_sym, compile_node(assoc.value)]
+        end
+
+      compiled_keywords = ->(other) do
+        deconstructed = other.deconstruct_keys(preprocessed.keys)
+
+        preprocessed.all? do |keyword, matcher|
+          deconstructed.key?(keyword) && matcher.call(deconstructed[keyword])
+        end
+      end
+
+      if compiled_constant
+        combine_and(compiled_constant, compiled_keywords)
+      else
+        compiled_keywords
+      end
+    end
+
+    # in nil
+    def compile_nil_node(node)
+      ->(attribute) { attribute.nil? }
+    end
+
+    # in /foo/
+    def compile_regular_expression_node(node)
+      regexp = Regexp.new(node.unescaped, node.closing[1..])
+
+      ->(attribute) { regexp === attribute }
+    end
+
+    # in ""
+    # in "foo"
+    def compile_string_node(node)
+      string = node.unescaped
+
+      ->(attribute) { string === attribute }
+    end
+
+    # in :+
+    # in :foo
+    def compile_symbol_node(node)
+      symbol = node.unescaped.to_sym
+
+      ->(attribute) { symbol === attribute }
+    end
+
+    # Compile any kind of node. Dispatch out to the individual compilation
+    # methods based on the type of node.
+    def compile_node(node)
+      case node
+      when AlternationPatternNode
+        compile_alternation_pattern_node(node)
+      when ArrayPatternNode
+        compile_array_pattern_node(node)
+      when ConstantPathNode
+        compile_constant_path_node(node)
+      when ConstantReadNode
+        compile_constant_read_node(node)
+      when HashPatternNode
+        compile_hash_pattern_node(node)
+      when NilNode
+        compile_nil_node(node)
+      when RegularExpressionNode
+        compile_regular_expression_node(node)
+      when StringNode
+        compile_string_node(node)
+      when SymbolNode
+        compile_symbol_node(node)
+      else
+        compile_error(node)
+      end
+    end
+  end
+end