prism 0.13.0

data/lib/prism/parse_result.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module Prism
+  # This represents a source of Ruby code that has been parsed. It is used in
+  # conjunction with locations to allow them to resolve line numbers and source
+  # ranges.
+  class Source
+    attr_reader :source, :offsets
+
+    def initialize(source, offsets = compute_offsets(source))
+      @source = source
+      @offsets = offsets
+    end
+
+    def slice(offset, length)
+      source.byteslice(offset, length)
+    end
+
+    def line(value)
+      offsets.bsearch_index { |offset| offset > value } || offsets.length
+    end
+
+    def line_offset(value)
+      offsets[line(value) - 1]
+    end
+
+    def column(value)
+      value - offsets[line(value) - 1]
+    end
+
+    private
+
+    def compute_offsets(code)
+      offsets = [0]
+      code.b.scan("\n") { offsets << $~.end(0) }
+      offsets
+    end
+  end
+
+  # This represents a location in the source.
+  class Location
+    # A Source object that is used to determine more information from the given
+    # offset and length.
+    protected attr_reader :source
+
+    # The byte offset from the beginning of the source where this location
+    # starts.
+    attr_reader :start_offset
+
+    # The length of this location in bytes.
+    attr_reader :length
+
+    # The list of comments attached to this location
+    attr_reader :comments
+
+    def initialize(source, start_offset, length)
+      @source = source
+      @start_offset = start_offset
+      @length = length
+      @comments = []
+    end
+
+    # Create a new location object with the given options.
+    def copy(**options)
+      Location.new(
+        options.fetch(:source) { source },
+        options.fetch(:start_offset) { start_offset },
+        options.fetch(:length) { length }
+      )
+    end
+
+    # Returns a string representation of this location.
+    def inspect
+      "#<Prism::Location @start_offset=#{@start_offset} @length=#{@length} start_line=#{start_line}>"
+    end
+
+    # The source code that this location represents.
+    def slice
+      source.slice(start_offset, length)
+    end
+
+    # The byte offset from the beginning of the source where this location ends.
+    def end_offset
+      start_offset + length
+    end
+
+    # The line number where this location starts.
+    def start_line
+      source.line(start_offset)
+    end
+
+    # The content of the line where this location starts before this location.
+    def start_line_slice
+      offset = source.line_offset(start_offset)
+      source.slice(offset, start_offset - offset)
+    end
+
+    # The line number where this location ends.
+    def end_line
+      source.line(end_offset - 1)
+    end
+
+    # The column number in bytes where this location starts from the start of
+    # the line.
+    def start_column
+      source.column(start_offset)
+    end
+
+    # The column number in bytes where this location ends from the start of the
+    # line.
+    def end_column
+      source.column(end_offset)
+    end
+
+    def deconstruct_keys(keys)
+      { start_offset: start_offset, end_offset: end_offset }
+    end
+
+    def pretty_print(q)
+      q.text("(#{start_line},#{start_column})-(#{end_line},#{end_column}))")
+    end
+
+    def ==(other)
+      other.is_a?(Location) &&
+        other.start_offset == start_offset &&
+        other.end_offset == end_offset
+    end
+
+    # Returns a new location that stretches from this location to the given
+    # other location. Raises an error if this location is not before the other
+    # location or if they don't share the same source.
+    def join(other)
+      raise "Incompatible sources" if source != other.source
+      raise "Incompatible locations" if start_offset > other.start_offset
+
+      Location.new(source, start_offset, other.end_offset - start_offset)
+    end
+
+    def self.null
+      new(0, 0)
+    end
+  end
+
+  # This represents a comment that was encountered during parsing.
+  class Comment
+    TYPES = [:inline, :embdoc, :__END__]
+
+    attr_reader :type, :location
+
+    def initialize(type, location)
+      @type = type
+      @location = location
+    end
+
+    def deconstruct_keys(keys)
+      { type: type, location: location }
+    end
+
+    # Returns true if the comment happens on the same line as other code and false if the comment is by itself
+    def trailing?
+      type == :inline && !location.start_line_slice.strip.empty?
+    end
+
+    def inspect
+      "#<Prism::Comment @type=#{@type.inspect} @location=#{@location.inspect}>"
+    end
+  end
+
+  # This represents an error that was encountered during parsing.
+  class ParseError
+    attr_reader :message, :location
+
+    def initialize(message, location)
+      @message = message
+      @location = location
+    end
+
+    def deconstruct_keys(keys)
+      { message: message, location: location }
+    end
+
+    def inspect
+      "#<Prism::ParseError @message=#{@message.inspect} @location=#{@location.inspect}>"
+    end
+  end
+
+  # This represents a warning that was encountered during parsing.
+  class ParseWarning
+    attr_reader :message, :location
+
+    def initialize(message, location)
+      @message = message
+      @location = location
+    end
+
+    def deconstruct_keys(keys)
+      { message: message, location: location }
+    end
+
+    def inspect
+      "#<Prism::ParseWarning @message=#{@message.inspect} @location=#{@location.inspect}>"
+    end
+  end
+
+  # This represents the result of a call to ::parse or ::parse_file. It contains
+  # the AST, any comments that were encounters, and any errors that were
+  # encountered.
+  class ParseResult
+    attr_reader :value, :comments, :errors, :warnings, :source
+
+    def initialize(value, comments, errors, warnings, source)
+      @value = value
+      @comments = comments
+      @errors = errors
+      @warnings = warnings
+      @source = source
+    end
+
+    def deconstruct_keys(keys)
+      { value: value, comments: comments, errors: errors, warnings: warnings }
+    end
+
+    def success?
+      errors.empty?
+    end
+
+    def failure?
+      !success?
+    end
+  end
+
+  # This represents a token from the Ruby source.
+  class Token
+    attr_reader :type, :value, :location
+
+    def initialize(type, value, location)
+      @type = type
+      @value = value
+      @location = location
+    end
+
+    def deconstruct_keys(keys)
+      { type: type, value: value, location: location }
+    end
+
+    def pretty_print(q)
+      q.group do
+        q.text(type.to_s)
+        self.location.pretty_print(q)
+        q.text("(")
+        q.nest(2) do
+          q.breakable("")
+          q.pp(value)
+        end
+        q.breakable("")
+        q.text(")")
+      end
+    end
+
+    def ==(other)
+      other.is_a?(Token) &&
+        other.type == type &&
+        other.value == value
+    end
+  end
+end

data/lib/prism/pattern.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+module Prism
+  # A pattern is an object that wraps a Ruby pattern matching expression. The
+  # expression would normally be passed to an `in` clause within a `case`
+  # expression or a rightward assignment expression. For example, in the
+  # following snippet:
+  #
+  #     case node
+  #     in ConstantPathNode[ConstantReadNode[name: :Prism], ConstantReadNode[name: :Pattern]]
+  #     end
+  #
+  # the pattern is the `ConstantPathNode[...]` expression.
+  #
+  # The pattern gets compiled into an object that responds to #call by running
+  # the #compile method. This method itself will run back through Prism to
+  # parse the expression into a tree, then walk the tree to generate the
+  # necessary callable objects. For example, if you wanted to compile the
+  # expression above into a callable, you would:
+  #
+  #     callable = Prism::Pattern.new("ConstantPathNode[ConstantReadNode[name: :Prism], ConstantReadNode[name: :Pattern]]").compile
+  #     callable.call(node)
+  #
+  # The callable object returned by #compile is guaranteed to respond to #call
+  # with a single argument, which is the node to match against. It also is
+  # guaranteed to respond to #===, which means it itself can be used in a `case`
+  # expression, as in:
+  #
+  #     case node
+  #     when callable
+  #     end
+  #
+  # If the query given to the initializer cannot be compiled into a valid
+  # matcher (either because of a syntax error or because it is using syntax we
+  # do not yet support) then a Prism::Pattern::CompilationError will be
+  # raised.
+  class Pattern
+    # Raised when the query given to a pattern is either invalid Ruby syntax or
+    # is using syntax that we don't yet support.
+    class CompilationError < StandardError
+      def initialize(repr)
+        super(<<~ERROR)
+          prism was unable to compile the pattern you provided into a usable
+          expression. It failed on to understand the node represented by:
+
+          #{repr}
+
+          Note that not all syntax supported by Ruby's pattern matching syntax
+          is also supported by prism's patterns. If you're using some syntax
+          that you believe should be supported, please open an issue on
+          GitHub at https://github.com/ruby/prism/issues/new.
+        ERROR
+      end
+    end
+
+    attr_reader :query
+
+    def initialize(query)
+      @query = query
+      @compiled = nil
+    end
+
+    def compile
+      result = Prism.parse("case nil\nin #{query}\nend")
+      compile_node(result.value.statements.body.last.conditions.last.pattern)
+    end
+
+    def scan(root)
+      return to_enum(__method__, root) unless block_given?
+
+      @compiled ||= compile
+      queue = [root]
+
+      while (node = queue.shift)
+        yield node if @compiled.call(node)
+        queue.concat(node.compact_child_nodes)
+      end
+    end
+
+    private
+
+    # Shortcut for combining two procs into one that returns true if both return
+    # true.
+    def combine_and(left, right)
+      ->(other) { left.call(other) && right.call(other) }
+    end
+
+    # Shortcut for combining two procs into one that returns true if either
+    # returns true.
+    def combine_or(left, right)
+      ->(other) { left.call(other) || right.call(other) }
+    end
+
+    # Raise an error because the given node is not supported.
+    def compile_error(node)
+      raise CompilationError, node.inspect
+    end
+
+    # in [foo, bar, baz]
+    def compile_array_pattern_node(node)
+      compile_error(node) if !node.rest.nil? || node.posts.any?
+
+      constant = node.constant
+      compiled_constant = compile_node(constant) if constant
+
+      preprocessed = node.requireds.map { |required| compile_node(required) }
+
+      compiled_requireds = ->(other) do
+        deconstructed = other.deconstruct
+
+        deconstructed.length == preprocessed.length &&
+          preprocessed
+            .zip(deconstructed)
+            .all? { |(matcher, value)| matcher.call(value) }
+      end
+
+      if compiled_constant
+        combine_and(compiled_constant, compiled_requireds)
+      else
+        compiled_requireds
+      end
+    end
+
+    # in foo | bar
+    def compile_alternation_pattern_node(node)
+      combine_or(compile_node(node.left), compile_node(node.right))
+    end
+
+    # in Prism::ConstantReadNode
+    def compile_constant_path_node(node)
+      parent = node.parent
+
+      if parent.is_a?(ConstantReadNode) && parent.slice == "Prism"
+        compile_node(node.child)
+      else
+        compile_error(node)
+      end
+    end
+
+    # in ConstantReadNode
+    # in String
+    def compile_constant_read_node(node)
+      value = node.slice
+
+      if Prism.const_defined?(value, false)
+        clazz = Prism.const_get(value)
+
+        ->(other) { clazz === other }
+      elsif Object.const_defined?(value, false)
+        clazz = Object.const_get(value)
+
+        ->(other) { clazz === other }
+      else
+        compile_error(node)
+      end
+    end
+
+    # in InstanceVariableReadNode[name: Symbol]
+    # in { name: Symbol }
+    def compile_hash_pattern_node(node)
+      compile_error(node) unless node.kwrest.nil?
+      compiled_constant = compile_node(node.constant) if node.constant
+
+      preprocessed =
+        node.assocs.to_h do |assoc|
+          [assoc.key.unescaped.to_sym, compile_node(assoc.value)]
+        end
+
+      compiled_keywords = ->(other) do
+        deconstructed = other.deconstruct_keys(preprocessed.keys)
+
+        preprocessed.all? do |keyword, matcher|
+          deconstructed.key?(keyword) && matcher.call(deconstructed[keyword])
+        end
+      end
+
+      if compiled_constant
+        combine_and(compiled_constant, compiled_keywords)
+      else
+        compiled_keywords
+      end
+    end
+
+    # in nil
+    def compile_nil_node(node)
+      ->(attribute) { attribute.nil? }
+    end
+
+    # in /foo/
+    def compile_regular_expression_node(node)
+      regexp = Regexp.new(node.unescaped, node.closing[1..])
+
+      ->(attribute) { regexp === attribute }
+    end
+
+    # in ""
+    # in "foo"
+    def compile_string_node(node)
+      string = node.unescaped
+
+      ->(attribute) { string === attribute }
+    end
+
+    # in :+
+    # in :foo
+    def compile_symbol_node(node)
+      symbol = node.unescaped.to_sym
+
+      ->(attribute) { symbol === attribute }
+    end
+
+    # Compile any kind of node. Dispatch out to the individual compilation
+    # methods based on the type of node.
+    def compile_node(node)
+      case node
+      when AlternationPatternNode
+        compile_alternation_pattern_node(node)
+      when ArrayPatternNode
+        compile_array_pattern_node(node)
+      when ConstantPathNode
+        compile_constant_path_node(node)
+      when ConstantReadNode
+        compile_constant_read_node(node)
+      when HashPatternNode
+        compile_hash_pattern_node(node)
+      when NilNode
+        compile_nil_node(node)
+      when RegularExpressionNode
+        compile_regular_expression_node(node)
+      when StringNode
+        compile_string_node(node)
+      when SymbolNode
+        compile_symbol_node(node)
+      else
+        compile_error(node)
+      end
+    end
+  end
+end

data/lib/prism/ripper_compat.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require "ripper"
+
+module Prism
+  # This class is meant to provide a compatibility layer between prism and
+  # Ripper. It functions by parsing the entire tree first and then walking it
+  # and executing each of the Ripper callbacks as it goes.
+  #
+  # This class is going to necessarily be slower than the native Ripper API. It
+  # is meant as a stopgap until developers migrate to using prism. It is also
+  # meant as a test harness for the prism parser.
+  class RipperCompat
+    # This class mirrors the ::Ripper::SexpBuilder subclass of ::Ripper that
+    # returns the arrays of [type, *children].
+    class SexpBuilder < RipperCompat
+      private
+
+      Ripper::PARSER_EVENTS.each do |event|
+        define_method(:"on_#{event}") do |*args|
+          [event, *args]
+        end
+      end
+
+      Ripper::SCANNER_EVENTS.each do |event|
+        define_method(:"on_#{event}") do |value|
+          [:"@#{event}", value, [lineno, column]]
+        end
+      end
+    end
+
+    # This class mirrors the ::Ripper::SexpBuilderPP subclass of ::Ripper that
+    # returns the same values as ::Ripper::SexpBuilder except with a couple of
+    # niceties that flatten linked lists into arrays.
+    class SexpBuilderPP < SexpBuilder
+      private
+
+      def _dispatch_event_new
+        []
+      end
+
+      def _dispatch_event_push(list, item)
+        list << item
+        list
+      end
+
+      Ripper::PARSER_EVENT_TABLE.each do |event, arity|
+        case event
+        when /_new\z/
+          alias_method :"on_#{event}", :_dispatch_event_new if arity == 0
+        when /_add\z/
+          alias_method :"on_#{event}", :_dispatch_event_push
+        end
+      end
+    end
+
+    attr_reader :source, :lineno, :column
+
+    def initialize(source)
+      @source = source
+      @result = nil
+      @lineno = nil
+      @column = nil
+    end
+
+    ############################################################################
+    # Public interface
+    ############################################################################
+
+    def error?
+      result.errors.any?
+    end
+
+    def parse
+      result.value.accept(self) unless error?
+    end
+
+    ############################################################################
+    # Visitor methods
+    ############################################################################
+
+    def visit(node)
+      node&.accept(self)
+    end
+
+    def visit_call_node(node)
+      if !node.opening_loc && node.arguments.arguments.length == 1
+        bounds(node.receiver.location)
+        left = visit(node.receiver)
+
+        bounds(node.arguments.arguments.first.location)
+        right = visit(node.arguments.arguments.first)
+
+        on_binary(left, source[node.message_loc.start_offset...node.message_loc.end_offset].to_sym, right)
+      else
+        raise NotImplementedError
+      end
+    end
+
+    def visit_integer_node(node)
+      bounds(node.location)
+      on_int(source[node.location.start_offset...node.location.end_offset])
+    end
+
+    def visit_statements_node(node)
+      bounds(node.location)
+      node.body.inject(on_stmts_new) do |stmts, stmt|
+        on_stmts_add(stmts, visit(stmt))
+      end
+    end
+
+    def visit_token(node)
+      bounds(node.location)
+
+      case node.type
+      when :MINUS
+        on_op(node.value)
+      when :PLUS
+        on_op(node.value)
+      else
+        raise NotImplementedError, "Unknown token: #{node.type}"
+      end
+    end
+
+    def visit_program_node(node)
+      bounds(node.location)
+      on_program(visit(node.statements))
+    end
+
+    ############################################################################
+    # Entrypoints for subclasses
+    ############################################################################
+
+    # This is a convenience method that runs the SexpBuilder subclass parser.
+    def self.sexp_raw(source)
+      SexpBuilder.new(source).parse
+    end
+
+    # This is a convenience method that runs the SexpBuilderPP subclass parser.
+    def self.sexp(source)
+      SexpBuilderPP.new(source).parse
+    end
+
+    private
+
+    # This method is responsible for updating lineno and column information
+    # to reflect the current node.
+    #
+    # This method could be drastically improved with some caching on the start
+    # of every line, but for now it's good enough.
+    def bounds(location)
+      start_offset = location.start_offset
+
+      @lineno = source[0..start_offset].count("\n") + 1
+      @column = start_offset - (source.rindex("\n", start_offset) || 0)
+    end
+
+    def result
+      @result ||= Prism.parse(source)
+    end
+
+    def _dispatch0; end
+    def _dispatch1(_); end
+    def _dispatch2(_, _); end
+    def _dispatch3(_, _, _); end
+    def _dispatch4(_, _, _, _); end
+    def _dispatch5(_, _, _, _, _); end
+    def _dispatch7(_, _, _, _, _, _, _); end
+
+    (Ripper::SCANNER_EVENT_TABLE.merge(Ripper::PARSER_EVENT_TABLE)).each do |event, arity|
+      alias_method :"on_#{event}", :"_dispatch#{arity}"
+    end
+  end
+end