prism 0.16.0 → 0.17.0

data/lib/prism/ripper_compat.rb

@@ -35,11 +35,11 @@ module Prism
     class SexpBuilderPP < SexpBuilder
       private
 
-      def _dispatch_event_new
+      def _dispatch_event_new # :nodoc:
         []
       end
 
-      def _dispatch_event_push(list, item)
+      def _dispatch_event_push(list, item) # :nodoc:
         list << item
         list
       end
@@ -54,8 +54,16 @@ module Prism
       end
     end
 
-    attr_reader :source, :lineno, :column
+    # The source that is being parsed.
+    attr_reader :source
 
+    # The current line number of the parser.
+    attr_reader :lineno
+
+    # The current column number of the parser.
+    attr_reader :column
+
+    # Create a new RipperCompat object with the given source.
     def initialize(source)
       @source = source
       @result = nil
@@ -67,10 +75,12 @@ module Prism
     # Public interface
     ############################################################################
 
+    # True if the parser encountered an error during parsing.
     def error?
       result.errors.any?
     end
 
+    # Parse the source and return the result.
     def parse
       result.value.accept(self) unless error?
     end
@@ -79,10 +89,13 @@ module Prism
     # Visitor methods
     ############################################################################
 
+    # This method is responsible for dispatching to the correct visitor method
+    # based on the type of the node.
     def visit(node)
       node&.accept(self)
     end
 
+    # Visit a CallNode node.
     def visit_call_node(node)
       if !node.opening_loc && node.arguments.arguments.length == 1
         bounds(node.receiver.location)
@@ -97,11 +110,13 @@ module Prism
       end
     end
 
+    # Visit an IntegerNode node.
     def visit_integer_node(node)
       bounds(node.location)
       on_int(source[node.location.start_offset...node.location.end_offset])
     end
 
+    # Visit a StatementsNode node.
     def visit_statements_node(node)
       bounds(node.location)
       node.body.inject(on_stmts_new) do |stmts, stmt|
@@ -109,6 +124,7 @@ module Prism
       end
     end
 
+    # Visit a token found during parsing.
     def visit_token(node)
       bounds(node.location)
 
@@ -122,6 +138,7 @@ module Prism
       end
     end
 
+    # Visit a ProgramNode node.
     def visit_program_node(node)
       bounds(node.location)
       on_program(visit(node.statements))
@@ -155,17 +172,18 @@ module Prism
       @column = start_offset - (source.rindex("\n", start_offset) || 0)
     end
 
+    # Lazily initialize the parse result.
     def result
       @result ||= Prism.parse(source)
     end
 
-    def _dispatch0; end
-    def _dispatch1(_); end
-    def _dispatch2(_, _); end
-    def _dispatch3(_, _, _); end
-    def _dispatch4(_, _, _, _); end
-    def _dispatch5(_, _, _, _, _); end
-    def _dispatch7(_, _, _, _, _, _, _); end
+    def _dispatch0; end # :nodoc:
+    def _dispatch1(_); end # :nodoc:
+    def _dispatch2(_, _); end # :nodoc:
+    def _dispatch3(_, _, _); end # :nodoc:
+    def _dispatch4(_, _, _, _); end # :nodoc:
+    def _dispatch5(_, _, _, _, _); end # :nodoc:
+    def _dispatch7(_, _, _, _, _, _, _); end # :nodoc:
 
     (Ripper::SCANNER_EVENT_TABLE.merge(Ripper::PARSER_EVENT_TABLE)).each do |event, arity|
       alias_method :"on_#{event}", :"_dispatch#{arity}"

data/lib/prism/serialize.rb

@@ -11,7 +11,7 @@ require "stringio"
 if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset }
   String.prepend(
     Module.new {
-      def unpack1(format, offset: 0)
+      def unpack1(format, offset: 0) # :nodoc:
         offset == 0 ? super(format) : self[offset..].unpack1(format)
       end
     }
@@ -19,11 +19,21 @@ if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset
 end
 
 module Prism
+  # A module responsible for deserializing parse results.
   module Serialize
+    # The major version of prism that we are expecting to find in the serialized
+    # strings.
     MAJOR_VERSION = 0
-    MINOR_VERSION = 16
+
+    # The minor version of prism that we are expecting to find in the serialized
+    # strings.
+    MINOR_VERSION = 17
+
+    # The patch version of prism that we are expecting to find in the serialized
+    # strings.
     PATCH_VERSION = 0
 
+    # Deserialize the AST represented by the given string into a parse result.
     def self.load(input, serialized)
       input = input.dup
       source = Source.new(input)
@@ -34,13 +44,16 @@ module Prism
       result
     end
 
+    # Deserialize the tokens represented by the given string into a parse
+    # result.
     def self.load_tokens(source, serialized)
       Loader.new(source, serialized).load_tokens_result
     end
 
-    class Loader
+    class Loader # :nodoc:
       attr_reader :encoding, :input, :serialized, :io
       attr_reader :constant_pool_offset, :constant_pool, :source
+      attr_reader :start_line
 
       def initialize(source, serialized)
         @encoding = Encoding::UTF_8
@@ -54,7 +67,7 @@ module Prism
         @constant_pool = nil
 
         @source = source
-        define_load_node_lambdas unless RUBY_ENGINE == 'ruby'
+        define_load_node_lambdas unless RUBY_ENGINE == "ruby"
       end
 
       def load_header
@@ -75,8 +88,18 @@ module Prism
         @input = input.force_encoding(@encoding).freeze
       end
 
+      def load_start_line
+        source.start_line = load_varint
+      end
+
       def load_comments
-        load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) }
+        load_varint.times.map do
+          case load_varint
+          when 0 then InlineComment.new(load_location)
+          when 1 then EmbDocComment.new(load_location)
+          when 2 then DATAComment.new(load_location)
+          end
+        end
       end
 
       def load_metadata
@@ -103,6 +126,7 @@ module Prism
       def load_tokens_result
         tokens = load_tokens
         encoding = load_encoding
+        load_start_line
         comments, magic_comments, errors, warnings = load_metadata
 
         if encoding != @encoding
@@ -116,6 +140,7 @@ module Prism
       def load_nodes
         load_header
         load_force_encoding
+        load_start_line
 
         comments, magic_comments, errors, warnings = load_metadata
 
@@ -389,47 +414,47 @@ module Prism
           when 84 then
             KeywordHashNode.new(Array.new(load_varint) { load_node }, location)
           when 85 then
-            KeywordParameterNode.new(load_required_constant, load_location, load_optional_node, location)
-          when 86 then
             KeywordRestParameterNode.new(load_optional_constant, load_optional_location, load_location, location)
-          when 87 then
+          when 86 then
             LambdaNode.new(Array.new(load_varint) { load_required_constant }, load_location, load_location, load_location, load_optional_node, load_optional_node, location)
-          when 88 then
+          when 87 then
             LocalVariableAndWriteNode.new(load_location, load_location, load_node, load_required_constant, load_varint, location)
-          when 89 then
+          when 88 then
             LocalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_required_constant, load_required_constant, load_varint, location)
-          when 90 then
+          when 89 then
             LocalVariableOrWriteNode.new(load_location, load_location, load_node, load_required_constant, load_varint, location)
-          when 91 then
+          when 90 then
             LocalVariableReadNode.new(load_required_constant, load_varint, location)
-          when 92 then
+          when 91 then
             LocalVariableTargetNode.new(load_required_constant, load_varint, location)
-          when 93 then
+          when 92 then
             LocalVariableWriteNode.new(load_required_constant, load_varint, load_location, load_node, load_location, location)
-          when 94 then
+          when 93 then
             MatchLastLineNode.new(load_location, load_location, load_location, load_string, load_varint, location)
-          when 95 then
+          when 94 then
             MatchPredicateNode.new(load_node, load_node, load_location, location)
-          when 96 then
+          when 95 then
             MatchRequiredNode.new(load_node, load_node, load_location, location)
-          when 97 then
+          when 96 then
             MatchWriteNode.new(load_node, Array.new(load_varint) { load_required_constant }, location)
-          when 98 then
+          when 97 then
             MissingNode.new(location)
-          when 99 then
+          when 98 then
             ModuleNode.new(Array.new(load_varint) { load_required_constant }, load_location, load_node, load_optional_node, load_location, load_required_constant, location)
-          when 100 then
+          when 99 then
             MultiTargetNode.new(Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
-          when 101 then
+          when 100 then
             MultiWriteNode.new(Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, load_location, load_node, location)
-          when 102 then
+          when 101 then
             NextNode.new(load_optional_node, load_location, location)
-          when 103 then
+          when 102 then
             NilNode.new(location)
-          when 104 then
+          when 103 then
             NoKeywordsParameterNode.new(load_location, load_location, location)
-          when 105 then
+          when 104 then
             NumberedReferenceReadNode.new(load_varint, location)
+          when 105 then
+            OptionalKeywordParameterNode.new(load_required_constant, load_location, load_node, location)
           when 106 then
             OptionalParameterNode.new(load_required_constant, load_location, load_location, load_node, location)
           when 107 then
@@ -457,54 +482,56 @@ module Prism
           when 118 then
             RegularExpressionNode.new(load_location, load_location, load_location, load_string, load_varint, location)
           when 119 then
-            RequiredParameterNode.new(load_required_constant, location)
+            RequiredKeywordParameterNode.new(load_required_constant, load_location, location)
           when 120 then
-            RescueModifierNode.new(load_node, load_location, load_node, location)
+            RequiredParameterNode.new(load_required_constant, location)
           when 121 then
-            RescueNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
+            RescueModifierNode.new(load_node, load_location, load_node, location)
           when 122 then
-            RestParameterNode.new(load_optional_constant, load_optional_location, load_location, location)
+            RescueNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
           when 123 then
-            RetryNode.new(location)
+            RestParameterNode.new(load_optional_constant, load_optional_location, load_location, location)
           when 124 then
-            ReturnNode.new(load_location, load_optional_node, location)
+            RetryNode.new(location)
           when 125 then
-            SelfNode.new(location)
+            ReturnNode.new(load_location, load_optional_node, location)
           when 126 then
-            SingletonClassNode.new(Array.new(load_varint) { load_required_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
+            SelfNode.new(location)
           when 127 then
-            SourceEncodingNode.new(location)
+            SingletonClassNode.new(Array.new(load_varint) { load_required_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
           when 128 then
-            SourceFileNode.new(load_string, location)
+            SourceEncodingNode.new(location)
           when 129 then
-            SourceLineNode.new(location)
+            SourceFileNode.new(load_string, location)
           when 130 then
-            SplatNode.new(load_location, load_optional_node, location)
+            SourceLineNode.new(location)
           when 131 then
-            StatementsNode.new(Array.new(load_varint) { load_node }, location)
+            SplatNode.new(load_location, load_optional_node, location)
           when 132 then
-            StringConcatNode.new(load_node, load_node, location)
+            StatementsNode.new(Array.new(load_varint) { load_node }, location)
           when 133 then
-            StringNode.new(load_varint, load_optional_location, load_location, load_optional_location, load_string, location)
+            StringConcatNode.new(load_node, load_node, location)
           when 134 then
-            SuperNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
+            StringNode.new(load_varint, load_optional_location, load_location, load_optional_location, load_string, location)
           when 135 then
-            SymbolNode.new(load_optional_location, load_optional_location, load_optional_location, load_string, location)
+            SuperNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
           when 136 then
-            TrueNode.new(location)
+            SymbolNode.new(load_optional_location, load_optional_location, load_optional_location, load_string, location)
           when 137 then
-            UndefNode.new(Array.new(load_varint) { load_node }, load_location, location)
+            TrueNode.new(location)
           when 138 then
-            UnlessNode.new(load_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
+            UndefNode.new(Array.new(load_varint) { load_node }, load_location, location)
           when 139 then
-            UntilNode.new(load_location, load_optional_location, load_node, load_optional_node, load_varint, location)
+            UnlessNode.new(load_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
           when 140 then
-            WhenNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_node, location)
+            UntilNode.new(load_location, load_optional_location, load_node, load_optional_node, load_varint, location)
           when 141 then
-            WhileNode.new(load_location, load_optional_location, load_node, load_optional_node, load_varint, location)
+            WhenNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_node, location)
           when 142 then
-            XStringNode.new(load_location, load_location, load_location, load_string, location)
+            WhileNode.new(load_location, load_optional_location, load_node, load_optional_node, load_varint, location)
           when 143 then
+            XStringNode.new(load_location, load_location, load_location, load_string, location)
+          when 144 then
             YieldNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, location)
           end
         end
@@ -854,10 +881,6 @@ module Prism
               location = load_location
               KeywordHashNode.new(Array.new(load_varint) { load_node }, location)
             },
-            -> {
-              location = load_location
-              KeywordParameterNode.new(load_required_constant, load_location, load_optional_node, location)
-            },
             -> {
               location = load_location
               KeywordRestParameterNode.new(load_optional_constant, load_optional_location, load_location, location)
@@ -938,6 +961,10 @@ module Prism
               location = load_location
               NumberedReferenceReadNode.new(load_varint, location)
             },
+            -> {
+              location = load_location
+              OptionalKeywordParameterNode.new(load_required_constant, load_location, load_node, location)
+            },
             -> {
               location = load_location
               OptionalParameterNode.new(load_required_constant, load_location, load_location, load_node, location)
@@ -990,6 +1017,10 @@ module Prism
               location = load_location
               RegularExpressionNode.new(load_location, load_location, load_location, load_string, load_varint, location)
             },
+            -> {
+              location = load_location
+              RequiredKeywordParameterNode.new(load_required_constant, load_location, location)
+            },
             -> {
               location = load_location
               RequiredParameterNode.new(load_required_constant, location)
@@ -1095,6 +1126,7 @@ module Prism
       end
     end
 
+    # The token types that can be indexed by their enum values.
     TOKEN_TYPES = [
       nil,
       :EOF,

data/lib/prism/visitor.rb

@@ -11,14 +11,18 @@ module Prism
   # implement each one that they need. For a default implementation that
   # continues walking the tree, see the Visitor class.
   class BasicVisitor
+    # Calls `accept` on the given node if it is not `nil`, which in turn should
+    # call back into this visitor by calling the appropriate `visit_*` method.
     def visit(node)
       node&.accept(self)
     end
 
+    # Visits each node in `nodes` by calling `accept` on each one.
     def visit_all(nodes)
       nodes.each { |node| node&.accept(self) }
     end
 
+    # Visits the child nodes of `node` by calling `accept` on each one.
     def visit_child_nodes(node)
       node.compact_child_nodes.each { |node| node.accept(self) }
     end
@@ -296,9 +300,6 @@ module Prism
     # Visit a KeywordHashNode node
     alias visit_keyword_hash_node visit_child_nodes
 
-    # Visit a KeywordParameterNode node
-    alias visit_keyword_parameter_node visit_child_nodes
-
     # Visit a KeywordRestParameterNode node
     alias visit_keyword_rest_parameter_node visit_child_nodes
 
@@ -359,6 +360,9 @@ module Prism
     # Visit a NumberedReferenceReadNode node
     alias visit_numbered_reference_read_node visit_child_nodes
 
+    # Visit a OptionalKeywordParameterNode node
+    alias visit_optional_keyword_parameter_node visit_child_nodes
+
     # Visit a OptionalParameterNode node
     alias visit_optional_parameter_node visit_child_nodes
 
@@ -398,6 +402,9 @@ module Prism
     # Visit a RegularExpressionNode node
     alias visit_regular_expression_node visit_child_nodes
 
+    # Visit a RequiredKeywordParameterNode node
+    alias visit_required_keyword_parameter_node visit_child_nodes
+
     # Visit a RequiredParameterNode node
     alias visit_required_parameter_node visit_child_nodes
 

data/lib/prism.rb

@@ -1,10 +1,16 @@
 # frozen_string_literal: true
 
+# The Prism Ruby parser.
+#
+# "Parsing Ruby is suddenly manageable!"
+#   - You, hopefully
+#
 module Prism
   # There are many files in prism that are templated to handle every node type,
   # which means the files can end up being quite large. We autoload them to make
   # our require speed faster since consuming libraries are unlikely to use all
   # of these features.
+
   autoload :BasicVisitor, "prism/visitor"
   autoload :Compiler, "prism/compiler"
   autoload :Debug, "prism/debug"
@@ -23,17 +29,26 @@ module Prism
 
   # Some of these constants are not meant to be exposed, so marking them as
   # private here.
+
   private_constant :Debug
   private_constant :LexCompat
   private_constant :LexRipper
 
+  # :call-seq:
+  #   Prism::lex_compat(source, **options) -> Array
+  #
   # Returns an array of tokens that closely resembles that of the Ripper lexer.
   # The only difference is that since we don't keep track of lexer state in the
   # same way, it's going to always return the NONE state.
-  def self.lex_compat(source, filepath = "")
-    LexCompat.new(source, filepath).result
+  #
+  # For supported options, see Prism::parse.
+  def self.lex_compat(source, **options)
+    LexCompat.new(source, **options).result
   end
 
+  # :call-seq:
+  #   Prism::lex_ripper(source) -> Array
+  #
   # This lexes with the Ripper lex. It drops any space events but otherwise
   # returns the same tokens. Raises SyntaxError if the syntax in source is
   # invalid.
@@ -41,6 +56,9 @@ module Prism
     LexRipper.new(source).result
   end
 
+  # :call-seq:
+  #   Prism::load(source, serialized) -> ParseResult
+  #
   # Load the serialized AST using the source as a reference into a tree.
   def self.load(source, serialized)
     Serialize.load(source, serialized)

data/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "0.16.0"
+  spec.version = "0.17.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -30,7 +30,6 @@ Gem::Specification.new do |spec|
     "docs/heredocs.md",
     "docs/javascript.md",
     "docs/mapping.md",
-    "docs/prism.png",
     "docs/releasing.md",
     "docs/ripper.md",
     "docs/ruby_api.md",
@@ -46,6 +45,7 @@ Gem::Specification.new do |spec|
     "include/prism/diagnostic.h",
     "include/prism/enc/pm_encoding.h",
     "include/prism/node.h",
+    "include/prism/options.h",
     "include/prism/pack.h",
     "include/prism/parser.h",
     "include/prism/prettyprint.h",
@@ -57,6 +57,7 @@ Gem::Specification.new do |spec|
     "include/prism/util/pm_memchr.h",
     "include/prism/util/pm_newline_list.h",
     "include/prism/util/pm_state_stack.h",
+    "include/prism/util/pm_strncasecmp.h",
     "include/prism/util/pm_string.h",
     "include/prism/util/pm_string_list.h",
     "include/prism/util/pm_strpbrk.h",
@@ -106,6 +107,7 @@ Gem::Specification.new do |spec|
     "src/util/pm_string_list.c",
     "src/util/pm_strncasecmp.c",
     "src/util/pm_strpbrk.c",
+    "src/options.c",
     "src/prism.c",
     "prism.gemspec",
     "sig/prism.rbs",