prism 0.21.0 → 0.22.0

data/lib/prism/parse_result.rb

@@ -91,7 +91,8 @@ module Prism
   class Location
     # A Source object that is used to determine more information from the given
     # offset and length.
-    protected attr_reader :source
+    attr_reader :source
+    protected :source
 
     # The byte offset from the beginning of the source where this location
     # starts.

data/lib/prism/ripper_compat.rb

@@ -97,6 +97,8 @@ module Prism
         result.errors.each do |error|
           on_parse_error(error.message)
         end
+
+        nil
       else
         result.value.accept(self)
       end
@@ -106,41 +108,91 @@ module Prism
     # Visitor methods
     ############################################################################
 
+    # Visit an ArrayNode node.
+    def visit_array_node(node)
+      elements = visit_elements(node.elements) unless node.elements.empty?
+      bounds(node.location)
+      on_array(elements)
+    end
+
     # Visit a CallNode node.
     def visit_call_node(node)
-      if !node.message.match?(/^[[:alpha:]_]/) && node.opening_loc.nil? && node.arguments&.arguments&.length == 1
-        left = visit(node.receiver)
-        right = visit(node.arguments.arguments.first)
+      if node.variable_call?
+        if node.message.match?(/^[[:alpha:]_]/)
+          bounds(node.message_loc)
+          return on_vcall(on_ident(node.message))
+        end
 
-        bounds(node.location)
-        on_binary(left, node.name, right)
+        raise NotImplementedError, "Non-alpha variable call"
+      end
+
+      if node.opening_loc.nil?
+        left = visit(node.receiver)
+        if node.arguments&.arguments&.length == 1
+          right = visit(node.arguments.arguments.first)
+
+          on_binary(left, node.name, right)
+        elsif !node.arguments || node.arguments.empty?
+          on_unary(node.name, left)
+        else
+          raise NotImplementedError, "More than two arguments for operator"
+        end
       else
-        raise NotImplementedError
+        raise NotImplementedError, "Non-nil opening_loc"
       end
     end
 
     # Visit a FloatNode node.
     def visit_float_node(node)
-      bounds(node.location)
-      on_float(node.slice)
+      visit_number(node) { |text| on_float(text) }
     end
 
     # Visit a ImaginaryNode node.
     def visit_imaginary_node(node)
-      bounds(node.location)
-      on_imaginary(node.slice)
+      visit_number(node) { |text| on_imaginary(text) }
     end
 
     # Visit an IntegerNode node.
     def visit_integer_node(node)
+      visit_number(node) { |text| on_int(text) }
+    end
+
+    # Visit a ParenthesesNode node.
+    def visit_parentheses_node(node)
+      body =
+        if node.body.nil?
+          on_stmts_add(on_stmts_new, on_void_stmt)
+        else
+          visit(node.body)
+        end
+
+      bounds(node.location)
+      on_paren(body)
+    end
+
+    # Visit a ProgramNode node.
+    def visit_program_node(node)
+      statements = visit(node.statements)
+      bounds(node.location)
+      on_program(statements)
+    end
+
+    # Visit a RangeNode node.
+    def visit_range_node(node)
+      left = visit(node.left)
+      right = visit(node.right)
+
       bounds(node.location)
-      on_int(node.slice)
+      if node.exclude_end?
+        on_dot3(left, right)
+      else
+        on_dot2(left, right)
+      end
     end
 
     # Visit a RationalNode node.
     def visit_rational_node(node)
-      bounds(node.location)
-      on_rational(node.slice)
+      visit_number(node) { |text| on_rational(text) }
     end
 
     # Visit a StatementsNode node.
@@ -151,13 +203,6 @@ module Prism
       end
     end
 
-    # Visit a ProgramNode node.
-    def visit_program_node(node)
-      statements = visit(node.statements)
-      bounds(node.location)
-      on_program(statements)
-    end
-
     ############################################################################
     # Entrypoints for subclasses
     ############################################################################
@@ -174,6 +219,32 @@ module Prism
 
     private
 
+    # Visit a list of elements, like the elements of an array or arguments.
+    def visit_elements(elements)
+      bounds(elements.first.location)
+      elements.inject(on_args_new) do |args, element|
+        on_args_add(args, visit(element))
+      end
+    end
+
+    # Visit a node that represents a number. We need to explicitly handle the
+    # unary - operator.
+    def visit_number(node)
+      slice = node.slice
+      location = node.location
+
+      if slice[0] == "-"
+        bounds_values(location.start_line, location.start_column + 1)
+        value = yield slice[1..-1]
+
+        bounds(node.location)
+        on_unary(RUBY_ENGINE == "jruby" ? :- : :-@, value)
+      else
+        bounds(location)
+        yield slice
+      end
+    end
+
     # This method is responsible for updating lineno and column information
     # to reflect the current node.
     #
@@ -184,6 +255,13 @@ module Prism
       @column = location.start_column
     end
 
+    # If we need to do something unusual, we can directly update the line number
+    # and column to reflect the current node.
+    def bounds_values(lineno, column)
+      @lineno = lineno
+      @column = column
+    end
+
     # Lazily initialize the parse result.
     def result
       @result ||= Prism.parse(source)

data/lib/prism/serialize.rb

@@ -27,7 +27,7 @@ module Prism
 
     # The minor version of prism that we are expecting to find in the serialized
     # strings.
-    MINOR_VERSION = 21
+    MINOR_VERSION = 22
 
     # The patch version of prism that we are expecting to find in the serialized
     # strings.
@@ -244,6 +244,8 @@ module Prism
         case level
         when 0
           :fatal
+        when 1
+          :argument
         else
           raise "Unknown level: #{level}"
         end

data/lib/prism/translation/parser/compiler.rb

@@ -1062,12 +1062,22 @@ module Prism
 
         # foo in bar
         # ^^^^^^^^^^
-        def visit_match_predicate_node(node)
-          builder.match_pattern_p(
-            visit(node.value),
-            token(node.operator_loc),
-            within_pattern { |compiler| node.pattern.accept(compiler) }
-          )
+        if RUBY_VERSION >= "3.0"
+          def visit_match_predicate_node(node)
+            builder.match_pattern_p(
+              visit(node.value),
+              token(node.operator_loc),
+              within_pattern { |compiler| node.pattern.accept(compiler) }
+            )
+          end
+        else
+          def visit_match_predicate_node(node)
+            builder.match_pattern(
+              visit(node.value),
+              token(node.operator_loc),
+              within_pattern { |compiler| node.pattern.accept(compiler) }
+            )
+          end
         end
 
         # foo => bar

data/lib/prism/translation/parser.rb

@@ -68,17 +68,23 @@ module Prism
 
       # Parses a source buffer and returns the AST, the source code comments,
       # and the tokens emitted by the lexer.
-      def tokenize(source_buffer, _recover = false)
+      def tokenize(source_buffer, recover = false)
         @source_buffer = source_buffer
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse_lex(source, filepath: source_buffer.name), offset_cache)
+        result =
+          begin
+            unwrap(Prism.parse_lex(source, filepath: source_buffer.name), offset_cache)
+          rescue ::Parser::SyntaxError
+            raise if !recover
+          end
 
         program, tokens = result.value
+        ast = build_ast(program, offset_cache) if result.success?
 
         [
-          build_ast(program, offset_cache),
+          ast,
           build_comments(result.comments, offset_cache),
           build_tokens(tokens, offset_cache)
         ]

data/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "0.21.0"
+  spec.version = "0.22.0"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.homepage = "https://github.com/ruby/prism"
   spec.license = "MIT"
 
-  spec.required_ruby_version = ">= 3.0.0"
+  spec.required_ruby_version = ">= 2.7.0"
 
   spec.require_paths = ["lib"]
   spec.files = [

data/src/diagnostic.c

@@ -63,7 +63,8 @@ typedef struct {
  *
  * For errors, they are:
  *
- * * `PM_ERROR_LEVEL_FATAL` - The level for all errors.
+ * * `PM_ERROR_LEVEL_FATAL` - The default level for errors.
+ * * `PM_ERROR_LEVEL_ARGUMENT` - Errors that should raise ArgumentError.
  *
  * For warnings, they are:
  *
@@ -71,9 +72,13 @@ typedef struct {
  * * `PM_WARNING_LEVEL_VERBOSE` - Warnings that appear with `-w`, as in `ruby -w -c -e 'code'`.
  */
 static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_LEN] = {
+    // Special error that can be replaced
     [PM_ERR_CANNOT_PARSE_EXPRESSION]            = { "cannot parse the expression", PM_ERROR_LEVEL_FATAL },
 
-    // Errors
+    // Errors that should raise argument errors
+    [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT]     = { "unknown or invalid encoding in the magic comment", PM_ERROR_LEVEL_ARGUMENT },
+
+    // Errors that should raise syntax errors
     [PM_ERR_ALIAS_ARGUMENT]                     = { "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_AMPAMPEQ_MULTI_ASSIGN]              = { "unexpected `&&=` in a multiple assignment", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_ARGUMENT_AFTER_BLOCK]               = { "unexpected argument after a block argument", PM_ERROR_LEVEL_FATAL },
@@ -187,15 +192,16 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_LEN] = {
     [PM_ERR_INCOMPLETE_QUESTION_MARK]           = { "incomplete expression at `?`", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INCOMPLETE_VARIABLE_CLASS]          = { "incomplete class variable", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE]       = { "incomplete instance variable", PM_ERROR_LEVEL_FATAL },
-    [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT]     = { "unknown or invalid encoding in the magic comment", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_FLOAT_EXPONENT]             = { "invalid exponent", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_BINARY]              = { "invalid binary number", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_DECIMAL]             = { "invalid decimal number", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_HEXADECIMAL]         = { "invalid hexadecimal number", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_OCTAL]               = { "invalid octal number", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_NUMBER_UNDERSCORE]          = { "invalid underscore placement in number", PM_ERROR_LEVEL_FATAL },
+    [PM_ERR_INVALID_CHARACTER]                  = { "invalid character 0x%X", PM_ERROR_LEVEL_FATAL },
+    [PM_ERR_INVALID_MULTIBYTE_CHARACTER]        = { "invalid multibyte character 0x%X", PM_ERROR_LEVEL_FATAL },
+    [PM_ERR_INVALID_PRINTABLE_CHARACTER]        = { "invalid character `%c`", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_INVALID_PERCENT]                    = { "invalid `%` token", PM_ERROR_LEVEL_FATAL }, // TODO WHAT?
-    [PM_ERR_INVALID_TOKEN]                      = { "invalid token", PM_ERROR_LEVEL_FATAL }, // TODO WHAT?
     [PM_ERR_INVALID_VARIABLE_GLOBAL]            = { "invalid global variable", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_IT_NOT_ALLOWED]                     = { "`it` is not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_FATAL },
     [PM_ERR_LAMBDA_OPEN]                        = { "expected a `do` keyword or a `{` to open the lambda block", PM_ERROR_LEVEL_FATAL },

data/src/prism.c

@@ -9590,11 +9590,21 @@ parser_lex(pm_parser_t *parser) {
                     if (*parser->current.start != '_') {
                         size_t width = char_is_identifier_start(parser, parser->current.start);
 
-                        // If this isn't the beginning of an identifier, then it's an invalid
-                        // token as we've exhausted all of the other options. We'll skip past
-                        // it and return the next token.
+                        // If this isn't the beginning of an identifier, then
+                        // it's an invalid token as we've exhausted all of the
+                        // other options. We'll skip past it and return the next
+                        // token after adding an appropriate error message.
                         if (!width) {
-                            pm_parser_err_current(parser, PM_ERR_INVALID_TOKEN);
+                            pm_diagnostic_id_t diag_id;
+                            if (*parser->current.start >= 0x80) {
+                                diag_id = PM_ERR_INVALID_MULTIBYTE_CHARACTER;
+                            } else if (char_is_ascii_printable(*parser->current.start) || (*parser->current.start == '\\')) {
+                                diag_id = PM_ERR_INVALID_PRINTABLE_CHARACTER;
+                            } else {
+                                diag_id = PM_ERR_INVALID_CHARACTER;
+                            }
+
+                            PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, *parser->current.start);
                             goto lex_next_token;
                         }
 
@@ -12377,25 +12387,10 @@ parse_rescues(pm_parser_t *parser, pm_begin_node_t *parent_node, bool def_p) {
 }
 
 static inline pm_begin_node_t *
-parse_rescues_as_begin(pm_parser_t *parser, pm_statements_node_t *statements, bool def_p) {
+parse_rescues_as_begin(pm_parser_t *parser, const uint8_t *start, pm_statements_node_t *statements, bool def_p) {
     pm_token_t no_begin_token = not_provided(parser);
     pm_begin_node_t *begin_node = pm_begin_node_create(parser, &no_begin_token, statements);
     parse_rescues(parser, begin_node, def_p);
-
-    // All nodes within a begin node are optional, so we look
-    // for the earliest possible node that we can use to set
-    // the BeginNode's start location
-    const uint8_t *start = begin_node->base.location.start;
-    if (begin_node->statements) {
-        start = begin_node->statements->base.location.start;
-    } else if (begin_node->rescue_clause) {
-        start = begin_node->rescue_clause->base.location.start;
-    } else if (begin_node->else_clause) {
-        start = begin_node->else_clause->base.location.start;
-    } else if (begin_node->ensure_clause) {
-        start = begin_node->ensure_clause->base.location.start;
-    }
-
     begin_node->base.location.start = start;
     return begin_node;
 }
@@ -12490,7 +12485,7 @@ parse_block(pm_parser_t *parser) {
 
             if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
                 assert(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
-                statements = (pm_node_t *) parse_rescues_as_begin(parser, (pm_statements_node_t *) statements, false);
+                statements = (pm_node_t *) parse_rescues_as_begin(parser, opening.start, (pm_statements_node_t *) statements, false);
             }
         }
 
@@ -15290,7 +15285,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
 
                 if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
                     assert(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
-                    statements = (pm_node_t *) parse_rescues_as_begin(parser, (pm_statements_node_t *) statements, false);
+                    statements = (pm_node_t *) parse_rescues_as_begin(parser, class_keyword.start, (pm_statements_node_t *) statements, false);
                 }
 
                 expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CLASS_TERM);
@@ -15343,7 +15338,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
 
             if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
                 assert(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
-                statements = (pm_node_t *) parse_rescues_as_begin(parser, (pm_statements_node_t *) statements, false);
+                statements = (pm_node_t *) parse_rescues_as_begin(parser, class_keyword.start, (pm_statements_node_t *) statements, false);
             }
 
             expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_CLASS_TERM);
@@ -15612,7 +15607,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
 
                 if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
                     assert(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
-                    statements = (pm_node_t *) parse_rescues_as_begin(parser, (pm_statements_node_t *) statements, true);
+                    statements = (pm_node_t *) parse_rescues_as_begin(parser, def_keyword.start, (pm_statements_node_t *) statements, true);
                 }
 
                 pm_accepts_block_stack_pop(parser);
@@ -15872,7 +15867,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
 
             if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
                 assert(statements == NULL || PM_NODE_TYPE_P(statements, PM_STATEMENTS_NODE));
-                statements = (pm_node_t *) parse_rescues_as_begin(parser, (pm_statements_node_t *) statements, false);
+                statements = (pm_node_t *) parse_rescues_as_begin(parser, module_keyword.start, (pm_statements_node_t *) statements, false);
             }
 
             pm_constant_id_list_t locals = parser->current_scope->locals;
@@ -16605,7 +16600,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
 
                 if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
                     assert(body == NULL || PM_NODE_TYPE_P(body, PM_STATEMENTS_NODE));
-                    body = (pm_node_t *) parse_rescues_as_begin(parser, (pm_statements_node_t *) body, false);
+                    body = (pm_node_t *) parse_rescues_as_begin(parser, opening.start, (pm_statements_node_t *) body, false);
                 }
 
                 expect1(parser, PM_TOKEN_KEYWORD_END, PM_ERR_LAMBDA_TERM_END);

data/src/util/pm_string.c

@@ -65,7 +65,6 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     HANDLE file = CreateFile(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
 
     if (file == INVALID_HANDLE_VALUE) {
-        perror("CreateFile failed");
         return false;
     }
 
@@ -73,7 +72,6 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     DWORD file_size = GetFileSize(file, NULL);
     if (file_size == INVALID_FILE_SIZE) {
         CloseHandle(file);
-        perror("GetFileSize failed");
         return false;
     }
 
@@ -90,7 +88,6 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     HANDLE mapping = CreateFileMapping(file, NULL, PAGE_READONLY, 0, 0, NULL);
     if (mapping == NULL) {
         CloseHandle(file);
-        perror("CreateFileMapping failed");
         return false;
     }
 
@@ -100,7 +97,6 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     CloseHandle(file);
 
     if (source == NULL) {
-        perror("MapViewOfFile failed");
         return false;
     }
 
@@ -110,7 +106,6 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     // Open the file for reading
     int fd = open(filepath, O_RDONLY);
     if (fd == -1) {
-        perror("open");
         return false;
     }
 
@@ -118,7 +113,6 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     struct stat sb;
     if (fstat(fd, &sb) == -1) {
         close(fd);
-        perror("fstat");
         return false;
     }
 
@@ -135,7 +129,6 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 
     source = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
     if (source == MAP_FAILED) {
-        perror("Map failed");
         return false;
     }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prism
 version: !ruby/object:Gem::Version
-  version: 0.21.0
+  version: 0.22.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-05 00:00:00.000000000 Z
+date: 2024-02-07 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -137,14 +137,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.6.0.dev
 signing_key:
 specification_version: 4
 summary: Prism Ruby parser