prism 0.22.0 → 0.24.0

data/ext/prism/extension.c

@@ -542,8 +542,9 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
     pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
     pm_parser_register_encoding_changed_callback(&parser, parse_lex_encoding_changed_callback);
 
+    VALUE source_string = rb_str_new((const char *) pm_string_source(input), pm_string_length(input));
     VALUE offsets = rb_ary_new();
-    VALUE source_argv[] = { rb_str_new((const char *) pm_string_source(input), pm_string_length(input)), ULONG2NUM(parser.start_line), offsets };
+    VALUE source_argv[] = { source_string, LONG2NUM(parser.start_line), offsets };
     VALUE source = rb_class_new_instance(3, source_argv, rb_cPrismSource);
 
     parse_lex_data_t parse_lex_data = {
@@ -561,17 +562,21 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
     parser.lex_callback = &lex_callback;
     pm_node_t *node = pm_parse(&parser);
 
-    // Here we need to update the source range to have the correct newline
-    // offsets. We do it here because we've already created the object and given
-    // it over to all of the tokens.
+    // Here we need to update the Source object to have the correct
+    // encoding for the source string and the correct newline offsets.
+    // We do it here because we've already created the Source object and given
+    // it over to all of the tokens, and both of these are only set after pm_parse().
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
+    rb_enc_associate(source_string, encoding);
+
     for (size_t index = 0; index < parser.newline_list.size; index++) {
-        rb_ary_push(offsets, INT2FIX(parser.newline_list.offsets[index]));
+        rb_ary_push(offsets, ULONG2NUM(parser.newline_list.offsets[index]));
     }
 
     VALUE value;
     if (return_nodes) {
         value = rb_ary_new_capa(2);
-        rb_ary_push(value, pm_ast_new(&parser, node, parse_lex_data.encoding));
+        rb_ary_push(value, pm_ast_new(&parser, node, parse_lex_data.encoding, source));
         rb_ary_push(value, parse_lex_data.tokens);
     } else {
         value = parse_lex_data.tokens;
@@ -650,7 +655,7 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
 
     VALUE source = pm_source_new(&parser, encoding);
     VALUE result_argv[] = {
-        pm_ast_new(&parser, node, encoding),
+        pm_ast_new(&parser, node, encoding, source),
         parser_comments(&parser, source),
         parser_magic_comments(&parser, source),
         parser_data_loc(&parser, source),

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "0.22.0"
+#define EXPECTED_PRISM_VERSION "0.24.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>
@@ -9,7 +9,7 @@
 
 VALUE pm_source_new(pm_parser_t *parser, rb_encoding *encoding);
 VALUE pm_token_new(pm_parser_t *parser, pm_token_t *token, rb_encoding *encoding, VALUE source);
-VALUE pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding);
+VALUE pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding, VALUE source);
 
 void Init_prism_api_node(void);
 void Init_prism_pack(void);

data/include/prism/diagnostic.h

@@ -219,6 +219,7 @@ typedef enum {
     PM_ERR_MODULE_NAME,
     PM_ERR_MODULE_TERM,
     PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
+    PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST,
     PM_ERR_NOT_EXPRESSION,
     PM_ERR_NO_LOCAL_VARIABLE,
     PM_ERR_NUMBER_LITERAL_UNDERSCORE,
@@ -272,6 +273,7 @@ typedef enum {
     PM_ERR_STATEMENT_UNDEF,
     PM_ERR_STRING_CONCATENATION,
     PM_ERR_STRING_INTERPOLATED_TERM,
+    PM_ERR_STRING_LITERAL_EOF,
     PM_ERR_STRING_LITERAL_TERM,
     PM_ERR_SYMBOL_INVALID,
     PM_ERR_SYMBOL_TERM_DYNAMIC,
@@ -279,10 +281,7 @@ typedef enum {
     PM_ERR_TERNARY_COLON,
     PM_ERR_TERNARY_EXPRESSION_FALSE,
     PM_ERR_TERNARY_EXPRESSION_TRUE,
-    PM_ERR_UNARY_RECEIVER_BANG,
-    PM_ERR_UNARY_RECEIVER_MINUS,
-    PM_ERR_UNARY_RECEIVER_PLUS,
-    PM_ERR_UNARY_RECEIVER_TILDE,
+    PM_ERR_UNARY_RECEIVER,
     PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT,
     PM_ERR_UNEXPECTED_TOKEN_IGNORE,
     PM_ERR_UNDEF_ARGUMENT,

data/include/prism/encoding.h

@@ -245,6 +245,13 @@ extern const pm_encoding_t pm_encodings[PM_ENCODING_MAXIMUM];
  */
 #define PM_ENCODING_US_ASCII_ENTRY (&pm_encodings[PM_ENCODING_US_ASCII])
 
+/**
+ * This is the ASCII-8BIT encoding. We need a reference to it so that pm_strpbrk
+ * can compare against it because invalid multibyte characters are not a thing
+ * in this encoding.
+ */
+#define PM_ENCODING_ASCII_8BIT_ENTRY (&pm_encodings[PM_ENCODING_ASCII_8BIT])
+
 /**
  * Parse the given name of an encoding and return a pointer to the corresponding
  * encoding struct if one can be found, otherwise return NULL.

data/include/prism/util/pm_constant_pool.h

@@ -163,7 +163,7 @@ pm_constant_t * pm_constant_pool_id_to_constant(const pm_constant_pool_t *pool,
  * @param length The length of the constant.
  * @return The id of the constant.
  */
-pm_constant_id_t pm_constant_pool_find(pm_constant_pool_t *pool, const uint8_t *start, size_t length);
+pm_constant_id_t pm_constant_pool_find(const pm_constant_pool_t *pool, const uint8_t *start, size_t length);
 
 /**
  * Insert a constant into a constant pool that is a slice of a source string.

data/include/prism/util/pm_newline_list.h

@@ -44,10 +44,10 @@ typedef struct {
  */
 typedef struct {
     /** The line number. */
-    size_t line;
+    int32_t line;
 
     /** The column number. */
-    size_t column;
+    uint32_t column;
 } pm_line_column_t;
 
 /**
@@ -79,9 +79,10 @@ bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor);
  *
  * @param list The list to search.
  * @param cursor A pointer to the offset to search for.
+ * @param start_line The line to start counting from.
  * @return The line and column of the given offset.
  */
-pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor);
+pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line);
 
 /**
  * Free the internal memory allocated for the newline list.

data/include/prism/util/pm_strpbrk.h

@@ -7,6 +7,7 @@
 #define PRISM_STRPBRK_H
 
 #include "prism/defines.h"
+#include "prism/diagnostic.h"
 #include "prism/parser.h"
 
 #include <stddef.h>
@@ -35,9 +36,11 @@
  * @param source The source to search.
  * @param charset The charset to search for.
  * @param length The maximum number of bytes to search.
+ * @param validate Whether to validate that the source string is valid in the
+ *     current encoding of the parser.
  * @return A pointer to the first character in the source string that is in the
  *     charset, or NULL if no such character exists.
  */
-const uint8_t * pm_strpbrk(const pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length);
+const uint8_t * pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate);
 
 #endif

data/include/prism/version.h

@@ -14,7 +14,7 @@
 /**
  * The minor version of the Prism library as an int.
  */
-#define PRISM_VERSION_MINOR 22
+#define PRISM_VERSION_MINOR 24
 
 /**
  * The patch version of the Prism library as an int.
@@ -24,6 +24,6 @@
 /**
  * The version of the Prism library as a constant string.
  */
-#define PRISM_VERSION "0.22.0"
+#define PRISM_VERSION "0.24.0"
 
 #endif

data/lib/prism/desugar_compiler.rb

@@ -1,6 +1,216 @@
 # frozen_string_literal: true
 
 module Prism
+  class DesugarAndWriteNode # :nodoc:
+    attr_reader :node, :source, :read_class, :write_class, :arguments
+
+    def initialize(node, source, read_class, write_class, *arguments)
+      @node = node
+      @source = source
+      @read_class = read_class
+      @write_class = write_class
+      @arguments = arguments
+    end
+
+    # Desugar `x &&= y` to `x && x = y`
+    def compile
+      AndNode.new(
+        source,
+        read_class.new(source, *arguments, node.name_loc),
+        write_class.new(source, *arguments, node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+  end
+
+  class DesugarOrWriteDefinedNode # :nodoc:
+    attr_reader :node, :source, :read_class, :write_class, :arguments
+
+    def initialize(node, source, read_class, write_class, *arguments)
+      @node = node
+      @source = source
+      @read_class = read_class
+      @write_class = write_class
+      @arguments = arguments
+    end
+
+    # Desugar `x ||= y` to `defined?(x) ? x : x = y`
+    def compile
+      IfNode.new(
+        source,
+        node.operator_loc,
+        DefinedNode.new(source, nil, read_class.new(source, *arguments, node.name_loc), nil, node.operator_loc, node.name_loc),
+        node.operator_loc,
+        StatementsNode.new(source, [read_class.new(source, *arguments, node.name_loc)], node.location),
+        ElseNode.new(
+          source,
+          node.operator_loc,
+          StatementsNode.new(
+            source,
+            [write_class.new(source, *arguments, node.name_loc, node.value, node.operator_loc, node.location)],
+            node.location
+          ),
+          node.operator_loc,
+          node.location
+        ),
+        node.operator_loc,
+        node.location
+      )
+    end
+  end
+
+  class DesugarOperatorWriteNode # :nodoc:
+    attr_reader :node, :source, :read_class, :write_class, :arguments
+
+    def initialize(node, source, read_class, write_class, *arguments)
+      @node = node
+      @source = source
+      @read_class = read_class
+      @write_class = write_class
+      @arguments = arguments
+    end
+
+    # Desugar `x += y` to `x = x + y`
+    def compile
+      write_class.new(
+        source,
+        *arguments,
+        node.name_loc,
+        CallNode.new(
+          source,
+          0,
+          read_class.new(source, *arguments, node.name_loc),
+          nil,
+          node.operator_loc.slice.chomp("="),
+          node.operator_loc.copy(length: node.operator_loc.length - 1),
+          nil,
+          ArgumentsNode.new(source, 0, [node.value], node.value.location),
+          nil,
+          nil,
+          node.location
+        ),
+        node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1),
+        node.location
+      )
+    end
+  end
+
+  class DesugarOrWriteNode # :nodoc:
+    attr_reader :node, :source, :read_class, :write_class, :arguments
+
+    def initialize(node, source, read_class, write_class, *arguments)
+      @node = node
+      @source = source
+      @read_class = read_class
+      @write_class = write_class
+      @arguments = arguments
+    end
+
+    # Desugar `x ||= y` to `x || x = y`
+    def compile
+      OrNode.new(
+        source,
+        read_class.new(source, *arguments, node.name_loc),
+        write_class.new(source, *arguments, node.name_loc, node.value, node.operator_loc, node.location),
+        node.operator_loc,
+        node.location
+      )
+    end
+  end
+
+  private_constant :DesugarAndWriteNode, :DesugarOrWriteNode, :DesugarOrWriteDefinedNode, :DesugarOperatorWriteNode
+
+  class ClassVariableAndWriteNode
+    def desugar # :nodoc:
+      DesugarAndWriteNode.new(self, source, ClassVariableReadNode, ClassVariableWriteNode, name).compile
+    end
+  end
+
+  class ClassVariableOrWriteNode
+    def desugar # :nodoc:
+      DesugarOrWriteDefinedNode.new(self, source, ClassVariableReadNode, ClassVariableWriteNode, name).compile
+    end
+  end
+
+  class ClassVariableOperatorWriteNode
+    def desugar # :nodoc:
+      DesugarOperatorWriteNode.new(self, source, ClassVariableReadNode, ClassVariableWriteNode, name).compile
+    end
+  end
+
+  class ConstantAndWriteNode
+    def desugar # :nodoc:
+      DesugarAndWriteNode.new(self, source, ConstantReadNode, ConstantWriteNode, name).compile
+    end
+  end
+
+  class ConstantOrWriteNode
+    def desugar # :nodoc:
+      DesugarOrWriteDefinedNode.new(self, source, ConstantReadNode, ConstantWriteNode, name).compile
+    end
+  end
+
+  class ConstantOperatorWriteNode
+    def desugar # :nodoc:
+      DesugarOperatorWriteNode.new(self, source, ConstantReadNode, ConstantWriteNode, name).compile
+    end
+  end
+
+  class GlobalVariableAndWriteNode
+    def desugar # :nodoc:
+      DesugarAndWriteNode.new(self, source, GlobalVariableReadNode, GlobalVariableWriteNode, name).compile
+    end
+  end
+
+  class GlobalVariableOrWriteNode
+    def desugar # :nodoc:
+      DesugarOrWriteDefinedNode.new(self, source, GlobalVariableReadNode, GlobalVariableWriteNode, name).compile
+    end
+  end
+
+  class GlobalVariableOperatorWriteNode
+    def desugar # :nodoc:
+      DesugarOperatorWriteNode.new(self, source, GlobalVariableReadNode, GlobalVariableWriteNode, name).compile
+    end
+  end
+
+  class InstanceVariableAndWriteNode
+    def desugar # :nodoc:
+      DesugarAndWriteNode.new(self, source, InstanceVariableReadNode, InstanceVariableWriteNode, name).compile
+    end
+  end
+
+  class InstanceVariableOrWriteNode
+    def desugar # :nodoc:
+      DesugarOrWriteNode.new(self, source, InstanceVariableReadNode, InstanceVariableWriteNode, name).compile
+    end
+  end
+
+  class InstanceVariableOperatorWriteNode
+    def desugar # :nodoc:
+      DesugarOperatorWriteNode.new(self, source, InstanceVariableReadNode, InstanceVariableWriteNode, name).compile
+    end
+  end
+
+  class LocalVariableAndWriteNode
+    def desugar # :nodoc:
+      DesugarAndWriteNode.new(self, source, LocalVariableReadNode, LocalVariableWriteNode, name, depth).compile
+    end
+  end
+
+  class LocalVariableOrWriteNode
+    def desugar # :nodoc:
+      DesugarOrWriteNode.new(self, source, LocalVariableReadNode, LocalVariableWriteNode, name, depth).compile
+    end
+  end
+
+  class LocalVariableOperatorWriteNode
+    def desugar # :nodoc:
+      DesugarOperatorWriteNode.new(self, source, LocalVariableReadNode, LocalVariableWriteNode, name, depth).compile
+    end
+  end
+
   # DesugarCompiler is a compiler that desugars Ruby code into a more primitive
   # form. This is useful for consumers that want to deal with fewer node types.
   class DesugarCompiler < MutationCompiler
@@ -10,7 +220,7 @@ module Prism
     #
     # @@foo && @@foo = bar
     def visit_class_variable_and_write_node(node)
-      desugar_and_write_node(node, ClassVariableReadNode, ClassVariableWriteNode, node.name)
+      node.desugar
     end
 
     # @@foo ||= bar
@@ -19,7 +229,7 @@ module Prism
     #
     # defined?(@@foo) ? @@foo : @@foo = bar
     def visit_class_variable_or_write_node(node)
-      desugar_or_write_defined_node(node, ClassVariableReadNode, ClassVariableWriteNode, node.name)
+      node.desugar
     end
 
     # @@foo += bar
@@ -28,7 +238,7 @@ module Prism
     #
     # @@foo = @@foo + bar
     def visit_class_variable_operator_write_node(node)
-      desugar_operator_write_node(node, ClassVariableReadNode, ClassVariableWriteNode, node.name)
+      node.desugar
     end
 
     # Foo &&= bar
@@ -37,7 +247,7 @@ module Prism
     #
     # Foo && Foo = bar
     def visit_constant_and_write_node(node)
-      desugar_and_write_node(node, ConstantReadNode, ConstantWriteNode, node.name)
+      node.desugar
     end
 
     # Foo ||= bar
@@ -46,7 +256,7 @@ module Prism
     #
     # defined?(Foo) ? Foo : Foo = bar
     def visit_constant_or_write_node(node)
-      desugar_or_write_defined_node(node, ConstantReadNode, ConstantWriteNode, node.name)
+      node.desugar
     end
 
     # Foo += bar
@@ -55,7 +265,7 @@ module Prism
     #
     # Foo = Foo + bar
     def visit_constant_operator_write_node(node)
-      desugar_operator_write_node(node, ConstantReadNode, ConstantWriteNode, node.name)
+      node.desugar
     end
 
     # $foo &&= bar
@@ -64,7 +274,7 @@ module Prism
     #
     # $foo && $foo = bar
     def visit_global_variable_and_write_node(node)
-      desugar_and_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode, node.name)
+      node.desugar
     end
 
     # $foo ||= bar
@@ -73,7 +283,7 @@ module Prism
     #
     # defined?($foo) ? $foo : $foo = bar
     def visit_global_variable_or_write_node(node)
-      desugar_or_write_defined_node(node, GlobalVariableReadNode, GlobalVariableWriteNode, node.name)
+      node.desugar
     end
 
     # $foo += bar
@@ -82,7 +292,7 @@ module Prism
     #
     # $foo = $foo + bar
     def visit_global_variable_operator_write_node(node)
-      desugar_operator_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode, node.name)
+      node.desugar
     end
 
     # @foo &&= bar
@@ -91,7 +301,7 @@ module Prism
     #
     # @foo && @foo = bar
     def visit_instance_variable_and_write_node(node)
-      desugar_and_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, node.name)
+      node.desugar
     end
 
     # @foo ||= bar
@@ -100,7 +310,7 @@ module Prism
     #
     # @foo || @foo = bar
     def visit_instance_variable_or_write_node(node)
-      desugar_or_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, node.name)
+      node.desugar
     end
 
     # @foo += bar
@@ -109,7 +319,7 @@ module Prism
     #
     # @foo = @foo + bar
     def visit_instance_variable_operator_write_node(node)
-      desugar_operator_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, node.name)
+      node.desugar
     end
 
     # foo &&= bar
@@ -118,7 +328,7 @@ module Prism
     #
     # foo && foo = bar
     def visit_local_variable_and_write_node(node)
-      desugar_and_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, node.name, node.depth)
+      node.desugar
     end
 
     # foo ||= bar
@@ -127,7 +337,7 @@ module Prism
     #
     # foo || foo = bar
     def visit_local_variable_or_write_node(node)
-      desugar_or_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, node.name, node.depth)
+      node.desugar
     end
 
     # foo += bar
@@ -136,72 +346,7 @@ module Prism
     #
     # foo = foo + bar
     def visit_local_variable_operator_write_node(node)
-      desugar_operator_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, node.name, node.depth)
-    end
-
-    private
-
-    # Desugar `x &&= y` to `x && x = y`
-    def desugar_and_write_node(node, read_class, write_class, *arguments)
-      AndNode.new(
-        read_class.new(*arguments, node.name_loc),
-        write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location),
-        node.operator_loc,
-        node.location
-      )
-    end
-
-    # Desugar `x += y` to `x = x + y`
-    def desugar_operator_write_node(node, read_class, write_class, *arguments)
-      write_class.new(
-        *arguments,
-        node.name_loc,
-        CallNode.new(
-          0,
-          read_class.new(*arguments, node.name_loc),
-          nil,
-          node.operator_loc.slice.chomp("="),
-          node.operator_loc.copy(length: node.operator_loc.length - 1),
-          nil,
-          ArgumentsNode.new(0, [node.value], node.value.location),
-          nil,
-          nil,
-          node.location
-        ),
-        node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1),
-        node.location
-      )
-    end
-
-    # Desugar `x ||= y` to `x || x = y`
-    def desugar_or_write_node(node, read_class, write_class, *arguments)
-      OrNode.new(
-        read_class.new(*arguments, node.name_loc),
-        write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location),
-        node.operator_loc,
-        node.location
-      )
-    end
-
-    # Desugar `x ||= y` to `defined?(x) ? x : x = y`
-    def desugar_or_write_defined_node(node, read_class, write_class, *arguments)
-      IfNode.new(
-        node.operator_loc,
-        DefinedNode.new(nil, read_class.new(*arguments, node.name_loc), nil, node.operator_loc, node.name_loc),
-        node.operator_loc,
-        StatementsNode.new([read_class.new(*arguments, node.name_loc)], node.location),
-        ElseNode.new(
-          node.operator_loc,
-          StatementsNode.new(
-            [write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location)],
-            node.location
-          ),
-          node.operator_loc,
-          node.location
-        ),
-        node.operator_loc,
-        node.location
-      )
+      node.desugar
     end
   end
 end