prism 1.0.0 → 1.1.0

data/include/prism/defines.h

@@ -25,6 +25,15 @@
 #define __STDC_FORMAT_MACROS
 #include <inttypes.h>
 
+/**
+ * When we are parsing using recursive descent, we want to protect against
+ * malicious payloads that could attempt to crash our parser. We do this by
+ * specifying a maximum depth to which we are allowed to recurse.
+ */
+#ifndef PRISM_DEPTH_MAXIMUM
+    #define PRISM_DEPTH_MAXIMUM 1000
+#endif
+
 /**
  * By default, we compile with -fvisibility=hidden. When this is enabled, we
  * need to mark certain functions as being publically-visible. This macro does
@@ -212,4 +221,22 @@
     #define PRISM_ENCODING_EXCLUDE_FULL
 #endif
 
+/**
+ * Support PRISM_LIKELY and PRISM_UNLIKELY to help the compiler optimize its
+ * branch predication.
+ */
+#if defined(__GNUC__) || defined(__clang__)
+    /** The compiler should predicate that this branch will be taken. */
+    #define PRISM_LIKELY(x) __builtin_expect(!!(x), 1)
+
+    /** The compiler should predicate that this branch will not be taken. */
+    #define PRISM_UNLIKELY(x) __builtin_expect(!!(x), 0)
+#else
+    /** Void because this platform does not support branch prediction hints. */
+    #define PRISM_LIKELY(x)   (x)
+
+    /** Void because this platform does not support branch prediction hints. */
+    #define PRISM_UNLIKELY(x) (x)
+#endif
+
 #endif

data/include/prism/diagnostic.h

@@ -44,7 +44,6 @@ typedef enum {
     PM_ERR_ARGUMENT_FORMAL_GLOBAL,
     PM_ERR_ARGUMENT_FORMAL_IVAR,
     PM_ERR_ARGUMENT_FORWARDING_UNBOUND,
-    PM_ERR_ARGUMENT_IN,
     PM_ERR_ARGUMENT_NO_FORWARDING_AMPERSAND,
     PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
     PM_ERR_ARGUMENT_NO_FORWARDING_STAR,
@@ -128,6 +127,7 @@ typedef enum {
     PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT,
     PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH,
     PM_ERR_EXPECT_EXPRESSION_AFTER_STAR,
+    PM_ERR_EXPECT_FOR_DELIMITER,
     PM_ERR_EXPECT_IDENT_REQ_PARAMETER,
     PM_ERR_EXPECT_IN_DELIMITER,
     PM_ERR_EXPECT_LPAREN_REQ_PARAMETER,
@@ -136,6 +136,7 @@ typedef enum {
     PM_ERR_EXPECT_RPAREN,
     PM_ERR_EXPECT_RPAREN_AFTER_MULTI,
     PM_ERR_EXPECT_RPAREN_REQ_PARAMETER,
+    PM_ERR_EXPECT_SINGLETON_CLASS_DELIMITER,
     PM_ERR_EXPECT_STRING_CONTENT,
     PM_ERR_EXPECT_WHEN_DELIMITER,
     PM_ERR_EXPRESSION_BARE_HASH,
@@ -214,7 +215,9 @@ typedef enum {
     PM_ERR_MODULE_TERM,
     PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
     PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST,
+    PM_ERR_NESTING_TOO_DEEP,
     PM_ERR_NO_LOCAL_VARIABLE,
+    PM_ERR_NON_ASSOCIATIVE_OPERATOR,
     PM_ERR_NOT_EXPRESSION,
     PM_ERR_NUMBER_LITERAL_UNDERSCORE,
     PM_ERR_NUMBERED_PARAMETER_INNER_BLOCK,
@@ -300,6 +303,7 @@ typedef enum {
     PM_ERR_UNEXPECTED_BLOCK_ARGUMENT,
     PM_ERR_UNEXPECTED_INDEX_BLOCK,
     PM_ERR_UNEXPECTED_INDEX_KEYWORDS,
+    PM_ERR_UNEXPECTED_LABEL,
     PM_ERR_UNEXPECTED_MULTI_WRITE,
     PM_ERR_UNEXPECTED_RANGE_OPERATOR,
     PM_ERR_UNEXPECTED_SAFE_NAVIGATION,

data/include/prism/options.h

@@ -7,6 +7,7 @@
 #define PRISM_OPTIONS_H
 
 #include "prism/defines.h"
+#include "prism/util/pm_char.h"
 #include "prism/util/pm_string.h"
 
 #include <stdbool.h>
@@ -139,6 +140,23 @@ typedef struct pm_options {
      * but ignore any encoding magic comments at the top of the file.
      */
     bool encoding_locked;
+
+    /**
+     * When the file being parsed is the main script, the shebang will be
+     * considered for command-line flags (or for implicit -x). The caller needs
+     * to pass this information to the parser so that it can behave correctly.
+     */
+    bool main_script;
+
+    /**
+     * When the file being parsed is considered a "partial" script, jumps will
+     * not be marked as errors if they are not contained within loops/blocks.
+     * This is used in the case that you're parsing a script that you know will
+     * be embedded inside another script later, but you do not have that context
+     * yet. For example, when parsing an ERB template that will be evaluated
+     * inside another script.
+     */
+    bool partial_script;
 } pm_options_t;
 
 /**
@@ -248,6 +266,22 @@ PRISM_EXPORTED_FUNCTION void pm_options_command_line_set(pm_options_t *options,
  */
 PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length);
 
+/**
+ * Set the main script option on the given options struct.
+ *
+ * @param options The options struct to set the main script value on.
+ * @param main_script The main script value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, bool main_script);
+
+/**
+ * Set the partial script option on the given options struct.
+ *
+ * @param options The options struct to set the partial script value on.
+ * @param partial_script The partial script value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script);
+
 /**
  * Allocate and zero out the scopes array on the given options struct.
  *
@@ -315,6 +349,9 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | `1`     | -l command line option     |
  * | `1`     | -a command line option     |
  * | `1`     | the version                |
+ * | `1`     | encoding locked            |
+ * | `1`     | main script                |
+ * | `1`     | partial script             |
  * | `4`     | the number of scopes       |
  * | ...     | the scopes                 |
  *
@@ -347,8 +384,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * * The encoding can have a length of 0, in which case we'll use the default
  *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
  *   encoding that can be passed to `Encoding.find` in Ruby.
- * * The frozen string literal and suppress warnings fields are booleans, so
- *   their values should be either 0 or 1.
+ * * The frozen string literal, encoding locked, main script, and partial script
+ *   fields are booleans, so their values should be either 0 or 1.
  * * The number of scopes can be 0.
  *
  * @param options The options struct to deserialize into.

data/include/prism/parser.h

@@ -861,6 +861,13 @@ struct pm_parser {
      */
     bool parsing_eval;
 
+    /**
+     * Whether or not we are parsing a "partial" script, which is a script that
+     * will be evaluated in the context of another script, so we should not
+     * check jumps (next/break/etc.) for validity.
+     */
+    bool partial_script;
+
     /** Whether or not we're at the beginning of a command. */
     bool command_start;
 

data/include/prism/util/pm_string.h

@@ -22,6 +22,9 @@
 #include <fcntl.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
+#elif defined(PRISM_HAS_FILESYSTEM)
+#include <fcntl.h>
+#include <sys/stat.h>
 #endif
 
 /**
@@ -93,6 +96,26 @@ void pm_string_owned_init(pm_string_t *string, uint8_t *source, size_t length);
  */
 void pm_string_constant_init(pm_string_t *string, const char *source, size_t length);
 
+/**
+ * Represents the result of calling pm_string_mapped_init or
+ * pm_string_file_init. We need this additional information because there is
+ * not a platform-agnostic way to indicate that the file that was attempted to
+ * be opened was a directory.
+ */
+typedef enum {
+    /** Indicates that the string was successfully initialized. */
+    PM_STRING_INIT_SUCCESS = 0,
+    /**
+     * Indicates a generic error from a string_*_init function, where the type
+     * of error should be read from `errno` or `GetLastError()`.
+     */
+    PM_STRING_INIT_ERROR_GENERIC = 1,
+    /**
+     * Indicates that the file that was attempted to be opened was a directory.
+     */
+    PM_STRING_INIT_ERROR_DIRECTORY = 2
+} pm_string_init_result_t;
+
 /**
  * Read the file indicated by the filepath parameter into source and load its
  * contents and size into the given `pm_string_t`. The given `pm_string_t`
@@ -106,9 +129,9 @@ void pm_string_constant_init(pm_string_t *string, const char *source, size_t len
  *
  * @param string The string to initialize.
  * @param filepath The filepath to read.
- * @return Whether or not the file was successfully mapped.
+ * @return The success of the read, indicated by the value of the enum.
  */
-PRISM_EXPORTED_FUNCTION bool pm_string_mapped_init(pm_string_t *string, const char *filepath);
+PRISM_EXPORTED_FUNCTION pm_string_init_result_t pm_string_mapped_init(pm_string_t *string, const char *filepath);
 
 /**
  * Read the file indicated by the filepath parameter into source and load its
@@ -117,9 +140,9 @@ PRISM_EXPORTED_FUNCTION bool pm_string_mapped_init(pm_string_t *string, const ch
  *
  * @param string The string to initialize.
  * @param filepath The filepath to read.
- * @return Whether or not the file was successfully read.
+ * @return The success of the read, indicated by the value of the enum.
  */
-PRISM_EXPORTED_FUNCTION bool pm_string_file_init(pm_string_t *string, const char *filepath);
+PRISM_EXPORTED_FUNCTION pm_string_init_result_t pm_string_file_init(pm_string_t *string, const char *filepath);
 
 /**
  * Ensure the string is owned. If it is not, then reinitialize it as owned and

data/include/prism/version.h

@@ -14,7 +14,7 @@
 /**
  * The minor version of the Prism library as an int.
  */
-#define PRISM_VERSION_MINOR 0
+#define PRISM_VERSION_MINOR 1
 
 /**
  * The patch version of the Prism library as an int.
@@ -24,6 +24,6 @@
 /**
  * The version of the Prism library as a constant string.
  */
-#define PRISM_VERSION "1.0.0"
+#define PRISM_VERSION "1.1.0"
 
 #endif

data/lib/prism/dot_visitor.rb

@@ -4592,9 +4592,11 @@ module Prism
     # comma-separated list.
     def arguments_node_flags_inspect(node)
       flags = [] #: Array[String]
+      flags << "contains_forwarding" if node.contains_forwarding?
       flags << "contains_keywords" if node.contains_keywords?
       flags << "contains_keyword_splat" if node.contains_keyword_splat?
       flags << "contains_splat" if node.contains_splat?
+      flags << "contains_multiple_splats" if node.contains_multiple_splats?
       flags.join(", ")
     end
 

data/lib/prism/dsl.rb

@@ -184,7 +184,7 @@ module Prism
     end
 
     # Create a new CapturePatternNode node.
-    def capture_pattern_node(source: default_source, node_id: 0, location: default_location, flags: 0, value: default_node(source, location), target: default_node(source, location), operator_loc: location)
+    def capture_pattern_node(source: default_source, node_id: 0, location: default_location, flags: 0, value: default_node(source, location), target: local_variable_target_node(source: source), operator_loc: location)
       CapturePatternNode.new(source, node_id, location, flags, value, target, operator_loc)
     end
 
@@ -199,7 +199,7 @@ module Prism
     end
 
     # Create a new ClassNode node.
-    def class_node(source: default_source, node_id: 0, location: default_location, flags: 0, locals: [], class_keyword_loc: location, constant_path: default_node(source, location), inheritance_operator_loc: nil, superclass: nil, body: nil, end_keyword_loc: location, name: :"")
+    def class_node(source: default_source, node_id: 0, location: default_location, flags: 0, locals: [], class_keyword_loc: location, constant_path: constant_read_node(source: source), inheritance_operator_loc: nil, superclass: nil, body: nil, end_keyword_loc: location, name: :"")
       ClassNode.new(source, node_id, location, flags, locals, class_keyword_loc, constant_path, inheritance_operator_loc, superclass, body, end_keyword_loc, name)
     end
 
@@ -314,7 +314,7 @@ module Prism
     end
 
     # Create a new EmbeddedVariableNode node.
-    def embedded_variable_node(source: default_source, node_id: 0, location: default_location, flags: 0, operator_loc: location, variable: default_node(source, location))
+    def embedded_variable_node(source: default_source, node_id: 0, location: default_location, flags: 0, operator_loc: location, variable: instance_variable_read_node(source: source))
       EmbeddedVariableNode.new(source, node_id, location, flags, operator_loc, variable)
     end
 
@@ -329,7 +329,7 @@ module Prism
     end
 
     # Create a new FindPatternNode node.
-    def find_pattern_node(source: default_source, node_id: 0, location: default_location, flags: 0, constant: nil, left: default_node(source, location), requireds: [], right: default_node(source, location), opening_loc: nil, closing_loc: nil)
+    def find_pattern_node(source: default_source, node_id: 0, location: default_location, flags: 0, constant: nil, left: splat_node(source: source), requireds: [], right: splat_node(source: source), opening_loc: nil, closing_loc: nil)
       FindPatternNode.new(source, node_id, location, flags, constant, left, requireds, right, opening_loc, closing_loc)
     end
 
@@ -344,7 +344,7 @@ module Prism
     end
 
     # Create a new ForNode node.
-    def for_node(source: default_source, node_id: 0, location: default_location, flags: 0, index: default_node(source, location), collection: default_node(source, location), statements: nil, for_keyword_loc: location, in_keyword_loc: location, do_keyword_loc: nil, end_keyword_loc: location)
+    def for_node(source: default_source, node_id: 0, location: default_location, flags: 0, index: local_variable_target_node(source: source), collection: default_node(source, location), statements: nil, for_keyword_loc: location, in_keyword_loc: location, do_keyword_loc: nil, end_keyword_loc: location)
       ForNode.new(source, node_id, location, flags, index, collection, statements, for_keyword_loc, in_keyword_loc, do_keyword_loc, end_keyword_loc)
     end
 
@@ -414,7 +414,7 @@ module Prism
     end
 
     # Create a new ImplicitNode node.
-    def implicit_node(source: default_source, node_id: 0, location: default_location, flags: 0, value: default_node(source, location))
+    def implicit_node(source: default_source, node_id: 0, location: default_location, flags: 0, value: local_variable_read_node(source: source))
       ImplicitNode.new(source, node_id, location, flags, value)
     end
 
@@ -589,7 +589,7 @@ module Prism
     end
 
     # Create a new ModuleNode node.
-    def module_node(source: default_source, node_id: 0, location: default_location, flags: 0, locals: [], module_keyword_loc: location, constant_path: default_node(source, location), body: nil, end_keyword_loc: location, name: :"")
+    def module_node(source: default_source, node_id: 0, location: default_location, flags: 0, locals: [], module_keyword_loc: location, constant_path: constant_read_node(source: source), body: nil, end_keyword_loc: location, name: :"")
       ModuleNode.new(source, node_id, location, flags, locals, module_keyword_loc, constant_path, body, end_keyword_loc, name)
     end
 
@@ -659,7 +659,7 @@ module Prism
     end
 
     # Create a new PinnedVariableNode node.
-    def pinned_variable_node(source: default_source, node_id: 0, location: default_location, flags: 0, variable: default_node(source, location), operator_loc: location)
+    def pinned_variable_node(source: default_source, node_id: 0, location: default_location, flags: 0, variable: local_variable_read_node(source: source), operator_loc: location)
       PinnedVariableNode.new(source, node_id, location, flags, variable, operator_loc)
     end
 
@@ -831,9 +831,11 @@ module Prism
     # Retrieve the value of one of the ArgumentsNodeFlags flags.
     def arguments_node_flag(name)
       case name
+      when :contains_forwarding then ArgumentsNodeFlags::CONTAINS_FORWARDING
       when :contains_keywords then ArgumentsNodeFlags::CONTAINS_KEYWORDS
       when :contains_keyword_splat then ArgumentsNodeFlags::CONTAINS_KEYWORD_SPLAT
       when :contains_splat then ArgumentsNodeFlags::CONTAINS_SPLAT
+      when :contains_multiple_splats then ArgumentsNodeFlags::CONTAINS_MULTIPLE_SPLATS
       else Kernel.raise ArgumentError, "invalid ArgumentsNodeFlags flag: #{name.inspect}"
       end
     end

data/lib/prism/ffi.rb

@@ -72,6 +72,7 @@ module Prism
     end
 
     callback :pm_parse_stream_fgets_t, [:pointer, :int, :pointer], :pointer
+    enum :pm_string_init_result_t, %i[PM_STRING_INIT_SUCCESS PM_STRING_INIT_ERROR_GENERIC PM_STRING_INIT_ERROR_DIRECTORY]
 
     load_exported_functions_from(
       "prism.h",
@@ -176,13 +177,26 @@ module Prism
       def self.with_file(filepath)
         raise TypeError unless filepath.is_a?(String)
 
+        # On Windows and Mac, it's expected that filepaths will be encoded in
+        # UTF-8. If they are not, we need to convert them to UTF-8 before
+        # passing them into pm_string_mapped_init.
+        if RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince|darwin/i) &&
+           (encoding = filepath.encoding) != Encoding::ASCII_8BIT && encoding != Encoding::UTF_8
+          filepath = filepath.encode(Encoding::UTF_8)
+        end
+
         FFI::MemoryPointer.new(SIZEOF) do |pm_string|
-          if LibRubyParser.pm_string_mapped_init(pm_string, filepath)
+          case (result = LibRubyParser.pm_string_mapped_init(pm_string, filepath))
+          when :PM_STRING_INIT_SUCCESS
             pointer = LibRubyParser.pm_string_source(pm_string)
             length = LibRubyParser.pm_string_length(pm_string)
             return yield new(pointer, length, false)
-          else
+          when :PM_STRING_INIT_ERROR_GENERIC
             raise SystemCallError.new(filepath, FFI.errno)
+          when :PM_STRING_INIT_ERROR_DIRECTORY
+            raise Errno::EISDIR.new(filepath)
+          else
+            raise "Unknown error initializing pm_string_t: #{result.inspect}"
           end
         ensure
           LibRubyParser.pm_string_free(pm_string)
@@ -397,6 +411,20 @@ module Prism
       end
     end
 
+    # Return the value that should be dumped for the version option.
+    def dump_options_version(version)
+      case version
+      when nil, "latest"
+        0
+      when /\A3\.3(\.\d+)?\z/
+        1
+      when /\A3\.4(\.\d+)?\z/
+        0
+      else
+        raise ArgumentError, "invalid version: #{version}"
+      end
+    end
+
     # Convert the given options into a serialized options string.
     def dump_options(options)
       template = +""
@@ -429,11 +457,17 @@ module Prism
       values << dump_options_command_line(options)
 
       template << "C"
-      values << { nil => 0, "3.3.0" => 1, "3.3.1" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version])
+      values << dump_options_version(options[:version])
 
       template << "C"
       values << (options[:encoding] == false ? 1 : 0)
 
+      template << "C"
+      values << (options.fetch(:main_script, false) ? 1 : 0)
+
+      template << "C"
+      values << (options.fetch(:partial_script, false) ? 1 : 0)
+
       template << "L"
       if (scopes = options[:scopes])
         values << scopes.length

data/lib/prism/inspect_visitor.rb

@@ -124,7 +124,7 @@ module Prism
     # Inspect a ArgumentsNode node.
     def visit_arguments_node(node)
       commands << [inspect_node("ArgumentsNode", node), indent]
-      flags = [("newline" if node.newline?), ("static_literal" if node.static_literal?), ("contains_keywords" if node.contains_keywords?), ("contains_keyword_splat" if node.contains_keyword_splat?), ("contains_splat" if node.contains_splat?)].compact
+      flags = [("newline" if node.newline?), ("static_literal" if node.static_literal?), ("contains_forwarding" if node.contains_forwarding?), ("contains_keywords" if node.contains_keywords?), ("contains_keyword_splat" if node.contains_keyword_splat?), ("contains_splat" if node.contains_splat?), ("contains_multiple_splats" if node.contains_multiple_splats?)].compact
       commands << ["├── flags: #{flags.empty? ? "∅" : flags.join(", ")}\n", indent]
       commands << ["└── arguments: (length: #{(arguments = node.arguments).length})\n", indent]
       if arguments.any?

data/lib/prism/lex_compat.rb

@@ -481,7 +481,7 @@ module Prism
                 embexpr_balance -= 1
               when :on_tstring_content
                 if embexpr_balance == 0
-                  while index < max_index && tokens[index].event == :on_tstring_content
+                  while index < max_index && tokens[index].event == :on_tstring_content && !token.value.match?(/\\\r?\n\z/)
                     token.value << tokens[index].value
                     index += 1
                   end