prism 0.24.0 → 0.26.0

data/include/prism/util/pm_integer.h

@@ -0,0 +1,119 @@
+/**
+ * @file pm_integer.h
+ *
+ * This module provides functions for working with arbitrary-sized integers.
+ */
+#ifndef PRISM_NUMBER_H
+#define PRISM_NUMBER_H
+
+#include "prism/defines.h"
+#include "prism/util/pm_buffer.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+/**
+ * A structure represents an arbitrary-sized integer.
+ */
+typedef struct {
+    /**
+     * Embedded value for small integer. This value is set to 0 if the value
+     * does not fit into uint32_t.
+     */
+    uint32_t value;
+
+    /**
+     * The number of allocated values. length is set to 0 if the integer fits
+     * into uint32_t.
+     */
+    size_t length;
+
+    /**
+     * List of 32-bit integers. Set to NULL if the integer fits into uint32_t.
+     */
+    uint32_t *values;
+
+    /**
+     * Whether or not the integer is negative. It is stored this way so that a
+     * zeroed pm_integer_t is always positive zero.
+     */
+    bool negative;
+} pm_integer_t;
+
+/**
+ * An enum controlling the base of an integer. It is expected that the base is
+ * already known before parsing the integer, even though it could be derived
+ * from the string itself.
+ */
+typedef enum {
+    /** The binary base, indicated by a 0b or 0B prefix. */
+    PM_INTEGER_BASE_BINARY,
+
+    /** The octal base, indicated by a 0, 0o, or 0O prefix. */
+    PM_INTEGER_BASE_OCTAL,
+
+    /** The decimal base, indicated by a 0d, 0D, or empty prefix. */
+    PM_INTEGER_BASE_DECIMAL,
+
+    /** The hexadecimal base, indicated by a 0x or 0X prefix. */
+    PM_INTEGER_BASE_HEXADECIMAL,
+
+    /**
+     * An unknown base, in which case pm_integer_parse will derive it based on
+     * the content of the string. This is less efficient and does more
+     * comparisons, so if callers know the base ahead of time, they should use
+     * that instead.
+     */
+    PM_INTEGER_BASE_UNKNOWN
+} pm_integer_base_t;
+
+/**
+ * Parse an integer from a string. This assumes that the format of the integer
+ * has already been validated, as internal validation checks are not performed
+ * here.
+ *
+ * @param integer The integer to parse into.
+ * @param base The base of the integer.
+ * @param start The start of the string.
+ * @param end The end of the string.
+ */
+PRISM_EXPORTED_FUNCTION void pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *start, const uint8_t *end);
+
+/**
+ * Return the memory size of the integer.
+ *
+ * @param integer The integer to get the memory size of.
+ * @return The size of the memory associated with the integer.
+ */
+size_t pm_integer_memsize(const pm_integer_t *integer);
+
+/**
+ * Compare two integers. This function returns -1 if the left integer is less
+ * than the right integer, 0 if they are equal, and 1 if the left integer is
+ * greater than the right integer.
+ *
+ * @param left The left integer to compare.
+ * @param right The right integer to compare.
+ * @return The result of the comparison.
+ */
+int pm_integer_compare(const pm_integer_t *left, const pm_integer_t *right);
+
+/**
+ * Convert an integer to a decimal string.
+ *
+ * @param buffer The buffer to append the string to.
+ * @param integer The integer to convert to a string.
+ */
+PRISM_EXPORTED_FUNCTION void pm_integer_string(pm_buffer_t *buffer, const pm_integer_t *integer);
+
+/**
+ * Free the internal memory of an integer. This memory will only be allocated if
+ * the integer exceeds the size of a single node in the linked list.
+ *
+ * @param integer The integer to free.
+ */
+PRISM_EXPORTED_FUNCTION void pm_integer_free(pm_integer_t *integer);
+
+#endif

data/include/prism/util/pm_list.h

@@ -33,7 +33,7 @@
  * } pm_int_node_t;
  *
  * pm_list_t list = { 0 };
- * pm_int_node_t *node = malloc(sizeof(pm_int_node_t));
+ * pm_int_node_t *node = xmalloc(sizeof(pm_int_node_t));
  * node->value = 5;
  *
  * pm_list_append(&list, &node->node);

data/include/prism/util/pm_newline_list.h

@@ -61,6 +61,14 @@ typedef struct {
  */
 bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity);
 
+/**
+ * Clear out the newlines that have been appended to the list.
+ *
+ * @param list The list to clear.
+ */
+void
+pm_newline_list_clear(pm_newline_list_t *list);
+
 /**
  * Append a new offset to the newline list. Returns true if the reallocation of
  * the offsets succeeds (if one was necessary), otherwise returns false.

data/include/prism/util/pm_string.h

@@ -17,11 +17,10 @@
 // The following headers are necessary to read files using demand paging.
 #ifdef _WIN32
 #include <windows.h>
-#else
+#elif defined(_POSIX_MAPPED_FILES)
 #include <fcntl.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
-#include <unistd.h>
 #endif
 
 /**
@@ -45,8 +44,10 @@ typedef struct {
         /** This string owns its memory, and should be freed using `pm_string_free`. */
         PM_STRING_OWNED,
 
+#ifdef PRISM_HAS_MMAP
         /** This string is a memory-mapped file, and should be freed using `pm_string_free`. */
         PM_STRING_MAPPED
+#endif
     } type;
 } pm_string_t;
 
@@ -108,6 +109,17 @@ void pm_string_constant_init(pm_string_t *string, const char *source, size_t len
  */
 PRISM_EXPORTED_FUNCTION bool pm_string_mapped_init(pm_string_t *string, const char *filepath);
 
+/**
+ * Read the file indicated by the filepath parameter into source and load its
+ * contents and size into the given `pm_string_t`. The given `pm_string_t`
+ * should be freed using `pm_string_free` when it is no longer used.
+ *
+ * @param string The string to initialize.
+ * @param filepath The filepath to read.
+ * @return Whether or not the file was successfully read.
+ */
+PRISM_EXPORTED_FUNCTION bool pm_string_file_init(pm_string_t *string, const char *filepath);
+
 /**
  * Returns the memory size associated with the string.
  *
@@ -124,6 +136,18 @@ size_t pm_string_memsize(const pm_string_t *string);
  */
 void pm_string_ensure_owned(pm_string_t *string);
 
+/**
+ * Compare the underlying lengths and bytes of two strings. Returns 0 if the
+ * strings are equal, a negative number if the left string is less than the
+ * right string, and a positive number if the left string is greater than the
+ * right string.
+ *
+ * @param left The left string to compare.
+ * @param right The right string to compare.
+ * @return The comparison result.
+ */
+int pm_string_compare(const pm_string_t *left, const pm_string_t *right);
+
 /**
  * Returns the length associated with the string.
  *

data/include/prism/version.h

@@ -14,7 +14,7 @@
 /**
  * The minor version of the Prism library as an int.
  */
-#define PRISM_VERSION_MINOR 24
+#define PRISM_VERSION_MINOR 26
 
 /**
  * The patch version of the Prism library as an int.
@@ -24,6 +24,6 @@
 /**
  * The version of the Prism library as a constant string.
  */
-#define PRISM_VERSION "0.24.0"
+#define PRISM_VERSION "0.26.0"
 
 #endif

data/include/prism.h

@@ -9,6 +9,7 @@
 #include "prism/defines.h"
 #include "prism/util/pm_buffer.h"
 #include "prism/util/pm_char.h"
+#include "prism/util/pm_integer.h"
 #include "prism/util/pm_memchr.h"
 #include "prism/util/pm_strncasecmp.h"
 #include "prism/util/pm_strpbrk.h"
@@ -20,10 +21,13 @@
 #include "prism/parser.h"
 #include "prism/prettyprint.h"
 #include "prism/regexp.h"
+#include "prism/static_literals.h"
 #include "prism/version.h"
 
 #include <assert.h>
 #include <errno.h>
+#include <locale.h>
+#include <math.h>
 #include <stdarg.h>
 #include <stdbool.h>
 #include <stdint.h>
@@ -76,6 +80,41 @@ PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser);
  */
 PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser);
 
+/**
+ * This function is used in pm_parse_stream to retrieve a line of input from a
+ * stream. It closely mirrors that of fgets so that fgets can be used as the
+ * default implementation.
+ */
+typedef char * (pm_parse_stream_fgets_t)(char *string, int size, void *stream);
+
+/**
+ * Parse a stream of Ruby source and return the tree.
+ *
+ * @param parser The parser to use.
+ * @param buffer The buffer to use.
+ * @param stream The stream to parse.
+ * @param fgets The function to use to read from the stream.
+ * @param options The optional options to use when parsing.
+ * @return The AST representing the source.
+ */
+PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse_stream(pm_parser_t *parser, pm_buffer_t *buffer, void *stream, pm_parse_stream_fgets_t *fgets, const pm_options_t *options);
+
+// We optionally support serializing to a binary string. For systems that don't
+// want or need this functionality, it can be turned off with the
+// PRISM_EXCLUDE_SERIALIZATION define.
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+
+/**
+ * Parse and serialize the AST represented by the source that is read out of the
+ * given stream into to the given buffer.
+ *
+ * @param buffer The buffer to serialize to.
+ * @param stream The stream to parse.
+ * @param fgets The function to use to read from the stream.
+ * @param data The optional data to pass to the parser.
+ */
+PRISM_EXPORTED_FUNCTION void pm_serialize_parse_stream(pm_buffer_t *buffer, void *stream, pm_parse_stream_fgets_t *fgets, const char *data);
+
 /**
  * Serialize the given list of comments to the given buffer.
  *
@@ -152,6 +191,8 @@ PRISM_EXPORTED_FUNCTION void pm_serialize_lex(pm_buffer_t *buffer, const uint8_t
  */
 PRISM_EXPORTED_FUNCTION void pm_serialize_parse_lex(pm_buffer_t *buffer, const uint8_t *source, size_t size, const char *data);
 
+#endif
+
 /**
  * Parse the source and return true if it parses without errors or warnings.
  *
@@ -182,10 +223,27 @@ const char * pm_token_type_human(pm_token_type_t token_type);
  * Format the errors on the parser into the given buffer.
  *
  * @param parser The parser to format the errors for.
+ * @param error_list The list of errors to format.
  * @param buffer The buffer to format the errors into.
  * @param colorize Whether or not to colorize the errors with ANSI escape sequences.
+ * @param inline_messages Whether or not to inline the messages with the source.
+ */
+PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages);
+
+// We optionally support dumping to JSON. For systems that don't want or need
+// this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define.
+#ifndef PRISM_EXCLUDE_JSON
+
+/**
+ * Dump JSON to the given buffer.
+ *
+ * @param buffer The buffer to serialize to.
+ * @param parser The parser that parsed the node.
+ * @param node The node to serialize.
  */
-PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize);
+PRISM_EXPORTED_FUNCTION void pm_dump_json(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node);
+
+#endif
 
 /**
  * @mainpage

data/lib/prism/compiler.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 =begin
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/compiler.rb.erb
@@ -23,7 +24,7 @@ module Prism
   #     Prism.parse("1 + 2").value.accept(SExpressions.new)
   #     # => [:program, [[[:call, [[:integer], [:arguments, [[:integer]]]]]]]]
   #
-  class Compiler
+  class Compiler < Visitor
     # Visit an individual node.
     def visit(node)
       node&.accept(self)
@@ -300,6 +301,9 @@ module Prism
     # Compile a InterpolatedXStringNode node
     alias visit_interpolated_x_string_node visit_child_nodes
 
+    # Compile a ItParametersNode node
+    alias visit_it_parameters_node visit_child_nodes
+
     # Compile a KeywordHashNode node
     alias visit_keyword_hash_node visit_child_nodes
 
@@ -432,6 +436,9 @@ module Prism
     # Compile a SelfNode node
     alias visit_self_node visit_child_nodes
 
+    # Compile a ShareableConstantNode node
+    alias visit_shareable_constant_node visit_child_nodes
+
     # Compile a SingletonClassNode node
     alias visit_singleton_class_node visit_child_nodes
 

data/lib/prism/debug.rb

@@ -55,7 +55,7 @@ module Prism
       verbose, $VERBOSE = $VERBOSE, nil
 
       begin
-        locals = []
+        locals = [] #: Array[Array[Symbol | Integer]]
         stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)]
 
         while (iseq = stack.pop)
@@ -96,8 +96,8 @@ module Prism
     # For the given source, parses with prism and returns a list of all of the
     # sets of local variables that were encountered.
     def self.prism_locals(source)
-      locals = []
-      stack = [Prism.parse(source).value]
+      locals = [] #: Array[Array[Symbol | Integer]]
+      stack = [Prism.parse(source).value] #: Array[Prism::node]
 
       while (node = stack.pop)
         case node
@@ -202,5 +202,48 @@ module Prism
     def self.newlines(source)
       Prism.parse(source).source.offsets
     end
+
+    # A wrapping around prism's internal encoding data structures. This is used
+    # for reflection and debugging purposes.
+    class Encoding
+      # The name of the encoding, that can be passed to Encoding.find.
+      attr_reader :name
+
+      # Initialize a new encoding with the given name and whether or not it is
+      # a multibyte encoding.
+      def initialize(name, multibyte)
+        @name = name
+        @multibyte = multibyte
+      end
+
+      # Whether or not the encoding is a multibyte encoding.
+      def multibyte?
+        @multibyte
+      end
+
+      # Returns the number of bytes of the first character in the source string,
+      # if it is valid for the encoding. Otherwise, returns 0.
+      def width(source)
+        Encoding._width(name, source)
+      end
+
+      # Returns true if the first character in the source string is a valid
+      # alphanumeric character for the encoding.
+      def alnum?(source)
+        Encoding._alnum?(name, source)
+      end
+
+      # Returns true if the first character in the source string is a valid
+      # alphabetic character for the encoding.
+      def alpha?(source)
+        Encoding._alpha?(name, source)
+      end
+
+      # Returns true if the first character in the source string is a valid
+      # uppercase character for the encoding.
+      def upper?(source)
+        Encoding._upper?(name, source)
+      end
+    end
   end
 end

data/lib/prism/desugar_compiler.rb

@@ -73,6 +73,8 @@ module Prism
 
     # Desugar `x += y` to `x = x + y`
     def compile
+      operator_loc = node.operator_loc.chop
+
       write_class.new(
         source,
         *arguments,
@@ -82,8 +84,8 @@ module Prism
           0,
           read_class.new(source, *arguments, node.name_loc),
           nil,
-          node.operator_loc.slice.chomp("="),
-          node.operator_loc.copy(length: node.operator_loc.length - 1),
+          operator_loc.slice.to_sym,
+          operator_loc,
           nil,
           ArgumentsNode.new(source, 0, [node.value], node.value.location),
           nil,

data/lib/prism/dispatcher.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 =begin
 This file is generated by the templates/template.rb script and should not be
 modified manually. See templates/lib/prism/dispatcher.rb.erb
@@ -761,6 +762,14 @@ module Prism
       listeners[:on_interpolated_x_string_node_leave]&.each { |listener| listener.on_interpolated_x_string_node_leave(node) }
     end
 
+    # Dispatch enter and leave events for ItParametersNode nodes and continue
+    # walking the tree.
+    def visit_it_parameters_node(node)
+      listeners[:on_it_parameters_node_enter]&.each { |listener| listener.on_it_parameters_node_enter(node) }
+      super
+      listeners[:on_it_parameters_node_leave]&.each { |listener| listener.on_it_parameters_node_leave(node) }
+    end
+
     # Dispatch enter and leave events for KeywordHashNode nodes and continue
     # walking the tree.
     def visit_keyword_hash_node(node)
@@ -1113,6 +1122,14 @@ module Prism
       listeners[:on_self_node_leave]&.each { |listener| listener.on_self_node_leave(node) }
     end
 
+    # Dispatch enter and leave events for ShareableConstantNode nodes and continue
+    # walking the tree.
+    def visit_shareable_constant_node(node)
+      listeners[:on_shareable_constant_node_enter]&.each { |listener| listener.on_shareable_constant_node_enter(node) }
+      super
+      listeners[:on_shareable_constant_node_leave]&.each { |listener| listener.on_shareable_constant_node_leave(node) }
+    end
+
     # Dispatch enter and leave events for SingletonClassNode nodes and continue
     # walking the tree.
     def visit_singleton_class_node(node)
@@ -1778,6 +1795,12 @@ module Prism
         listeners[:on_interpolated_x_string_node_leave]&.each { |listener| listener.on_interpolated_x_string_node_leave(node) }
       end
 
+      # Dispatch enter and leave events for ItParametersNode nodes.
+      def visit_it_parameters_node(node)
+        listeners[:on_it_parameters_node_enter]&.each { |listener| listener.on_it_parameters_node_enter(node) }
+        listeners[:on_it_parameters_node_leave]&.each { |listener| listener.on_it_parameters_node_leave(node) }
+      end
+
       # Dispatch enter and leave events for KeywordHashNode nodes.
       def visit_keyword_hash_node(node)
         listeners[:on_keyword_hash_node_enter]&.each { |listener| listener.on_keyword_hash_node_enter(node) }
@@ -2042,6 +2065,12 @@ module Prism
         listeners[:on_self_node_leave]&.each { |listener| listener.on_self_node_leave(node) }
       end
 
+      # Dispatch enter and leave events for ShareableConstantNode nodes.
+      def visit_shareable_constant_node(node)
+        listeners[:on_shareable_constant_node_enter]&.each { |listener| listener.on_shareable_constant_node_enter(node) }
+        listeners[:on_shareable_constant_node_leave]&.each { |listener| listener.on_shareable_constant_node_leave(node) }
+      end
+
       # Dispatch enter and leave events for SingletonClassNode nodes.
       def visit_singleton_class_node(node)
         listeners[:on_singleton_class_node_enter]&.each { |listener| listener.on_singleton_class_node_enter(node) }