prism 1.9.0 → 2.0.0.rc2

data/sig/generated/prism.rbs

@@ -0,0 +1,84 @@
+# Generated from lib/prism.rb with RBS::Inline
+
+# The Prism Ruby parser.
+#
+# "Parsing Ruby is suddenly manageable!"
+#   - You, hopefully
+module Prism
+  # Raised when requested to parse as the currently running Ruby version but Prism has no support for it.
+  class CurrentVersionError < ArgumentError
+    # Initialize a new exception for the given ruby version string.
+    #
+    # : (String version) -> void
+    def initialize: (String version) -> void
+  end
+
+  # :call-seq:
+  #   lex_compat(source, **options) -> LexCompat::Result
+  #
+  # Returns a parse result whose value is an array of tokens that closely
+  # resembles the return value of Ripper.lex.
+  #
+  # For supported options, see Prism.parse.
+  #
+  # : (String source, **untyped options) -> LexCompat::Result
+  def self.lex_compat: (String source, **untyped options) -> LexCompat::Result
+
+  # :call-seq:
+  #   load(source, serialized, freeze) -> ParseResult
+  #
+  # Load the serialized AST using the source as a reference into a tree.
+  #
+  # : (String source, String serialized, ?bool freeze) -> ParseResult
+  def self.load: (String source, String serialized, ?bool freeze) -> ParseResult
+
+  # Given a Method, UnboundMethod, Proc, or Thread::Backtrace::Location,
+  # returns the Prism node representing it. On CRuby, this uses node_id for
+  # an exact match. On other implementations, it falls back to best-effort
+  # matching by source location line number.
+  #
+  # : (Method | UnboundMethod | Proc | Thread::Backtrace::Location callable) -> Node?
+  def self.find: (Method | UnboundMethod | Proc | Thread::Backtrace::Location callable) -> Node?
+
+  VERSION: String
+
+  BACKEND: :CEXT | :FFI
+
+  interface _Stream
+    def gets: (?Integer integer) -> (String | nil)
+  end
+
+  def self.parse: (String source, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+
+  def self.profile: (String source, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void
+
+  def self.lex: (String source, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult
+
+  def self.parse_lex: (String source, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult
+
+  def self.dump: (String source, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String
+
+  def self.parse_comments: (String source, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment]
+
+  def self.parse_success?: (String source, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+
+  def self.parse_failure?: (String source, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+
+  def self.parse_stream: (_Stream stream, ?filepath: String, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+
+  def self.parse_file: (path filepath, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+
+  def self.profile_file: (path filepath, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void
+
+  def self.lex_file: (path filepath, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult
+
+  def self.parse_lex_file: (path filepath, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult
+
+  def self.dump_file: (path filepath, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String
+
+  def self.parse_file_comments: (path filepath, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment]
+
+  def self.parse_file_success?: (path filepath, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+
+  def self.parse_file_failure?: (path filepath, ?attach_comments: bool, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+end

data/sig/manifest.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - name: pp

data/src/arena.c

@@ -0,0 +1,117 @@
+#include "prism/internal/arena.h"
+
+#include "prism/internal/allocator.h"
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/**
+ * Compute the block allocation size using offsetof so it is correct regardless
+ * of PM_FLEX_ARRAY_LENGTH.
+ */
+#define PM_ARENA_BLOCK_SIZE(data_size) (offsetof(pm_arena_block_t, data) + (data_size))
+
+/** Initial block data size: 8 KB. */
+#define PM_ARENA_INITIAL_SIZE 8192
+
+/** Double the block size every this many blocks. */
+#define PM_ARENA_GROWTH_INTERVAL 8
+
+/** Maximum block data size: 1 MB. */
+#define PM_ARENA_MAX_SIZE (1024 * 1024)
+
+/**
+ * Compute the data size for the next block.
+ */
+static size_t
+pm_arena_next_block_size(const pm_arena_t *arena, size_t min_size) {
+    size_t size = PM_ARENA_INITIAL_SIZE;
+
+    for (size_t exp = PM_ARENA_GROWTH_INTERVAL; exp <= arena->block_count; exp += PM_ARENA_GROWTH_INTERVAL) {
+        if (size < PM_ARENA_MAX_SIZE) size *= 2;
+    }
+
+    return size > min_size ? size : min_size;
+}
+
+/**
+ * Allocate a new block with the given data capacity and initial usage, link it
+ * into the arena, and return it. Aborts on allocation failure.
+ */
+static pm_arena_block_t *
+pm_arena_block_new(pm_arena_t *arena, size_t data_size, size_t initial_used) {
+    assert(initial_used <= data_size);
+    pm_arena_block_t *block = (pm_arena_block_t *) xmalloc(PM_ARENA_BLOCK_SIZE(data_size));
+
+    if (block == NULL) {
+        fprintf(stderr, "prism: out of memory; aborting\n");
+        abort();
+    }
+
+    block->capacity = data_size;
+    block->used = initial_used;
+    block->prev = arena->current;
+    arena->current = block;
+    arena->block_count++;
+
+    return block;
+}
+
+/**
+ * Ensure the arena has at least `capacity` bytes available in its current
+ * block, allocating a new block if necessary. This allows callers to
+ * pre-size the arena to avoid repeated small block allocations.
+ */
+void
+pm_arena_reserve(pm_arena_t *arena, size_t capacity) {
+    if (capacity <= PM_ARENA_INITIAL_SIZE) return;
+    if (arena->current != NULL && (arena->current->capacity - arena->current->used) >= capacity) return;
+    pm_arena_block_new(arena, capacity, 0);
+}
+
+/**
+ * Slow path for pm_arena_alloc: allocate a new block and return a pointer to
+ * the first `size` bytes. Called when the current block has insufficient space.
+ */
+void *
+pm_arena_alloc_slow(pm_arena_t *arena, size_t size) {
+    size_t block_data_size = pm_arena_next_block_size(arena, size);
+    pm_arena_block_t *block = pm_arena_block_new(arena, block_data_size, size);
+    return block->data;
+}
+
+/**
+ * Returns a newly allocated and initialized arena.
+ */
+pm_arena_t *
+pm_arena_new(void) {
+    pm_arena_t *arena = (pm_arena_t *) xcalloc(1, sizeof(pm_arena_t));
+    if (arena == NULL) abort();
+    return arena;
+}
+
+/**
+ * Free all blocks in the arena.
+ */
+void
+pm_arena_cleanup(pm_arena_t *arena) {
+    pm_arena_block_t *block = arena->current;
+
+    while (block != NULL) {
+        pm_arena_block_t *prev = block->prev;
+        xfree_sized(block, PM_ARENA_BLOCK_SIZE(block->capacity));
+        block = prev;
+    }
+
+    *arena = (pm_arena_t) { 0 };
+}
+
+/**
+ * Frees both the held memory and the arena itself.
+ */
+void
+pm_arena_free(pm_arena_t *arena) {
+    pm_arena_cleanup(arena);
+    xfree_sized(arena, sizeof(pm_arena_t));
+}

data/src/{util/pm_buffer.c → buffer.c}

@@ -1,31 +1,38 @@
-#include "prism/util/pm_buffer.h"
+#include "prism/internal/buffer.h"
 
-/**
- * Return the size of the pm_buffer_t struct.
- */
-size_t
-pm_buffer_sizeof(void) {
-    return sizeof(pm_buffer_t);
-}
+#include "prism/compiler/inline.h"
+
+#include "prism/internal/char.h"
+#include "prism/internal/allocator.h"
+
+#include <assert.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 
 /**
  * Initialize a pm_buffer_t with the given capacity.
  */
-bool
-pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity) {
+void
+pm_buffer_init(pm_buffer_t *buffer, size_t capacity) {
     buffer->length = 0;
     buffer->capacity = capacity;
 
     buffer->value = (char *) xmalloc(capacity);
-    return buffer->value != NULL;
+    if (buffer->value == NULL) abort();
 }
 
 /**
- * Initialize a pm_buffer_t with its default values.
+ * Allocate and initialize a new buffer.
  */
-bool
-pm_buffer_init(pm_buffer_t *buffer) {
-    return pm_buffer_init_capacity(buffer, 1024);
+pm_buffer_t *
+pm_buffer_new(void) {
+    pm_buffer_t *buffer = (pm_buffer_t *) xmalloc(sizeof(pm_buffer_t));
+    if (buffer == NULL) abort();
+
+    pm_buffer_init(buffer, 1024);
+    return buffer;
 }
 
 /**
@@ -45,11 +52,18 @@ pm_buffer_length(const pm_buffer_t *buffer) {
 }
 
 /**
- * Append the given amount of space to the buffer.
+ * Append the given amount of space to the buffer. Returns false when there is
+ * nothing to write: either the allocation failed or the length is zero. A
+ * zero-initialized buffer has a NULL value until its first allocation, and
+ * passing NULL to memcpy/memset/memmove is undefined behavior even for zero
+ * bytes, so callers must not write in the zero case.
  */
-static inline bool
+static PRISM_INLINE bool
 pm_buffer_append_length(pm_buffer_t *buffer, size_t length) {
+    if (length == 0) return false;
+
     size_t next_length = buffer->length + length;
+    const size_t original_capacity = buffer->capacity;
 
     if (next_length > buffer->capacity) {
         if (buffer->capacity == 0) {
@@ -60,7 +74,7 @@ pm_buffer_append_length(pm_buffer_t *buffer, size_t length) {
             buffer->capacity *= 2;
         }
 
-        buffer->value = xrealloc(buffer->value, buffer->capacity);
+        buffer->value = xrealloc_sized(buffer->value, buffer->capacity, original_capacity);
         if (buffer->value == NULL) return false;
     }
 
@@ -71,7 +85,7 @@ pm_buffer_append_length(pm_buffer_t *buffer, size_t length) {
 /**
  * Append a generic pointer to memory to the buffer.
  */
-static inline void
+static PRISM_INLINE void
 pm_buffer_append(pm_buffer_t *buffer, const void *source, size_t length) {
     size_t cursor = buffer->length;
     if (pm_buffer_append_length(buffer, length)) {
@@ -172,6 +186,15 @@ pm_buffer_append_double(pm_buffer_t *buffer, double value) {
     pm_buffer_append(buffer, source, sizeof(double));
 }
 
+/**
+ * Append a 64-bit unsigned integer to the buffer in native endian order.
+ */
+void
+pm_buffer_append_uint64(pm_buffer_t *buffer, uint64_t value) {
+    const void *source = &value;
+    pm_buffer_append(buffer, source, sizeof(uint64_t));
+}
+
 /**
  * Append a unicode codepoint to the buffer.
  */
@@ -349,9 +372,18 @@ pm_buffer_insert(pm_buffer_t *buffer, size_t index, const char *value, size_t le
 }
 
 /**
- * Free the memory associated with the buffer.
+ * Free the memory held by the buffer.
+ */
+void
+pm_buffer_cleanup(pm_buffer_t *buffer) {
+    xfree_sized(buffer->value, buffer->capacity);
+}
+
+/**
+ * Free both the memory held by the buffer and the buffer itself.
  */
 void
 pm_buffer_free(pm_buffer_t *buffer) {
-    xfree(buffer->value);
+    pm_buffer_cleanup(buffer);
+    xfree_sized(buffer, sizeof(pm_buffer_t));
 }

data/src/{util/pm_char.c → char.c}

@@ -1,7 +1,8 @@
-#include "prism/util/pm_char.h"
+#include "prism/internal/char.h"
+
+#include "prism/compiler/inline.h"
+#include "prism/internal/line_offset_list.h"
 
-#define PRISM_CHAR_BIT_WHITESPACE (1 << 0)
-#define PRISM_CHAR_BIT_INLINE_WHITESPACE (1 << 1)
 #define PRISM_CHAR_BIT_REGEXP_OPTION (1 << 2)
 
 #define PRISM_NUMBER_BIT_BINARY_DIGIT (1 << 0)
@@ -13,7 +14,7 @@
 #define PRISM_NUMBER_BIT_HEXADECIMAL_DIGIT (1 << 6)
 #define PRISM_NUMBER_BIT_HEXADECIMAL_NUMBER (1 << 7)
 
-static const uint8_t pm_byte_table[256] = {
+const uint8_t pm_byte_table[256] = {
 //  0  1  2  3  4  5  6  7  8  9  A  B  C  D  E  F
     0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, 0, 0, // 0x
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x
@@ -57,7 +58,7 @@ static const uint8_t pm_number_table[256] = {
  * Returns the number of characters at the start of the string that match the
  * given kind. Disallows searching past the given maximum number of characters.
  */
-static inline size_t
+static PRISM_INLINE size_t
 pm_strspn_char_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
     if (length <= 0) return 0;
 
@@ -77,21 +78,36 @@ pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length) {
     return pm_strspn_char_kind(string, length, PRISM_CHAR_BIT_WHITESPACE);
 }
 
+/**
+ * Returns the number of characters at the end of the string that are
+ * whitespace. Disallows searching past the given maximum number of characters.
+ */
+size_t
+pm_strrspn_whitespace(const uint8_t *string, ptrdiff_t length) {
+    if (length <= 0) return 0;
+
+    size_t size = 0;
+    size_t maximum = (size_t) length;
+
+    while (size < maximum && (pm_byte_table[string[maximum - size - 1]] & PRISM_CHAR_BIT_WHITESPACE)) size++;
+    return size;
+}
+
 /**
  * Returns the number of characters at the start of the string that are
  * whitespace while also tracking the location of each newline. Disallows
  * searching past the given maximum number of characters.
  */
 size_t
-pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list) {
+pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_arena_t *arena, pm_line_offset_list_t *line_offsets, uint32_t start_offset) {
     if (length <= 0) return 0;
 
-    size_t size = 0;
-    size_t maximum = (size_t) length;
+    uint32_t size = 0;
+    uint32_t maximum = (uint32_t) length;
 
     while (size < maximum && (pm_byte_table[string[size]] & PRISM_CHAR_BIT_WHITESPACE)) {
         if (string[size] == '\n') {
-            pm_newline_list_append(newline_list, string + size);
+            pm_line_offset_list_append(arena, line_offsets, start_offset + size + 1);
         }
 
         size++;
@@ -100,15 +116,6 @@ pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newlin
     return size;
 }
 
-/**
- * Returns the number of characters at the start of the string that are inline
- * whitespace. Disallows searching past the given maximum number of characters.
- */
-size_t
-pm_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length) {
-    return pm_strspn_char_kind(string, length, PRISM_CHAR_BIT_INLINE_WHITESPACE);
-}
-
 /**
  * Returns the number of characters at the start of the string that are regexp
  * options. Disallows searching past the given maximum number of characters.
@@ -118,36 +125,13 @@ pm_strspn_regexp_option(const uint8_t *string, ptrdiff_t length) {
     return pm_strspn_char_kind(string, length, PRISM_CHAR_BIT_REGEXP_OPTION);
 }
 
-/**
- * Returns true if the given character matches the given kind.
- */
-static inline bool
-pm_char_is_char_kind(const uint8_t b, uint8_t kind) {
-    return (pm_byte_table[b] & kind) != 0;
-}
-
-/**
- * Returns true if the given character is a whitespace character.
- */
-bool
-pm_char_is_whitespace(const uint8_t b) {
-    return pm_char_is_char_kind(b, PRISM_CHAR_BIT_WHITESPACE);
-}
-
-/**
- * Returns true if the given character is an inline whitespace character.
- */
-bool
-pm_char_is_inline_whitespace(const uint8_t b) {
-    return pm_char_is_char_kind(b, PRISM_CHAR_BIT_INLINE_WHITESPACE);
-}
 
 /**
  * Scan through the string and return the number of characters at the start of
  * the string that match the given kind. Disallows searching past the given
  * maximum number of characters.
  */
-static inline size_t
+static PRISM_INLINE size_t
 pm_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
     if (length <= 0) return 0;
 
@@ -166,7 +150,7 @@ pm_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
  * Additionally, report the location of the last invalid underscore character
  * found in the string through the out invalid parameter.
  */
-static inline size_t
+static PRISM_INLINE size_t
 pm_strspn_number_kind_underscores(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid, uint8_t kind) {
     if (length <= 0) return 0;
 
@@ -267,7 +251,7 @@ pm_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint
 /**
  * Returns true if the given character matches the given kind.
  */
-static inline bool
+static PRISM_INLINE bool
 pm_char_is_number_kind(const uint8_t b, uint8_t kind) {
     return (pm_number_table[b] & kind) != 0;
 }
@@ -303,16 +287,3 @@ bool
 pm_char_is_hexadecimal_digit(const uint8_t b) {
     return pm_char_is_number_kind(b, PRISM_NUMBER_BIT_HEXADECIMAL_DIGIT);
 }
-
-#undef PRISM_CHAR_BIT_WHITESPACE
-#undef PRISM_CHAR_BIT_INLINE_WHITESPACE
-#undef PRISM_CHAR_BIT_REGEXP_OPTION
-
-#undef PRISM_NUMBER_BIT_BINARY_DIGIT
-#undef PRISM_NUMBER_BIT_BINARY_NUMBER
-#undef PRISM_NUMBER_BIT_OCTAL_DIGIT
-#undef PRISM_NUMBER_BIT_OCTAL_NUMBER
-#undef PRISM_NUMBER_BIT_DECIMAL_DIGIT
-#undef PRISM_NUMBER_BIT_DECIMAL_NUMBER
-#undef PRISM_NUMBER_BIT_HEXADECIMAL_NUMBER
-#undef PRISM_NUMBER_BIT_HEXADECIMAL_DIGIT