jruby-prism-parser 0.24.0-java → 1.4.0-java

data/src/token_type.c

@@ -1,10 +1,11 @@
-/******************************************************************************/
+/*----------------------------------------------------------------------------*/
 /* This file is generated by the templates/template.rb script and should not  */
 /* be modified manually. See                                                  */
 /* templates/src/token_type.c.erb                                             */
 /* if you are looking to modify the                                           */
 /* template                                                                   */
-/******************************************************************************/
+/*----------------------------------------------------------------------------*/
+
 #include <string.h>
 
 #include "prism/ast.h"
@@ -361,7 +362,7 @@ const char *
 pm_token_type_human(pm_token_type_t token_type) {
     switch (token_type) {
         case PM_TOKEN_EOF:
-            return "end of file";
+            return "end-of-input";
         case PM_TOKEN_MISSING:
             return "missing token";
         case PM_TOKEN_NOT_PROVIDED:
@@ -421,9 +422,9 @@ pm_token_type_human(pm_token_type_t token_type) {
         case PM_TOKEN_DOT:
             return "'.'";
         case PM_TOKEN_DOT_DOT:
-            return "'..'";
+            return "..";
         case PM_TOKEN_DOT_DOT_DOT:
-            return "'...'";
+            return "...";
         case PM_TOKEN_EMBDOC_BEGIN:
             return "'=begin'";
         case PM_TOKEN_EMBDOC_END:
@@ -461,9 +462,9 @@ pm_token_type_human(pm_token_type_t token_type) {
         case PM_TOKEN_GREATER_EQUAL:
             return "'>='";
         case PM_TOKEN_GREATER_GREATER:
-            return "'>>'";
+            return ">>";
         case PM_TOKEN_GREATER_GREATER_EQUAL:
-            return "'>>='";
+            return ">>=";
         case PM_TOKEN_HEREDOC_END:
             return "heredoc ending";
         case PM_TOKEN_HEREDOC_START:
@@ -539,7 +540,7 @@ pm_token_type_human(pm_token_type_t token_type) {
         case PM_TOKEN_KEYWORD_RESCUE:
             return "'rescue'";
         case PM_TOKEN_KEYWORD_RESCUE_MODIFIER:
-            return "'rescue'";
+            return "'rescue' modifier";
         case PM_TOKEN_KEYWORD_RETRY:
             return "'retry'";
         case PM_TOKEN_KEYWORD_RETURN:
@@ -589,9 +590,9 @@ pm_token_type_human(pm_token_type_t token_type) {
         case PM_TOKEN_LESS_EQUAL_GREATER:
             return "'<=>'";
         case PM_TOKEN_LESS_LESS:
-            return "'<<'";
+            return "<<";
         case PM_TOKEN_LESS_LESS_EQUAL:
-            return "'<<='";
+            return "<<=";
         case PM_TOKEN_METHOD_NAME:
             return "method name";
         case PM_TOKEN_MINUS:
@@ -657,13 +658,13 @@ pm_token_type_human(pm_token_type_t token_type) {
         case PM_TOKEN_STAR_STAR_EQUAL:
             return "'**='";
         case PM_TOKEN_STRING_BEGIN:
-            return "string beginning";
+            return "string literal";
         case PM_TOKEN_STRING_CONTENT:
             return "string content";
         case PM_TOKEN_STRING_END:
             return "string ending";
         case PM_TOKEN_SYMBOL_BEGIN:
-            return "symbol beginning";
+            return "symbol literal";
         case PM_TOKEN_TILDE:
             return "'~'";
         case PM_TOKEN_UAMPERSAND:
@@ -683,7 +684,7 @@ pm_token_type_human(pm_token_type_t token_type) {
         case PM_TOKEN_USTAR:
             return "*";
         case PM_TOKEN_USTAR_STAR:
-            return "'**'";
+            return "**";
         case PM_TOKEN_WORDS_SEP:
             return "string separator";
         case PM_TOKEN___END__:

data/src/util/pm_buffer.c

@@ -16,7 +16,7 @@ pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity) {
     buffer->length = 0;
     buffer->capacity = capacity;
 
-    buffer->value = (char *) malloc(capacity);
+    buffer->value = (char *) xmalloc(capacity);
     return buffer->value != NULL;
 }
 
@@ -32,7 +32,7 @@ pm_buffer_init(pm_buffer_t *buffer) {
  * Return the value of the buffer.
  */
 char *
-pm_buffer_value(pm_buffer_t *buffer) {
+pm_buffer_value(const pm_buffer_t *buffer) {
     return buffer->value;
 }
 
@@ -40,14 +40,14 @@ pm_buffer_value(pm_buffer_t *buffer) {
  * Return the length of the buffer.
  */
 size_t
-pm_buffer_length(pm_buffer_t *buffer) {
+pm_buffer_length(const pm_buffer_t *buffer) {
     return buffer->length;
 }
 
 /**
  * Append the given amount of space to the buffer.
  */
-static inline void
+static inline bool
 pm_buffer_append_length(pm_buffer_t *buffer, size_t length) {
     size_t next_length = buffer->length + length;
 
@@ -60,10 +60,12 @@ pm_buffer_append_length(pm_buffer_t *buffer, size_t length) {
             buffer->capacity *= 2;
         }
 
-        buffer->value = realloc(buffer->value, buffer->capacity);
+        buffer->value = xrealloc(buffer->value, buffer->capacity);
+        if (buffer->value == NULL) return false;
     }
 
     buffer->length = next_length;
+    return true;
 }
 
 /**
@@ -72,8 +74,9 @@ pm_buffer_append_length(pm_buffer_t *buffer, size_t length) {
 static inline void
 pm_buffer_append(pm_buffer_t *buffer, const void *source, size_t length) {
     size_t cursor = buffer->length;
-    pm_buffer_append_length(buffer, length);
-    memcpy(buffer->value + cursor, source, length);
+    if (pm_buffer_append_length(buffer, length)) {
+        memcpy(buffer->value + cursor, source, length);
+    }
 }
 
 /**
@@ -82,8 +85,9 @@ pm_buffer_append(pm_buffer_t *buffer, const void *source, size_t length) {
 void
 pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length) {
     size_t cursor = buffer->length;
-    pm_buffer_append_length(buffer, length);
-    memset(buffer->value + cursor, 0, length);
+    if (pm_buffer_append_length(buffer, length)) {
+        memset(buffer->value + cursor, 0, length);
+    }
 }
 
 /**
@@ -100,13 +104,12 @@ pm_buffer_append_format(pm_buffer_t *buffer, const char *format, ...) {
     size_t length = (size_t) (result + 1);
 
     size_t cursor = buffer->length;
-    pm_buffer_append_length(buffer, length);
-
-    va_start(arguments, format);
-    vsnprintf(buffer->value + cursor, length, format, arguments);
-    va_end(arguments);
-
-    buffer->length--;
+    if (pm_buffer_append_length(buffer, length)) {
+        va_start(arguments, format);
+        vsnprintf(buffer->value + cursor, length, format, arguments);
+        va_end(arguments);
+        buffer->length--;
+    }
 }
 
 /**
@@ -160,15 +163,135 @@ pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value) {
     pm_buffer_append_varuint(buffer, unsigned_int);
 }
 
+/**
+ * Append a double to the buffer.
+ */
+void
+pm_buffer_append_double(pm_buffer_t *buffer, double value) {
+    const void *source = &value;
+    pm_buffer_append(buffer, source, sizeof(double));
+}
+
+/**
+ * Append a unicode codepoint to the buffer.
+ */
+bool
+pm_buffer_append_unicode_codepoint(pm_buffer_t *buffer, uint32_t value) {
+    if (value <= 0x7F) {
+        pm_buffer_append_byte(buffer, (uint8_t) value); // 0xxxxxxx
+        return true;
+    } else if (value <= 0x7FF) {
+        uint8_t bytes[] = {
+            (uint8_t) (0xC0 | ((value >> 6) & 0x3F)), // 110xxxxx
+            (uint8_t) (0x80 | (value & 0x3F))         // 10xxxxxx
+        };
+
+        pm_buffer_append_bytes(buffer, bytes, 2);
+        return true;
+    } else if (value <= 0xFFFF) {
+        uint8_t bytes[] = {
+            (uint8_t) (0xE0 | ((value >> 12) & 0x3F)), // 1110xxxx
+            (uint8_t) (0x80 | ((value >> 6) & 0x3F)),  // 10xxxxxx
+            (uint8_t) (0x80 | (value & 0x3F))          // 10xxxxxx
+        };
+
+        pm_buffer_append_bytes(buffer, bytes, 3);
+        return true;
+    } else if (value <= 0x10FFFF) {
+        uint8_t bytes[] = {
+            (uint8_t) (0xF0 | ((value >> 18) & 0x3F)), // 11110xxx
+            (uint8_t) (0x80 | ((value >> 12) & 0x3F)), // 10xxxxxx
+            (uint8_t) (0x80 | ((value >> 6) & 0x3F)),  // 10xxxxxx
+            (uint8_t) (0x80 | (value & 0x3F))          // 10xxxxxx
+        };
+
+        pm_buffer_append_bytes(buffer, bytes, 4);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+/**
+ * Append a slice of source code to the buffer.
+ */
+void
+pm_buffer_append_source(pm_buffer_t *buffer, const uint8_t *source, size_t length, pm_buffer_escaping_t escaping) {
+    for (size_t index = 0; index < length; index++) {
+        const uint8_t byte = source[index];
+
+        if ((byte <= 0x06) || (byte >= 0x0E && byte <= 0x1F) || (byte >= 0x7F)) {
+            if (escaping == PM_BUFFER_ESCAPING_RUBY) {
+                pm_buffer_append_format(buffer, "\\x%02X", byte);
+            } else {
+                pm_buffer_append_format(buffer, "\\u%04X", byte);
+            }
+        } else {
+            switch (byte) {
+                case '\a':
+                    if (escaping == PM_BUFFER_ESCAPING_RUBY) {
+                        pm_buffer_append_string(buffer, "\\a", 2);
+                    } else {
+                        pm_buffer_append_format(buffer, "\\u%04X", byte);
+                    }
+                    break;
+                case '\b':
+                    pm_buffer_append_string(buffer, "\\b", 2);
+                    break;
+                case '\t':
+                    pm_buffer_append_string(buffer, "\\t", 2);
+                    break;
+                case '\n':
+                    pm_buffer_append_string(buffer, "\\n", 2);
+                    break;
+                case '\v':
+                    if (escaping == PM_BUFFER_ESCAPING_RUBY) {
+                        pm_buffer_append_string(buffer, "\\v", 2);
+                    } else {
+                        pm_buffer_append_format(buffer, "\\u%04X", byte);
+                    }
+                    break;
+                case '\f':
+                    pm_buffer_append_string(buffer, "\\f", 2);
+                    break;
+                case '\r':
+                    pm_buffer_append_string(buffer, "\\r", 2);
+                    break;
+                case '"':
+                    pm_buffer_append_string(buffer, "\\\"", 2);
+                    break;
+                case '#': {
+                    if (escaping == PM_BUFFER_ESCAPING_RUBY && index + 1 < length) {
+                        const uint8_t next_byte = source[index + 1];
+                        if (next_byte == '{' || next_byte == '@' || next_byte == '$') {
+                            pm_buffer_append_byte(buffer, '\\');
+                        }
+                    }
+
+                    pm_buffer_append_byte(buffer, '#');
+                    break;
+                }
+                case '\\':
+                    pm_buffer_append_string(buffer, "\\\\", 2);
+                    break;
+                default:
+                    pm_buffer_append_byte(buffer, byte);
+                    break;
+            }
+        }
+    }
+}
+
 /**
  * Prepend the given string to the buffer.
  */
 void
 pm_buffer_prepend_string(pm_buffer_t *buffer, const char *value, size_t length) {
     size_t cursor = buffer->length;
-    pm_buffer_append_length(buffer, length);
-    memmove(buffer->value + length, buffer->value, cursor);
-    memcpy(buffer->value, value, length);
+    if (pm_buffer_append_length(buffer, length)) {
+        memmove(buffer->value + length, buffer->value, cursor);
+        memcpy(buffer->value, value, length);
+    }
 }
 
 /**
@@ -181,10 +304,54 @@ pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source) {
     }
 }
 
+/**
+ * Clear the buffer by reducing its size to 0. This does not free the allocated
+ * memory, but it does allow the buffer to be reused.
+ */
+void
+pm_buffer_clear(pm_buffer_t *buffer) {
+    buffer->length = 0;
+}
+
+/**
+ * Strip the whitespace from the end of the buffer.
+ */
+void
+pm_buffer_rstrip(pm_buffer_t *buffer) {
+    while (buffer->length > 0 && pm_char_is_whitespace((uint8_t) buffer->value[buffer->length - 1])) {
+        buffer->length--;
+    }
+}
+
+/**
+ * Checks if the buffer includes the given value.
+ */
+size_t
+pm_buffer_index(const pm_buffer_t *buffer, char value) {
+    const char *first = memchr(buffer->value, value, buffer->length);
+    return (first == NULL) ? SIZE_MAX : (size_t) (first - buffer->value);
+}
+
+/**
+ * Insert the given string into the buffer at the given index.
+ */
+void
+pm_buffer_insert(pm_buffer_t *buffer, size_t index, const char *value, size_t length) {
+    assert(index <= buffer->length);
+
+    if (index == buffer->length) {
+        pm_buffer_append_string(buffer, value, length);
+    } else {
+        pm_buffer_append_zeroes(buffer, length);
+        memmove(buffer->value + index + length, buffer->value + index, buffer->length - length - index);
+        memcpy(buffer->value + index, value, length);
+    }
+}
+
 /**
  * Free the memory associated with the buffer.
  */
 void
 pm_buffer_free(pm_buffer_t *buffer) {
-    free(buffer->value);
+    xfree(buffer->value);
 }

data/src/util/pm_char.c

@@ -19,10 +19,10 @@ static const uint8_t pm_byte_table[256] = {
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x
     3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2x
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 3x
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 4x
-    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 5x
-    0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4, 4, 4, // 6x
-    0, 0, 0, 4, 0, 4, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, // 7x
+    0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 4x
+    4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // 5x
+    0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 6x
+    4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // 7x
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 8x
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 9x
     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Ax
@@ -185,7 +185,7 @@ pm_strspn_number_kind_underscores(const uint8_t *string, ptrdiff_t length, const
         size++;
     }
 
-    if (string[size - 1] == '_') *invalid = string + size - 1;
+    if (size > 0 && string[size - 1] == '_') *invalid = string + size - 1;
     return size;
 }
 

data/src/util/pm_constant_pool.c

@@ -10,6 +10,22 @@ pm_constant_id_list_init(pm_constant_id_list_t *list) {
     list->capacity = 0;
 }
 
+/**
+ * Initialize a list of constant ids with a given capacity.
+ */
+void
+pm_constant_id_list_init_capacity(pm_constant_id_list_t *list, size_t capacity) {
+    if (capacity) {
+        list->ids = xcalloc(capacity, sizeof(pm_constant_id_t));
+        if (list->ids == NULL) abort();
+    } else {
+        list->ids = NULL;
+    }
+
+    list->size = 0;
+    list->capacity = capacity;
+}
+
 /**
  * Append a constant id to a list of constant ids. Returns false if any
  * potential reallocations fail.
@@ -18,7 +34,7 @@ bool
 pm_constant_id_list_append(pm_constant_id_list_t *list, pm_constant_id_t id) {
     if (list->size >= list->capacity) {
         list->capacity = list->capacity == 0 ? 8 : list->capacity * 2;
-        list->ids = (pm_constant_id_t *) realloc(list->ids, sizeof(pm_constant_id_t) * list->capacity);
+        list->ids = (pm_constant_id_t *) xrealloc(list->ids, sizeof(pm_constant_id_t) * list->capacity);
         if (list->ids == NULL) return false;
     }
 
@@ -26,6 +42,18 @@ pm_constant_id_list_append(pm_constant_id_list_t *list, pm_constant_id_t id) {
     return true;
 }
 
+/**
+ * Insert a constant id into a list of constant ids at the specified index.
+ */
+void
+pm_constant_id_list_insert(pm_constant_id_list_t *list, size_t index, pm_constant_id_t id) {
+    assert(index < list->capacity);
+    assert(list->ids[index] == PM_CONSTANT_ID_UNSET);
+
+    list->ids[index] = id;
+    list->size++;
+}
+
 /**
  * Checks if the current constant id list includes the given constant id.
  */
@@ -37,21 +65,13 @@ pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id) {
     return false;
 }
 
-/**
- * Get the memory size of a list of constant ids.
- */
-size_t
-pm_constant_id_list_memsize(pm_constant_id_list_t *list) {
-    return sizeof(pm_constant_id_list_t) + (list->capacity * sizeof(pm_constant_id_t));
-}
-
 /**
  * Free the memory associated with a list of constant ids.
  */
 void
 pm_constant_id_list_free(pm_constant_id_list_t *list) {
     if (list->ids != NULL) {
-        free(list->ids);
+        xfree(list->ids);
     }
 }
 
@@ -111,7 +131,7 @@ pm_constant_pool_resize(pm_constant_pool_t *pool) {
     const uint32_t mask = next_capacity - 1;
     const size_t element_size = sizeof(pm_constant_pool_bucket_t) + sizeof(pm_constant_t);
 
-    void *next = calloc(next_capacity, element_size);
+    void *next = xcalloc(next_capacity, element_size);
     if (next == NULL) return false;
 
     pm_constant_pool_bucket_t *next_buckets = next;
@@ -145,7 +165,7 @@ pm_constant_pool_resize(pm_constant_pool_t *pool) {
 
     // pool->constants and pool->buckets are allocated out of the same chunk
     // of memory, with the buckets coming first.
-    free(pool->buckets);
+    xfree(pool->buckets);
     pool->constants = next_constants;
     pool->buckets = next_buckets;
     pool->capacity = next_capacity;
@@ -162,7 +182,7 @@ pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity) {
 
     capacity = next_power_of_two(capacity);
     const size_t element_size = sizeof(pm_constant_pool_bucket_t) + sizeof(pm_constant_t);
-    void *memory = calloc(capacity, element_size);
+    void *memory = xcalloc(capacity, element_size);
     if (memory == NULL) return false;
 
     pool->buckets = memory;
@@ -237,12 +257,12 @@ pm_constant_pool_insert(pm_constant_pool_t *pool, const uint8_t *start, size_t l
                 // an existing constant, then either way we don't want the given
                 // memory. Either it's duplicated with the existing constant or
                 // it's not necessary because we have a shared version.
-                free((void *) start);
+                xfree((void *) start);
             } else if (bucket->type == PM_CONSTANT_POOL_BUCKET_OWNED) {
                 // If we're attempting to insert a shared constant and the
                 // existing constant is owned, then we can free the owned
                 // constant and replace it with the shared constant.
-                free((void *) constant->start);
+                xfree((void *) constant->start);
                 constant->start = start;
                 bucket->type = (unsigned int) (PM_CONSTANT_POOL_BUCKET_DEFAULT & 0x3);
             }
@@ -253,7 +273,7 @@ pm_constant_pool_insert(pm_constant_pool_t *pool, const uint8_t *start, size_t l
         index = (index + 1) & mask;
     }
 
-    // IDs are allocated starting at 1, since the value 0 denotes a non-existant
+    // IDs are allocated starting at 1, since the value 0 denotes a non-existent
     // constant.
     uint32_t id = ++pool->size;
     assert(pool->size < ((uint32_t) (1 << 30)));
@@ -287,7 +307,7 @@ pm_constant_pool_insert_shared(pm_constant_pool_t *pool, const uint8_t *start, s
  * potential calls to resize fail.
  */
 pm_constant_id_t
-pm_constant_pool_insert_owned(pm_constant_pool_t *pool, const uint8_t *start, size_t length) {
+pm_constant_pool_insert_owned(pm_constant_pool_t *pool, uint8_t *start, size_t length) {
     return pm_constant_pool_insert(pool, start, length, PM_CONSTANT_POOL_BUCKET_OWNED);
 }
 
@@ -314,9 +334,9 @@ pm_constant_pool_free(pm_constant_pool_t *pool) {
         // If an id is set on this constant, then we know we have content here.
         if (bucket->id != PM_CONSTANT_ID_UNSET && bucket->type == PM_CONSTANT_POOL_BUCKET_OWNED) {
             pm_constant_t *constant = &pool->constants[bucket->id - 1];
-            free((void *) constant->start);
+            xfree((void *) constant->start);
         }
     }
 
-    free(pool->buckets);
+    xfree(pool->buckets);
 }