jruby-prism-parser 0.24.0-java → 1.4.0-java

data/ext/prism/extension.c

@@ -19,34 +19,44 @@ VALUE rb_cPrismEmbDocComment;
 VALUE rb_cPrismMagicComment;
 VALUE rb_cPrismParseError;
 VALUE rb_cPrismParseWarning;
+VALUE rb_cPrismResult;
 VALUE rb_cPrismParseResult;
-
-ID rb_option_id_filepath;
-ID rb_option_id_encoding;
-ID rb_option_id_line;
-ID rb_option_id_frozen_string_literal;
-ID rb_option_id_version;
-ID rb_option_id_scopes;
+VALUE rb_cPrismLexResult;
+VALUE rb_cPrismParseLexResult;
+VALUE rb_cPrismStringQuery;
+VALUE rb_cPrismScope;
+
+VALUE rb_cPrismDebugEncoding;
+
+ID rb_id_option_command_line;
+ID rb_id_option_encoding;
+ID rb_id_option_filepath;
+ID rb_id_option_freeze;
+ID rb_id_option_frozen_string_literal;
+ID rb_id_option_line;
+ID rb_id_option_main_script;
+ID rb_id_option_partial_script;
+ID rb_id_option_scopes;
+ID rb_id_option_version;
+ID rb_id_source_for;
+ID rb_id_forwarding_positionals;
+ID rb_id_forwarding_keywords;
+ID rb_id_forwarding_block;
+ID rb_id_forwarding_all;
 
 /******************************************************************************/
 /* IO of Ruby code                                                            */
 /******************************************************************************/
 
 /**
- * Check if the given VALUE is a string. If it's nil, then return NULL. If it's
- * not a string, then raise a type error. Otherwise return the VALUE as a C
- * string.
+ * Check if the given VALUE is a string. If it's not a string, then raise a
+ * TypeError. Otherwise return the VALUE as a C string.
  */
 static const char *
 check_string(VALUE value) {
-    // If the value is nil, then we don't need to do anything.
-    if (NIL_P(value)) {
-        return NULL;
-    }
-
     // Check if the value is a string. If it's not, then raise a type error.
     if (!RB_TYPE_P(value, T_STRING)) {
-        rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(value));
+        rb_raise(rb_eTypeError, "wrong argument type %" PRIsVALUE " (expected String)", rb_obj_class(value));
     }
 
     // Otherwise, return the value as a C string.
@@ -60,7 +70,7 @@ static void
 input_load_string(pm_string_t *input, VALUE string) {
     // Check if the string is a string. If it's not, then raise a type error.
     if (!RB_TYPE_P(string, T_STRING)) {
-        rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(string));
+        rb_raise(rb_eTypeError, "wrong argument type %" PRIsVALUE " (expected String)", rb_obj_class(string));
     }
 
     pm_string_constant_init(input, RSTRING_PTR(string), RSTRING_LEN(string));
@@ -82,26 +92,69 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
 
     // Initialize the scopes array.
     size_t scopes_count = RARRAY_LEN(scopes);
-    pm_options_scopes_init(options, scopes_count);
+    if (!pm_options_scopes_init(options, scopes_count)) {
+        rb_raise(rb_eNoMemError, "failed to allocate memory");
+    }
 
     // Iterate over the scopes and add them to the options.
     for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) {
         VALUE scope = rb_ary_entry(scopes, scope_index);
 
-        // Check that the scope is an array. If it's not, then raise a type
-        // error.
-        if (!RB_TYPE_P(scope, T_ARRAY)) {
-            rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Array)", rb_obj_class(scope));
+        // The scope can be either an array or it can be a Prism::Scope object.
+        // Parse out the correct values here from either.
+        VALUE locals;
+        uint8_t forwarding = PM_OPTIONS_SCOPE_FORWARDING_NONE;
+
+        if (RB_TYPE_P(scope, T_ARRAY)) {
+            locals = scope;
+        } else if (rb_obj_is_kind_of(scope, rb_cPrismScope)) {
+            locals = rb_ivar_get(scope, rb_intern("@locals"));
+            if (!RB_TYPE_P(locals, T_ARRAY)) {
+                rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Array)", rb_obj_class(locals));
+            }
+
+            VALUE names = rb_ivar_get(scope, rb_intern("@forwarding"));
+            if (!RB_TYPE_P(names, T_ARRAY)) {
+                rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Array)", rb_obj_class(names));
+            }
+
+            size_t names_count = RARRAY_LEN(names);
+            for (size_t name_index = 0; name_index < names_count; name_index++) {
+                VALUE name = rb_ary_entry(names, name_index);
+
+                // Check that the name is a symbol. If it's not, then raise
+                // a type error.
+                if (!RB_TYPE_P(name, T_SYMBOL)) {
+                    rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Symbol)", rb_obj_class(name));
+                }
+
+                ID id = SYM2ID(name);
+                if (id == rb_id_forwarding_positionals) {
+                    forwarding |= PM_OPTIONS_SCOPE_FORWARDING_POSITIONALS;
+                } else if (id == rb_id_forwarding_keywords) {
+                    forwarding |= PM_OPTIONS_SCOPE_FORWARDING_KEYWORDS;
+                } else if (id == rb_id_forwarding_block) {
+                    forwarding |= PM_OPTIONS_SCOPE_FORWARDING_BLOCK;
+                } else if (id == rb_id_forwarding_all) {
+                    forwarding |= PM_OPTIONS_SCOPE_FORWARDING_ALL;
+                } else {
+                    rb_raise(rb_eArgError, "invalid forwarding value: %" PRIsVALUE, name);
+                }
+            }
+        } else {
+            rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Array or Prism::Scope)", rb_obj_class(scope));
         }
 
         // Initialize the scope array.
-        size_t locals_count = RARRAY_LEN(scope);
+        size_t locals_count = RARRAY_LEN(locals);
         pm_options_scope_t *options_scope = &options->scopes[scope_index];
-        pm_options_scope_init(options_scope, locals_count);
+        if (!pm_options_scope_init(options_scope, locals_count)) {
+            rb_raise(rb_eNoMemError, "failed to allocate memory");
+        }
 
         // Iterate over the locals and add them to the scope.
         for (size_t local_index = 0; local_index < locals_count; local_index++) {
-            VALUE local = rb_ary_entry(scope, local_index);
+            VALUE local = rb_ary_entry(locals, local_index);
 
             // Check that the local is a symbol. If it's not, then raise a
             // type error.
@@ -114,6 +167,9 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
             const char *name = rb_id2name(SYM2ID(local));
             pm_string_constant_init(scope_local, name, strlen(name));
         }
+
+        // Now set the forwarding options.
+        pm_options_scope_forwarding_set(options_scope, forwarding);
     }
 }
 
@@ -125,26 +181,57 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
     pm_options_t *options = (pm_options_t *) argument;
     ID key_id = SYM2ID(key);
 
-    if (key_id == rb_option_id_filepath) {
+    if (key_id == rb_id_option_filepath) {
         if (!NIL_P(value)) pm_options_filepath_set(options, check_string(value));
-    } else if (key_id == rb_option_id_encoding) {
-        if (!NIL_P(value)) pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value)));
-    } else if (key_id == rb_option_id_line) {
+    } else if (key_id == rb_id_option_encoding) {
+        if (!NIL_P(value)) {
+            if (value == Qfalse) {
+                pm_options_encoding_locked_set(options, true);
+            } else {
+                pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value)));
+            }
+        }
+    } else if (key_id == rb_id_option_line) {
         if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value));
-    } else if (key_id == rb_option_id_frozen_string_literal) {
-        if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, value == Qtrue);
-    } else if (key_id == rb_option_id_version) {
+    } else if (key_id == rb_id_option_frozen_string_literal) {
+        if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, RTEST(value));
+    } else if (key_id == rb_id_option_version) {
         if (!NIL_P(value)) {
             const char *version = check_string(value);
 
             if (!pm_options_version_set(options, version, RSTRING_LEN(value))) {
-                rb_raise(rb_eArgError, "invalid version: %"PRIsVALUE, value);
+                rb_raise(rb_eArgError, "invalid version: %" PRIsVALUE, value);
             }
         }
-    } else if (key_id == rb_option_id_scopes) {
+    } else if (key_id == rb_id_option_scopes) {
         if (!NIL_P(value)) build_options_scopes(options, value);
+    } else if (key_id == rb_id_option_command_line) {
+        if (!NIL_P(value)) {
+            const char *string = check_string(value);
+            uint8_t command_line = 0;
+
+            for (size_t index = 0; index < strlen(string); index++) {
+                switch (string[index]) {
+                    case 'a': command_line |= PM_OPTIONS_COMMAND_LINE_A; break;
+                    case 'e': command_line |= PM_OPTIONS_COMMAND_LINE_E; break;
+                    case 'l': command_line |= PM_OPTIONS_COMMAND_LINE_L; break;
+                    case 'n': command_line |= PM_OPTIONS_COMMAND_LINE_N; break;
+                    case 'p': command_line |= PM_OPTIONS_COMMAND_LINE_P; break;
+                    case 'x': command_line |= PM_OPTIONS_COMMAND_LINE_X; break;
+                    default: rb_raise(rb_eArgError, "invalid command line flag: '%c'", string[index]); break;
+                }
+            }
+
+            pm_options_command_line_set(options, command_line);
+        }
+    } else if (key_id == rb_id_option_main_script) {
+        if (!NIL_P(value)) pm_options_main_script_set(options, RTEST(value));
+    } else if (key_id == rb_id_option_partial_script) {
+        if (!NIL_P(value)) pm_options_partial_script_set(options, RTEST(value));
+    } else if (key_id == rb_id_option_freeze) {
+        if (!NIL_P(value)) pm_options_freeze_set(options, RTEST(value));
     } else {
-        rb_raise(rb_eArgError, "unknown keyword: %"PRIsVALUE, key);
+        rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, key);
     }
 
     return ST_CONTINUE;
@@ -177,6 +264,7 @@ build_options(VALUE argument) {
 static void
 extract_options(pm_options_t *options, VALUE filepath, VALUE keywords) {
     options->line = 1; // default
+
     if (!NIL_P(keywords)) {
         struct build_options_data data = { .options = options, .keywords = keywords };
         struct build_options_data *argument = &data;
@@ -217,30 +305,46 @@ string_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options)
  * Read options for methods that look like (filepath, **options).
  */
 static void
-file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) {
+file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options, VALUE *encoded_filepath) {
     VALUE filepath;
     VALUE keywords;
     rb_scan_args(argc, argv, "1:", &filepath, &keywords);
 
     Check_Type(filepath, T_STRING);
+    *encoded_filepath = rb_str_encode_ospath(filepath);
+    extract_options(options, *encoded_filepath, keywords);
 
-    extract_options(options, filepath, keywords);
+    const char *source = (const char *) pm_string_source(&options->filepath);
+    pm_string_init_result_t result;
 
-    const char * string_source = (const char *) pm_string_source(&options->filepath);
-
-    if (!pm_string_mapped_init(input, string_source)) {
-        pm_options_free(options);
+    switch (result = pm_string_file_init(input, source)) {
+        case PM_STRING_INIT_SUCCESS:
+            break;
+        case PM_STRING_INIT_ERROR_GENERIC: {
+            pm_options_free(options);
 
 #ifdef _WIN32
-        int e = rb_w32_map_errno(GetLastError());
+            int e = rb_w32_map_errno(GetLastError());
 #else
-        int e = errno;
+            int e = errno;
 #endif
 
-        rb_syserr_fail(e, string_source);
+            rb_syserr_fail(e, source);
+            break;
+        }
+        case PM_STRING_INIT_ERROR_DIRECTORY:
+            pm_options_free(options);
+            rb_syserr_fail(EISDIR, source);
+            break;
+        default:
+            pm_options_free(options);
+            rb_raise(rb_eRuntimeError, "Unknown error (%d) initializing file: %s", result, source);
+            break;
     }
 }
 
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+
 /******************************************************************************/
 /* Serializing the AST                                                        */
 /******************************************************************************/
@@ -282,17 +386,18 @@ dump(int argc, VALUE *argv, VALUE self) {
     pm_options_t options = { 0 };
     string_options(argc, argv, &input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
+#ifdef PRISM_BUILD_DEBUG
     size_t length = pm_string_length(&input);
-    char* dup = malloc(length);
+    char* dup = xmalloc(length);
     memcpy(dup, pm_string_source(&input), length);
     pm_string_constant_init(&input, dup, length);
 #endif
 
     VALUE value = dump_input(&input, &options);
+    if (options.freeze) rb_obj_freeze(value);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
-    free(dup);
+#ifdef PRISM_BUILD_DEBUG
+    xfree(dup);
 #endif
 
     pm_string_free(&input);
@@ -313,7 +418,8 @@ dump_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
 
-    file_options(argc, argv, &input, &options);
+    VALUE encoded_filepath;
+    file_options(argc, argv, &input, &options, &encoded_filepath);
 
     VALUE value = dump_input(&input, &options);
     pm_string_free(&input);
@@ -322,60 +428,96 @@ dump_file(int argc, VALUE *argv, VALUE self) {
     return value;
 }
 
+#endif
+
 /******************************************************************************/
 /* Extracting values for the parse result                                     */
 /******************************************************************************/
 
+/**
+ * The same as rb_class_new_instance, but accepts an additional boolean to
+ * indicate whether or not the resulting class instance should be frozen.
+ */
+static inline VALUE
+rb_class_new_instance_freeze(int argc, const VALUE *argv, VALUE klass, bool freeze) {
+    VALUE value = rb_class_new_instance(argc, argv, klass);
+    if (freeze) rb_obj_freeze(value);
+    return value;
+}
+
+/**
+ * Create a new Location instance from the given parser and bounds.
+ */
+static inline VALUE
+parser_location(const pm_parser_t *parser, VALUE source, bool freeze, const uint8_t *start, size_t length) {
+    VALUE argv[] = { source, LONG2FIX(start - parser->start), LONG2FIX(length) };
+    return rb_class_new_instance_freeze(3, argv, rb_cPrismLocation, freeze);
+}
+
+/**
+ * Create a new Location instance from the given parser and location.
+ */
+#define PARSER_LOCATION_LOC(parser, source, freeze, loc) \
+    parser_location(parser, source, freeze, loc.start, (size_t) (loc.end - loc.start))
+
+/**
+ * Build a new Comment instance from the given parser and comment.
+ */
+static inline VALUE
+parser_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_comment_t *comment) {
+    VALUE argv[] = { PARSER_LOCATION_LOC(parser, source, freeze, comment->location) };
+    VALUE type = (comment->type == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment;
+    return rb_class_new_instance_freeze(1, argv, type, freeze);
+}
+
 /**
  * Extract the comments out of the parser into an array.
  */
 static VALUE
-parser_comments(pm_parser_t *parser, VALUE source) {
-    VALUE comments = rb_ary_new();
-
-    for (pm_comment_t *comment = (pm_comment_t *) parser->comment_list.head; comment != NULL; comment = (pm_comment_t *) comment->node.next) {
-        VALUE location_argv[] = {
-            source,
-            LONG2FIX(comment->location.start - parser->start),
-            LONG2FIX(comment->location.end - comment->location.start)
-        };
-
-        VALUE type = (comment->type == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment;
-        VALUE comment_argv[] = { rb_class_new_instance(3, location_argv, rb_cPrismLocation) };
-        rb_ary_push(comments, rb_class_new_instance(1, comment_argv, type));
+parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
+    VALUE comments = rb_ary_new_capa(parser->comment_list.size);
+
+    for (
+        const pm_comment_t *comment = (const pm_comment_t *) parser->comment_list.head;
+        comment != NULL;
+        comment = (const pm_comment_t *) comment->node.next
+    ) {
+        VALUE value = parser_comment(parser, source, freeze, comment);
+        rb_ary_push(comments, value);
     }
 
+    if (freeze) rb_obj_freeze(comments);
     return comments;
 }
 
+/**
+ * Build a new MagicComment instance from the given parser and magic comment.
+ */
+static inline VALUE
+parser_magic_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) {
+    VALUE key_loc = parser_location(parser, source, freeze, magic_comment->key_start, magic_comment->key_length);
+    VALUE value_loc = parser_location(parser, source, freeze, magic_comment->value_start, magic_comment->value_length);
+    VALUE argv[] = { key_loc, value_loc };
+    return rb_class_new_instance_freeze(2, argv, rb_cPrismMagicComment, freeze);
+}
+
 /**
  * Extract the magic comments out of the parser into an array.
  */
 static VALUE
-parser_magic_comments(pm_parser_t *parser, VALUE source) {
-    VALUE magic_comments = rb_ary_new();
-
-    for (pm_magic_comment_t *magic_comment = (pm_magic_comment_t *) parser->magic_comment_list.head; magic_comment != NULL; magic_comment = (pm_magic_comment_t *) magic_comment->node.next) {
-        VALUE key_loc_argv[] = {
-            source,
-            LONG2FIX(magic_comment->key_start - parser->start),
-            LONG2FIX(magic_comment->key_length)
-        };
-
-        VALUE value_loc_argv[] = {
-            source,
-            LONG2FIX(magic_comment->value_start - parser->start),
-            LONG2FIX(magic_comment->value_length)
-        };
-
-        VALUE magic_comment_argv[] = {
-            rb_class_new_instance(3, key_loc_argv, rb_cPrismLocation),
-            rb_class_new_instance(3, value_loc_argv, rb_cPrismLocation)
-        };
-
-        rb_ary_push(magic_comments, rb_class_new_instance(2, magic_comment_argv, rb_cPrismMagicComment));
+parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
+    VALUE magic_comments = rb_ary_new_capa(parser->magic_comment_list.size);
+
+    for (
+        const pm_magic_comment_t *magic_comment = (const pm_magic_comment_t *) parser->magic_comment_list.head;
+        magic_comment != NULL;
+        magic_comment = (const pm_magic_comment_t *) magic_comment->node.next
+    ) {
+        VALUE value = parser_magic_comment(parser, source, freeze, magic_comment);
+        rb_ary_push(magic_comments, value);
     }
 
+    if (freeze) rb_obj_freeze(magic_comments);
     return magic_comments;
 }
 
@@ -384,17 +526,11 @@ parser_magic_comments(pm_parser_t *parser, VALUE source) {
  * exists.
  */
 static VALUE
-parser_data_loc(const pm_parser_t *parser, VALUE source) {
+parser_data_loc(const pm_parser_t *parser, VALUE source, bool freeze) {
     if (parser->data_loc.end == NULL) {
         return Qnil;
     } else {
-        VALUE argv[] = {
-            source,
-            LONG2FIX(parser->data_loc.start - parser->start),
-            LONG2FIX(parser->data_loc.end - parser->data_loc.start)
-        };
-
-        return rb_class_new_instance(3, argv, rb_cPrismLocation);
+        return PARSER_LOCATION_LOC(parser, source, freeze, parser->data_loc);
     }
 }
 
@@ -402,38 +538,39 @@ parser_data_loc(const pm_parser_t *parser, VALUE source) {
  * Extract the errors out of the parser into an array.
  */
 static VALUE
-parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
-    VALUE errors = rb_ary_new();
-    pm_diagnostic_t *error;
-
-    for (error = (pm_diagnostic_t *) parser->error_list.head; error != NULL; error = (pm_diagnostic_t *) error->node.next) {
-        VALUE location_argv[] = {
-            source,
-            LONG2FIX(error->location.start - parser->start),
-            LONG2FIX(error->location.end - error->location.start)
-        };
+parser_errors(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bool freeze) {
+    VALUE errors = rb_ary_new_capa(parser->error_list.size);
+
+    for (
+        const pm_diagnostic_t *error = (const pm_diagnostic_t *) parser->error_list.head;
+        error != NULL;
+        error = (const pm_diagnostic_t *) error->node.next
+    ) {
+        VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(error->diag_id)));
+        VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(error->message, encoding));
+        VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, error->location);
 
         VALUE level = Qnil;
         switch (error->level) {
-            case PM_ERROR_LEVEL_FATAL:
-                level = ID2SYM(rb_intern("fatal"));
+            case PM_ERROR_LEVEL_SYNTAX:
+                level = ID2SYM(rb_intern("syntax"));
                 break;
             case PM_ERROR_LEVEL_ARGUMENT:
                 level = ID2SYM(rb_intern("argument"));
                 break;
+            case PM_ERROR_LEVEL_LOAD:
+                level = ID2SYM(rb_intern("load"));
+                break;
             default:
                 rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, error->level);
         }
 
-        VALUE error_argv[] = {
-            rb_enc_str_new_cstr(error->message, encoding),
-            rb_class_new_instance(3, location_argv, rb_cPrismLocation),
-            level
-        };
-
-        rb_ary_push(errors, rb_class_new_instance(3, error_argv, rb_cPrismParseError));
+        VALUE argv[] = { type, message, location, level };
+        VALUE value = rb_class_new_instance_freeze(4, argv, rb_cPrismParseError, freeze);
+        rb_ary_push(errors, value);
     }
 
+    if (freeze) rb_obj_freeze(errors);
     return errors;
 }
 
@@ -441,16 +578,17 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
  * Extract the warnings out of the parser into an array.
  */
 static VALUE
-parser_warnings(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
-    VALUE warnings = rb_ary_new();
-    pm_diagnostic_t *warning;
-
-    for (warning = (pm_diagnostic_t *) parser->warning_list.head; warning != NULL; warning = (pm_diagnostic_t *) warning->node.next) {
-        VALUE location_argv[] = {
-            source,
-            LONG2FIX(warning->location.start - parser->start),
-            LONG2FIX(warning->location.end - warning->location.start)
-        };
+parser_warnings(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bool freeze) {
+    VALUE warnings = rb_ary_new_capa(parser->warning_list.size);
+
+    for (
+        const pm_diagnostic_t *warning = (const pm_diagnostic_t *) parser->warning_list.head;
+        warning != NULL;
+        warning = (const pm_diagnostic_t *) warning->node.next
+    ) {
+        VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(warning->diag_id)));
+        VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(warning->message, encoding));
+        VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, warning->location);
 
         VALUE level = Qnil;
         switch (warning->level) {
@@ -464,18 +602,33 @@ parser_warnings(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
                 rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, warning->level);
         }
 
-        VALUE warning_argv[] = {
-            rb_enc_str_new_cstr(warning->message, encoding),
-            rb_class_new_instance(3, location_argv, rb_cPrismLocation),
-            level
-        };
-
-        rb_ary_push(warnings, rb_class_new_instance(3, warning_argv, rb_cPrismParseWarning));
+        VALUE argv[] = { type, message, location, level };
+        VALUE value = rb_class_new_instance_freeze(4, argv, rb_cPrismParseWarning, freeze);
+        rb_ary_push(warnings, value);
     }
 
+    if (freeze) rb_obj_freeze(warnings);
     return warnings;
 }
 
+/**
+ * Create a new parse result from the given parser, value, encoding, and source.
+ */
+static VALUE
+parse_result_create(VALUE class, const pm_parser_t *parser, VALUE value, rb_encoding *encoding, VALUE source, bool freeze) {
+    VALUE result_argv[] = {
+        value,
+        parser_comments(parser, source, freeze),
+        parser_magic_comments(parser, source, freeze),
+        parser_data_loc(parser, source, freeze),
+        parser_errors(parser, encoding, source, freeze),
+        parser_warnings(parser, encoding, source, freeze),
+        source
+    };
+
+    return rb_class_new_instance_freeze(7, result_argv, class, freeze);
+}
+
 /******************************************************************************/
 /* Lexing Ruby code                                                           */
 /******************************************************************************/
@@ -489,6 +642,7 @@ typedef struct {
     VALUE source;
     VALUE tokens;
     rb_encoding *encoding;
+    bool freeze;
 } parse_lex_data_t;
 
 /**
@@ -500,9 +654,13 @@ static void
 parse_lex_token(void *data, pm_parser_t *parser, pm_token_t *token) {
     parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) parser->lex_callback->data;
 
-    VALUE yields = rb_ary_new_capa(2);
-    rb_ary_push(yields, pm_token_new(parser, token, parse_lex_data->encoding, parse_lex_data->source));
-    rb_ary_push(yields, INT2FIX(parser->lex_state));
+    VALUE value = pm_token_new(parser, token, parse_lex_data->encoding, parse_lex_data->source, parse_lex_data->freeze);
+    VALUE yields = rb_assoc_new(value, INT2FIX(parser->lex_state));
+
+    if (parse_lex_data->freeze) {
+        rb_obj_freeze(value);
+        rb_obj_freeze(yields);
+    }
 
     rb_ary_push(parse_lex_data->tokens, yields);
 }
@@ -522,14 +680,37 @@ parse_lex_encoding_changed_callback(pm_parser_t *parser) {
     // one or two tokens, since the encoding can only change at the top of the
     // file.
     VALUE tokens = parse_lex_data->tokens;
+    VALUE next_tokens = rb_ary_new();
+
     for (long index = 0; index < RARRAY_LEN(tokens); index++) {
         VALUE yields = rb_ary_entry(tokens, index);
         VALUE token = rb_ary_entry(yields, 0);
 
         VALUE value = rb_ivar_get(token, rb_intern("@value"));
-        rb_enc_associate(value, parse_lex_data->encoding);
-        ENC_CODERANGE_CLEAR(value);
+        VALUE next_value = rb_str_dup(value);
+
+        rb_enc_associate(next_value, parse_lex_data->encoding);
+        if (parse_lex_data->freeze) rb_obj_freeze(next_value);
+
+        VALUE next_token_argv[] = {
+            parse_lex_data->source,
+            rb_ivar_get(token, rb_intern("@type")),
+            next_value,
+            rb_ivar_get(token, rb_intern("@location"))
+        };
+
+        VALUE next_token = rb_class_new_instance(4, next_token_argv, rb_cPrismToken);
+        VALUE next_yields = rb_assoc_new(next_token, rb_ary_entry(yields, 1));
+
+        if (parse_lex_data->freeze) {
+            rb_obj_freeze(next_token);
+            rb_obj_freeze(next_yields);
+        }
+
+        rb_ary_push(next_tokens, next_yields);
     }
+
+    rb_ary_replace(parse_lex_data->tokens, next_tokens);
 }
 
 /**
@@ -543,14 +724,14 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
     pm_parser_register_encoding_changed_callback(&parser, parse_lex_encoding_changed_callback);
 
     VALUE source_string = rb_str_new((const char *) pm_string_source(input), pm_string_length(input));
-    VALUE offsets = rb_ary_new();
-    VALUE source_argv[] = { source_string, LONG2NUM(parser.start_line), offsets };
-    VALUE source = rb_class_new_instance(3, source_argv, rb_cPrismSource);
+    VALUE offsets = rb_ary_new_capa(parser.newline_list.size);
+    VALUE source = rb_funcall(rb_cPrismSource, rb_id_source_for, 3, source_string, LONG2NUM(parser.start_line), offsets);
 
     parse_lex_data_t parse_lex_data = {
         .source = source,
         .tokens = rb_ary_new(),
-        .encoding = rb_utf8_encoding()
+        .encoding = rb_utf8_encoding(),
+        .freeze = options->freeze,
     };
 
     parse_lex_data_t *data = &parse_lex_data;
@@ -573,36 +754,36 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
         rb_ary_push(offsets, ULONG2NUM(parser.newline_list.offsets[index]));
     }
 
-    VALUE value;
+    if (options->freeze) {
+        rb_obj_freeze(source_string);
+        rb_obj_freeze(offsets);
+        rb_obj_freeze(source);
+        rb_obj_freeze(parse_lex_data.tokens);
+    }
+
+    VALUE result;
     if (return_nodes) {
-        value = rb_ary_new_capa(2);
-        rb_ary_push(value, pm_ast_new(&parser, node, parse_lex_data.encoding, source));
+        VALUE value = rb_ary_new_capa(2);
+        rb_ary_push(value, pm_ast_new(&parser, node, parse_lex_data.encoding, source, options->freeze));
         rb_ary_push(value, parse_lex_data.tokens);
+        if (options->freeze) rb_obj_freeze(value);
+        result = parse_result_create(rb_cPrismParseLexResult, &parser, value, parse_lex_data.encoding, source, options->freeze);
     } else {
-        value = parse_lex_data.tokens;
+        result = parse_result_create(rb_cPrismLexResult, &parser, parse_lex_data.tokens, parse_lex_data.encoding, source, options->freeze);
     }
 
-    VALUE result_argv[] = {
-        value,
-        parser_comments(&parser, source),
-        parser_magic_comments(&parser, source),
-        parser_data_loc(&parser, source),
-        parser_errors(&parser, parse_lex_data.encoding, source),
-        parser_warnings(&parser, parse_lex_data.encoding, source),
-        source
-    };
-
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
-    return rb_class_new_instance(7, result_argv, rb_cPrismParseResult);
+
+    return result;
 }
 
 /**
  * call-seq:
- *   Prism::lex(source, **options) -> Array
+ *   Prism::lex(source, **options) -> LexResult
  *
- * Return an array of Token instances corresponding to the given string. For
- * supported options, see Prism::parse.
+ * Return a LexResult instance that contains an array of Token instances
+ * corresponding to the given string. For supported options, see Prism::parse.
  */
 static VALUE
 lex(int argc, VALUE *argv, VALUE self) {
@@ -619,17 +800,18 @@ lex(int argc, VALUE *argv, VALUE self) {
 
 /**
  * call-seq:
- *   Prism::lex_file(filepath, **options) -> Array
+ *   Prism::lex_file(filepath, **options) -> LexResult
  *
- * Return an array of Token instances corresponding to the given file. For
- * supported options, see Prism::parse.
+ * Return a LexResult instance that contains an array of Token instances
+ * corresponding to the given file. For supported options, see Prism::parse.
  */
 static VALUE
 lex_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
 
-    file_options(argc, argv, &input, &options);
+    VALUE encoded_filepath;
+    file_options(argc, argv, &input, &options, &encoded_filepath);
 
     VALUE value = parse_lex_input(&input, &options, false);
     pm_string_free(&input);
@@ -653,18 +835,13 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
     pm_node_t *node = pm_parse(&parser);
     rb_encoding *encoding = rb_enc_find(parser.encoding->name);
 
-    VALUE source = pm_source_new(&parser, encoding);
-    VALUE result_argv[] = {
-        pm_ast_new(&parser, node, encoding, source),
-        parser_comments(&parser, source),
-        parser_magic_comments(&parser, source),
-        parser_data_loc(&parser, source),
-        parser_errors(&parser, encoding, source),
-        parser_warnings(&parser, encoding, source),
-        source
-    };
+    VALUE source = pm_source_new(&parser, encoding, options->freeze);
+    VALUE value = pm_ast_new(&parser, node, encoding, source, options->freeze);
+    VALUE result = parse_result_create(rb_cPrismParseResult, &parser, value, encoding, source, options->freeze);
 
-    VALUE result = rb_class_new_instance(7, result_argv, rb_cPrismParseResult);
+    if (options->freeze) {
+        rb_obj_freeze(source);
+    }
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
@@ -679,21 +856,40 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
  * Parse the given string and return a ParseResult instance. The options that
  * are supported are:
  *
- * * `filepath` - the filepath of the source being parsed. This should be a
- *       string or nil
+ * * `command_line` - either nil or a string of the various options that were
+ *       set on the command line. Valid values are combinations of "a", "l",
+ *       "n", "p", and "x".
  * * `encoding` - the encoding of the source being parsed. This should be an
- *       encoding or nil
- * * `line` - the line number that the parse starts on. This should be an
- *       integer or nil. Note that this is 1-indexed.
+ *       encoding or nil.
+ * * `filepath` - the filepath of the source being parsed. This should be a
+ *       string or nil.
+ * * `freeze` - whether or not to deeply freeze the AST. This should be a
+ *       boolean or nil.
  * * `frozen_string_literal` - whether or not the frozen string literal pragma
  *       has been set. This should be a boolean or nil.
- * * `version` - the version of prism that should be used to parse Ruby code. By
- *       default prism assumes you want to parse with the latest vesion of
- *       prism (which you can trigger with `nil` or `"latest"`). If you want to
- *       parse exactly as CRuby 3.3.0 would, then you can pass `"3.3.0"`.
+ * * `line` - the line number that the parse starts on. This should be an
+ *       integer or nil. Note that this is 1-indexed.
+ * * `main_script` - a boolean indicating whether or not the source being parsed
+ *       is the main script being run by the interpreter. This controls whether
+ *       or not shebangs are parsed for additional flags and whether or not the
+ *       parser will attempt to find a matching shebang if the first one does
+ *       not contain the word "ruby".
+ * * `partial_script` - when the file being parsed is considered a "partial"
+ *       script, jumps will not be marked as errors if they are not contained
+ *       within loops/blocks. This is used in the case that you're parsing a
+ *       script that you know will be embedded inside another script later, but
+ *       you do not have that context yet. For example, when parsing an ERB
+ *       template that will be evaluated inside another script.
  * * `scopes` - the locals that are in scope surrounding the code that is being
  *       parsed. This should be an array of arrays of symbols or nil. Scopes are
  *       ordered from the outermost scope to the innermost one.
+ * * `version` - the version of Ruby syntax that prism should used to parse Ruby
+ *       code. By default prism assumes you want to parse with the latest
+ *       version of Ruby syntax (which you can trigger with `nil` or
+ *       `"latest"`). You may also restrict the syntax to a specific version of
+ *       Ruby, e.g., with `"3.3.0"`. To parse with the same syntax version that
+ *       the current Ruby is running use `version: RUBY_VERSION`. Raises
+ *       ArgumentError if the version is not currently supported by Prism.
  */
 static VALUE
 parse(int argc, VALUE *argv, VALUE self) {
@@ -701,17 +897,17 @@ parse(int argc, VALUE *argv, VALUE self) {
     pm_options_t options = { 0 };
     string_options(argc, argv, &input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
+#ifdef PRISM_BUILD_DEBUG
     size_t length = pm_string_length(&input);
-    char* dup = malloc(length);
+    char* dup = xmalloc(length);
     memcpy(dup, pm_string_source(&input), length);
     pm_string_constant_init(&input, dup, length);
 #endif
 
     VALUE value = parse_input(&input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
-    free(dup);
+#ifdef PRISM_BUILD_DEBUG
+    xfree(dup);
 #endif
 
     pm_string_free(&input);
@@ -731,7 +927,8 @@ parse_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
 
-    file_options(argc, argv, &input, &options);
+    VALUE encoded_filepath;
+    file_options(argc, argv, &input, &options, &encoded_filepath);
 
     VALUE value = parse_input(&input, &options);
     pm_string_free(&input);
@@ -740,6 +937,117 @@ parse_file(int argc, VALUE *argv, VALUE self) {
     return value;
 }
 
+/**
+ * Parse the given input and return nothing.
+ */
+static void
+profile_input(pm_string_t *input, const pm_options_t *options) {
+    pm_parser_t parser;
+    pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
+
+    pm_node_t *node = pm_parse(&parser);
+    pm_node_destroy(&parser, node);
+    pm_parser_free(&parser);
+}
+
+/**
+ * call-seq:
+ *   Prism::profile(source, **options) -> nil
+ *
+ * Parse the given string and return nothing. This method is meant to allow
+ * profilers to avoid the overhead of reifying the AST to Ruby. For supported
+ * options, see Prism::parse.
+ */
+static VALUE
+profile(int argc, VALUE *argv, VALUE self) {
+    pm_string_t input;
+    pm_options_t options = { 0 };
+
+    string_options(argc, argv, &input, &options);
+    profile_input(&input, &options);
+    pm_string_free(&input);
+    pm_options_free(&options);
+
+    return Qnil;
+}
+
+/**
+ * call-seq:
+ *   Prism::profile_file(filepath, **options) -> nil
+ *
+ * Parse the given file and return nothing. This method is meant to allow
+ * profilers to avoid the overhead of reifying the AST to Ruby. For supported
+ * options, see Prism::parse.
+ */
+static VALUE
+profile_file(int argc, VALUE *argv, VALUE self) {
+    pm_string_t input;
+    pm_options_t options = { 0 };
+
+    VALUE encoded_filepath;
+    file_options(argc, argv, &input, &options, &encoded_filepath);
+
+    profile_input(&input, &options);
+    pm_string_free(&input);
+    pm_options_free(&options);
+
+    return Qnil;
+}
+
+/**
+ * An implementation of fgets that is suitable for use with Ruby IO objects.
+ */
+static char *
+parse_stream_fgets(char *string, int size, void *stream) {
+    RUBY_ASSERT(size > 0);
+
+    VALUE line = rb_funcall((VALUE) stream, rb_intern("gets"), 1, INT2FIX(size - 1));
+    if (NIL_P(line)) {
+        return NULL;
+    }
+
+    const char *cstr = RSTRING_PTR(line);
+    long length = RSTRING_LEN(line);
+
+    memcpy(string, cstr, length);
+    string[length] = '\0';
+
+    return string;
+}
+
+/**
+ * call-seq:
+ *   Prism::parse_stream(stream, **options) -> ParseResult
+ *
+ * Parse the given object that responds to `gets` and return a ParseResult
+ * instance. The options that are supported are the same as Prism::parse.
+ */
+static VALUE
+parse_stream(int argc, VALUE *argv, VALUE self) {
+    VALUE stream;
+    VALUE keywords;
+    rb_scan_args(argc, argv, "1:", &stream, &keywords);
+
+    pm_options_t options = { 0 };
+    extract_options(&options, Qnil, keywords);
+
+    pm_parser_t parser;
+    pm_buffer_t buffer;
+
+    pm_node_t *node = pm_parse_stream(&parser, &buffer, (void *) stream, parse_stream_fgets, &options);
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
+
+    VALUE source = pm_source_new(&parser, encoding, options.freeze);
+    VALUE value = pm_ast_new(&parser, node, encoding, source, options.freeze);
+    VALUE result = parse_result_create(rb_cPrismParseResult, &parser, value, encoding, source, options.freeze);
+
+    pm_node_destroy(&parser, node);
+    pm_buffer_free(&buffer);
+    pm_parser_free(&parser);
+
+    return result;
+}
+
 /**
  * Parse the given input and return an array of Comment objects.
  */
@@ -751,8 +1059,8 @@ parse_input_comments(pm_string_t *input, const pm_options_t *options) {
     pm_node_t *node = pm_parse(&parser);
     rb_encoding *encoding = rb_enc_find(parser.encoding->name);
 
-    VALUE source = pm_source_new(&parser, encoding);
-    VALUE comments = parser_comments(&parser, source);
+    VALUE source = pm_source_new(&parser, encoding, options->freeze);
+    VALUE comments = parser_comments(&parser, source, options->freeze);
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
@@ -792,7 +1100,8 @@ parse_file_comments(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
 
-    file_options(argc, argv, &input, &options);
+    VALUE encoded_filepath;
+    file_options(argc, argv, &input, &options, &encoded_filepath);
 
     VALUE value = parse_input_comments(&input, &options);
     pm_string_free(&input);
@@ -803,9 +1112,9 @@ parse_file_comments(int argc, VALUE *argv, VALUE self) {
 
 /**
  * call-seq:
- *   Prism::parse_lex(source, **options) -> ParseResult
+ *   Prism::parse_lex(source, **options) -> ParseLexResult
  *
- * Parse the given string and return a ParseResult instance that contains a
+ * Parse the given string and return a ParseLexResult instance that contains a
  * 2-element array, where the first element is the AST and the second element is
  * an array of Token instances.
  *
@@ -830,9 +1139,9 @@ parse_lex(int argc, VALUE *argv, VALUE self) {
 
 /**
  * call-seq:
- *   Prism::parse_lex_file(filepath, **options) -> ParseResult
+ *   Prism::parse_lex_file(filepath, **options) -> ParseLexResult
  *
- * Parse the given file and return a ParseResult instance that contains a
+ * Parse the given file and return a ParseLexResult instance that contains a
  * 2-element array, where the first element is the AST and the second element is
  * an array of Token instances.
  *
@@ -847,7 +1156,8 @@ parse_lex_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
 
-    file_options(argc, argv, &input, &options);
+    VALUE encoded_filepath;
+    file_options(argc, argv, &input, &options, &encoded_filepath);
 
     VALUE value = parse_lex_input(&input, &options, true);
     pm_string_free(&input);
@@ -875,7 +1185,7 @@ parse_input_success_p(pm_string_t *input, const pm_options_t *options) {
 
 /**
  * call-seq:
- *   Prism::parse_success?(source, **options) -> Array
+ *   Prism::parse_success?(source, **options) -> bool
  *
  * Parse the given string and return true if it parses without errors. For
  * supported options, see Prism::parse.
@@ -895,7 +1205,19 @@ parse_success_p(int argc, VALUE *argv, VALUE self) {
 
 /**
  * call-seq:
- *   Prism::parse_file_success?(filepath, **options) -> Array
+ *   Prism::parse_failure?(source, **options) -> bool
+ *
+ * Parse the given string and return true if it parses with errors. For
+ * supported options, see Prism::parse.
+ */
+static VALUE
+parse_failure_p(int argc, VALUE *argv, VALUE self) {
+    return RTEST(parse_success_p(argc, argv, self)) ? Qfalse : Qtrue;
+}
+
+/**
+ * call-seq:
+ *   Prism::parse_file_success?(filepath, **options) -> bool
  *
  * Parse the given file and return true if it parses without errors. For
  * supported options, see Prism::parse.
@@ -905,7 +1227,8 @@ parse_file_success_p(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
 
-    file_options(argc, argv, &input, &options);
+    VALUE encoded_filepath;
+    file_options(argc, argv, &input, &options, &encoded_filepath);
 
     VALUE result = parse_input_success_p(&input, &options);
     pm_string_free(&input);
@@ -914,160 +1237,78 @@ parse_file_success_p(int argc, VALUE *argv, VALUE self) {
     return result;
 }
 
-/******************************************************************************/
-/* Utility functions exposed to make testing easier                           */
-/******************************************************************************/
-
 /**
  * call-seq:
- *   Debug::named_captures(source) -> Array
+ *   Prism::parse_file_failure?(filepath, **options) -> bool
  *
- * Returns an array of strings corresponding to the named capture groups in the
- * given source string. If prism was unable to parse the regular expression,
- * this function returns nil.
+ * Parse the given file and return true if it parses with errors. For
+ * supported options, see Prism::parse.
  */
 static VALUE
-named_captures(VALUE self, VALUE source) {
-    pm_string_list_t string_list = { 0 };
-
-    if (!pm_regexp_named_capture_group_names((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, PM_ENCODING_UTF_8_ENTRY)) {
-        pm_string_list_free(&string_list);
-        return Qnil;
-    }
-
-    VALUE names = rb_ary_new();
-    for (size_t index = 0; index < string_list.length; index++) {
-        const pm_string_t *string = &string_list.strings[index];
-        rb_ary_push(names, rb_str_new((const char *) pm_string_source(string), pm_string_length(string)));
-    }
-
-    pm_string_list_free(&string_list);
-    return names;
+parse_file_failure_p(int argc, VALUE *argv, VALUE self) {
+    return RTEST(parse_file_success_p(argc, argv, self)) ? Qfalse : Qtrue;
 }
 
+/******************************************************************************/
+/* String query methods                                                       */
+/******************************************************************************/
+
 /**
- * call-seq:
- *   Debug::memsize(source) -> { length: xx, memsize: xx, node_count: xx }
- *
- * Return a hash of information about the given source string's memory usage.
+ * Process the result of a call to a string query method and return an
+ * appropriate value.
  */
 static VALUE
-memsize(VALUE self, VALUE string) {
-    pm_parser_t parser;
-    size_t length = RSTRING_LEN(string);
-    pm_parser_init(&parser, (const uint8_t *) RSTRING_PTR(string), length, NULL);
-
-    pm_node_t *node = pm_parse(&parser);
-    pm_memsize_t memsize;
-    pm_node_memsize(node, &memsize);
-
-    pm_node_destroy(&parser, node);
-    pm_parser_free(&parser);
-
-    VALUE result = rb_hash_new();
-    rb_hash_aset(result, ID2SYM(rb_intern("length")), INT2FIX(length));
-    rb_hash_aset(result, ID2SYM(rb_intern("memsize")), INT2FIX(memsize.memsize));
-    rb_hash_aset(result, ID2SYM(rb_intern("node_count")), INT2FIX(memsize.node_count));
-    return result;
+string_query(pm_string_query_t result) {
+    switch (result) {
+        case PM_STRING_QUERY_ERROR:
+            rb_raise(rb_eArgError, "Invalid or non ascii-compatible encoding");
+            return Qfalse;
+        case PM_STRING_QUERY_FALSE:
+            return Qfalse;
+        case PM_STRING_QUERY_TRUE:
+            return Qtrue;
+    }
+    return Qfalse;
 }
 
 /**
  * call-seq:
- *   Debug::profile_file(filepath) -> nil
+ *   Prism::StringQuery::local?(string) -> bool
  *
- * Parse the file, but do nothing with the result. This is used to profile the
- * parser for memory and speed.
+ * Returns true if the string constitutes a valid local variable name. Note that
+ * this means the names that can be set through Binding#local_variable_set, not
+ * necessarily the ones that can be set through a local variable assignment.
  */
 static VALUE
-profile_file(VALUE self, VALUE filepath) {
-    pm_string_t input;
-
-    const char *checked = check_string(filepath);
-    Check_Type(filepath, T_STRING);
-
-    if (!pm_string_mapped_init(&input, checked)) {
-#ifdef _WIN32
-        int e = rb_w32_map_errno(GetLastError());
-#else
-        int e = errno;
-#endif
-
-        rb_syserr_fail(e, checked);
-    }
-
-    pm_options_t options = { 0 };
-    pm_options_filepath_set(&options, checked);
-
-    pm_parser_t parser;
-    pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options);
-
-    pm_node_t *node = pm_parse(&parser);
-    pm_node_destroy(&parser, node);
-    pm_parser_free(&parser);
-    pm_options_free(&options);
-    pm_string_free(&input);
-
-    return Qnil;
+string_query_local_p(VALUE self, VALUE string) {
+    const uint8_t *source = (const uint8_t *) check_string(string);
+    return string_query(pm_string_query_local(source, RSTRING_LEN(string), rb_enc_get(string)->name));
 }
 
 /**
  * call-seq:
- *   Debug::inspect_node(source) -> inspected
+ *   Prism::StringQuery::constant?(string) -> bool
  *
- * Inspect the AST that represents the given source using the prism pretty print
- * as opposed to the Ruby implementation.
+ * Returns true if the string constitutes a valid constant name. Note that this
+ * means the names that can be set through Module#const_set, not necessarily the
+ * ones that can be set through a constant assignment.
  */
 static VALUE
-inspect_node(VALUE self, VALUE source) {
-    pm_string_t input;
-    input_load_string(&input, source);
-
-    pm_parser_t parser;
-    pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), NULL);
-
-    pm_node_t *node = pm_parse(&parser);
-    pm_buffer_t buffer = { 0 };
-
-    pm_prettyprint(&buffer, &parser, node);
-
-    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
-    VALUE string = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding);
-
-    pm_buffer_free(&buffer);
-    pm_node_destroy(&parser, node);
-    pm_parser_free(&parser);
-
-    return string;
+string_query_constant_p(VALUE self, VALUE string) {
+    const uint8_t *source = (const uint8_t *) check_string(string);
+    return string_query(pm_string_query_constant(source, RSTRING_LEN(string), rb_enc_get(string)->name));
 }
 
 /**
  * call-seq:
- *   Debug::format_errors(source, colorize) -> String
+ *   Prism::StringQuery::method_name?(string) -> bool
  *
- * Format the errors that are found when parsing the given source string.
+ * Returns true if the string constitutes a valid method name.
  */
 static VALUE
-format_errors(VALUE self, VALUE source, VALUE colorize) {
-    pm_string_t input;
-    input_load_string(&input, source);
-
-    pm_parser_t parser;
-    pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), NULL);
-
-    pm_node_t *node = pm_parse(&parser);
-    pm_buffer_t buffer = { 0 };
-
-    pm_parser_errors_format(&parser, &buffer, RTEST(colorize));
-
-    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
-    VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding);
-
-    pm_buffer_free(&buffer);
-    pm_node_destroy(&parser, node);
-    pm_parser_free(&parser);
-    pm_string_free(&input);
-
-    return result;
+string_query_method_name_p(VALUE self, VALUE string) {
+    const uint8_t *source = (const uint8_t *) check_string(string);
+    return string_query(pm_string_query_method_name(source, RSTRING_LEN(string), rb_enc_get(string)->name));
 }
 
 /******************************************************************************/
@@ -1103,51 +1344,61 @@ Init_prism(void) {
     rb_cPrismMagicComment = rb_define_class_under(rb_cPrism, "MagicComment", rb_cObject);
     rb_cPrismParseError = rb_define_class_under(rb_cPrism, "ParseError", rb_cObject);
     rb_cPrismParseWarning = rb_define_class_under(rb_cPrism, "ParseWarning", rb_cObject);
-    rb_cPrismParseResult = rb_define_class_under(rb_cPrism, "ParseResult", rb_cObject);
-
-    // Intern all of the options that we support so that we don't have to do it
-    // every time we parse.
-    rb_option_id_filepath = rb_intern_const("filepath");
-    rb_option_id_encoding = rb_intern_const("encoding");
-    rb_option_id_line = rb_intern_const("line");
-    rb_option_id_frozen_string_literal = rb_intern_const("frozen_string_literal");
-    rb_option_id_version = rb_intern_const("version");
-    rb_option_id_scopes = rb_intern_const("scopes");
+    rb_cPrismResult = rb_define_class_under(rb_cPrism, "Result", rb_cObject);
+    rb_cPrismParseResult = rb_define_class_under(rb_cPrism, "ParseResult", rb_cPrismResult);
+    rb_cPrismLexResult = rb_define_class_under(rb_cPrism, "LexResult", rb_cPrismResult);
+    rb_cPrismParseLexResult = rb_define_class_under(rb_cPrism, "ParseLexResult", rb_cPrismResult);
+    rb_cPrismStringQuery = rb_define_class_under(rb_cPrism, "StringQuery", rb_cObject);
+    rb_cPrismScope = rb_define_class_under(rb_cPrism, "Scope", rb_cObject);
+
+    // Intern all of the IDs eagerly that we support so that we don't have to do
+    // it every time we parse.
+    rb_id_option_command_line = rb_intern_const("command_line");
+    rb_id_option_encoding = rb_intern_const("encoding");
+    rb_id_option_filepath = rb_intern_const("filepath");
+    rb_id_option_freeze = rb_intern_const("freeze");
+    rb_id_option_frozen_string_literal = rb_intern_const("frozen_string_literal");
+    rb_id_option_line = rb_intern_const("line");
+    rb_id_option_main_script = rb_intern_const("main_script");
+    rb_id_option_partial_script = rb_intern_const("partial_script");
+    rb_id_option_scopes = rb_intern_const("scopes");
+    rb_id_option_version = rb_intern_const("version");
+    rb_id_source_for = rb_intern("for");
+    rb_id_forwarding_positionals = rb_intern("*");
+    rb_id_forwarding_keywords = rb_intern("**");
+    rb_id_forwarding_block = rb_intern("&");
+    rb_id_forwarding_all = rb_intern("...");
 
     /**
      * The version of the prism library.
      */
-    rb_define_const(rb_cPrism, "VERSION", rb_str_new2(EXPECTED_PRISM_VERSION));
-
-    /**
-     * The backend of the parser that prism is using to parse Ruby code. This
-     * can be either :CEXT or :FFI. On runtimes that support C extensions, we
-     * default to :CEXT. Otherwise we use :FFI.
-     */
-    rb_define_const(rb_cPrism, "BACKEND", ID2SYM(rb_intern("CEXT")));
+    rb_define_const(rb_cPrism, "VERSION", rb_str_freeze(rb_str_new_cstr(EXPECTED_PRISM_VERSION)));
 
     // First, the functions that have to do with lexing and parsing.
-    rb_define_singleton_method(rb_cPrism, "dump", dump, -1);
-    rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1);
     rb_define_singleton_method(rb_cPrism, "lex", lex, -1);
     rb_define_singleton_method(rb_cPrism, "lex_file", lex_file, -1);
     rb_define_singleton_method(rb_cPrism, "parse", parse, -1);
     rb_define_singleton_method(rb_cPrism, "parse_file", parse_file, -1);
+    rb_define_singleton_method(rb_cPrism, "profile", profile, -1);
+    rb_define_singleton_method(rb_cPrism, "profile_file", profile_file, -1);
+    rb_define_singleton_method(rb_cPrism, "parse_stream", parse_stream, -1);
     rb_define_singleton_method(rb_cPrism, "parse_comments", parse_comments, -1);
     rb_define_singleton_method(rb_cPrism, "parse_file_comments", parse_file_comments, -1);
     rb_define_singleton_method(rb_cPrism, "parse_lex", parse_lex, -1);
     rb_define_singleton_method(rb_cPrism, "parse_lex_file", parse_lex_file, -1);
     rb_define_singleton_method(rb_cPrism, "parse_success?", parse_success_p, -1);
+    rb_define_singleton_method(rb_cPrism, "parse_failure?", parse_failure_p, -1);
     rb_define_singleton_method(rb_cPrism, "parse_file_success?", parse_file_success_p, -1);
+    rb_define_singleton_method(rb_cPrism, "parse_file_failure?", parse_file_failure_p, -1);
+
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+    rb_define_singleton_method(rb_cPrism, "dump", dump, -1);
+    rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1);
+#endif
 
-    // Next, the functions that will be called by the parser to perform various
-    // internal tasks. We expose these to make them easier to test.
-    VALUE rb_cPrismDebug = rb_define_module_under(rb_cPrism, "Debug");
-    rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "format_errors", format_errors, 2);
+    rb_define_singleton_method(rb_cPrismStringQuery, "local?", string_query_local_p, 1);
+    rb_define_singleton_method(rb_cPrismStringQuery, "constant?", string_query_constant_p, 1);
+    rb_define_singleton_method(rb_cPrismStringQuery, "method_name?", string_query_method_name_p, 1);
 
     // Next, initialize the other APIs.
     Init_prism_api_node();