prism 0.23.0 → 0.25.0

data/ext/prism/api_pack.c

@@ -1,5 +1,12 @@
 #include "prism/extension.h"
 
+#ifdef PRISM_EXCLUDE_PACK
+
+void
+Init_prism_pack(void) {}
+
+#else
+
 static VALUE rb_cPrism;
 static VALUE rb_cPrismPack;
 static VALUE rb_cPrismPackDirective;
@@ -265,3 +272,5 @@ Init_prism_pack(void) {
     pack_symbol = ID2SYM(rb_intern("pack"));
     unpack_symbol = ID2SYM(rb_intern("unpack"));
 }
+
+#endif

data/ext/prism/extconf.rb

@@ -6,17 +6,24 @@ if ARGV.delete("--help")
 
       Flags that are always valid:
 
-          --enable-debug-mode-build
-              Enable debug mode build.
-              You may also use set PRISM_DEBUG_MODE_BUILD environment variable.
+          --enable-build-debug
+              Enable debug build.
+              You may also set the PRISM_BUILD_DEBUG environment variable.
+
+          --enable-build-minimal
+              Enable minimal build.
+              You may also set the PRISM_BUILD_MINIMAL environment variable.
 
           --help
               Display this message.
 
       Environment variables used:
 
-          PRISM_DEBUG_MODE_BUILD
-              Equivalent to `--enable-debug-mode-build` when set, even if nil or blank.
+          PRISM_BUILD_DEBUG
+              Equivalent to `--enable-build-debug` when set, even if nil or blank.
+
+          PRISM_BUILD_MINIMAL
+              Equivalent to `--enable-build-minimal` when set, even if nil or blank.
 
   TEXT
   exit!(0)
@@ -38,7 +45,7 @@ end
 # by this script.`
 def make(target)
   Dir.chdir(File.expand_path("../..", __dir__)) do
-    system("make", target, exception: true)
+    system(RUBY_PLATFORM.include?("openbsd") ? "gmake" : "make", target, exception: true)
   end
 end
 
@@ -71,13 +78,22 @@ unless find_header("prism/extension.h", File.expand_path("..", __dir__))
   raise "prism/extension.h is required"
 end
 
-# If `--enable-debug-mode-build` is passed to this script or the
-# `PRISM_DEBUG_MODE_BUILD` environment variable is defined, we'll build with the
-# `PRISM_DEBUG_MODE_BUILD` macro defined. This causes parse functions to
+# If `--enable-build-debug` is passed to this script or the
+# `PRISM_BUILD_DEBUG` environment variable is defined, we'll build with the
+# `PRISM_BUILD_DEBUG` macro defined. This causes parse functions to
 # duplicate their input so that they have clearly set bounds, which is useful
 # for finding bugs that cause the parser to read off the end of the input.
-if enable_config("debug-mode-build", ENV["PRISM_DEBUG_MODE_BUILD"] || false)
-  append_cflags("-DPRISM_DEBUG_MODE_BUILD")
+if enable_config("build-debug", ENV["PRISM_BUILD_DEBUG"] || false)
+  append_cflags("-DPRISM_BUILD_DEBUG")
+end
+
+# If `--enable-build-minimal` is passed to this script or the
+# `PRISM_BUILD_MINIMAL` environment variable is defined, we'll build with the
+# set of defines that comprise the minimal set. This causes the parser to be
+# built with minimal features, necessary for stripping out functionality when
+# the size of the final built artifact is a concern.
+if enable_config("build-minimal", ENV["PRISM_BUILD_MINIMAL"] || false)
+  append_cflags("-DPRISM_BUILD_MINIMAL")
 end
 
 # By default, all symbols are hidden in the shared library.

data/ext/prism/extension.c

@@ -21,12 +21,15 @@ VALUE rb_cPrismParseError;
 VALUE rb_cPrismParseWarning;
 VALUE rb_cPrismParseResult;
 
-ID rb_option_id_filepath;
+VALUE rb_cPrismDebugEncoding;
+
+ID rb_option_id_command_line;
 ID rb_option_id_encoding;
-ID rb_option_id_line;
+ID rb_option_id_filepath;
 ID rb_option_id_frozen_string_literal;
-ID rb_option_id_version;
+ID rb_option_id_line;
 ID rb_option_id_scopes;
+ID rb_option_id_version;
 
 /******************************************************************************/
 /* IO of Ruby code                                                            */
@@ -82,7 +85,9 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
 
     // Initialize the scopes array.
     size_t scopes_count = RARRAY_LEN(scopes);
-    pm_options_scopes_init(options, scopes_count);
+    if (!pm_options_scopes_init(options, scopes_count)) {
+        rb_raise(rb_eNoMemError, "failed to allocate memory");
+    }
 
     // Iterate over the scopes and add them to the options.
     for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) {
@@ -97,7 +102,9 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
         // Initialize the scope array.
         size_t locals_count = RARRAY_LEN(scope);
         pm_options_scope_t *options_scope = &options->scopes[scope_index];
-        pm_options_scope_init(options_scope, locals_count);
+        if (!pm_options_scope_init(options_scope, locals_count)) {
+            rb_raise(rb_eNoMemError, "failed to allocate memory");
+        }
 
         // Iterate over the locals and add them to the scope.
         for (size_t local_index = 0; local_index < locals_count; local_index++) {
@@ -132,19 +139,38 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
     } else if (key_id == rb_option_id_line) {
         if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value));
     } else if (key_id == rb_option_id_frozen_string_literal) {
-        if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, value == Qtrue);
+        if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, RTEST(value));
     } else if (key_id == rb_option_id_version) {
         if (!NIL_P(value)) {
             const char *version = check_string(value);
 
             if (!pm_options_version_set(options, version, RSTRING_LEN(value))) {
-                rb_raise(rb_eArgError, "invalid version: %"PRIsVALUE, value);
+                rb_raise(rb_eArgError, "invalid version: %" PRIsVALUE, value);
             }
         }
     } else if (key_id == rb_option_id_scopes) {
         if (!NIL_P(value)) build_options_scopes(options, value);
+    } else if (key_id == rb_option_id_command_line) {
+        if (!NIL_P(value)) {
+            const char *string = check_string(value);
+            uint8_t command_line = 0;
+
+            for (size_t index = 0; index < strlen(string); index++) {
+                switch (string[index]) {
+                    case 'a': command_line |= PM_OPTIONS_COMMAND_LINE_A; break;
+                    case 'e': command_line |= PM_OPTIONS_COMMAND_LINE_E; break;
+                    case 'l': command_line |= PM_OPTIONS_COMMAND_LINE_L; break;
+                    case 'n': command_line |= PM_OPTIONS_COMMAND_LINE_N; break;
+                    case 'p': command_line |= PM_OPTIONS_COMMAND_LINE_P; break;
+                    case 'x': command_line |= PM_OPTIONS_COMMAND_LINE_X; break;
+                    default: rb_raise(rb_eArgError, "invalid command line flag: '%c'", string[index]); break;
+                }
+            }
+
+            pm_options_command_line_set(options, command_line);
+        }
     } else {
-        rb_raise(rb_eArgError, "unknown keyword: %"PRIsVALUE, key);
+        rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, key);
     }
 
     return ST_CONTINUE;
@@ -228,7 +254,7 @@ file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) {
 
     const char * string_source = (const char *) pm_string_source(&options->filepath);
 
-    if (!pm_string_mapped_init(input, string_source)) {
+    if (!pm_string_file_init(input, string_source)) {
         pm_options_free(options);
 
 #ifdef _WIN32
@@ -241,6 +267,8 @@ file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) {
     }
 }
 
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+
 /******************************************************************************/
 /* Serializing the AST                                                        */
 /******************************************************************************/
@@ -282,17 +310,17 @@ dump(int argc, VALUE *argv, VALUE self) {
     pm_options_t options = { 0 };
     string_options(argc, argv, &input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
+#ifdef PRISM_BUILD_DEBUG
     size_t length = pm_string_length(&input);
-    char* dup = malloc(length);
+    char* dup = xmalloc(length);
     memcpy(dup, pm_string_source(&input), length);
     pm_string_constant_init(&input, dup, length);
 #endif
 
     VALUE value = dump_input(&input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
-    free(dup);
+#ifdef PRISM_BUILD_DEBUG
+    xfree(dup);
 #endif
 
     pm_string_free(&input);
@@ -322,6 +350,8 @@ dump_file(int argc, VALUE *argv, VALUE self) {
     return value;
 }
 
+#endif
+
 /******************************************************************************/
 /* Extracting values for the parse result                                     */
 /******************************************************************************/
@@ -415,23 +445,27 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
 
         VALUE level = Qnil;
         switch (error->level) {
-            case PM_ERROR_LEVEL_FATAL:
-                level = ID2SYM(rb_intern("fatal"));
+            case PM_ERROR_LEVEL_SYNTAX:
+                level = ID2SYM(rb_intern("syntax"));
                 break;
             case PM_ERROR_LEVEL_ARGUMENT:
                 level = ID2SYM(rb_intern("argument"));
                 break;
+            case PM_ERROR_LEVEL_LOAD:
+                level = ID2SYM(rb_intern("load"));
+                break;
             default:
                 rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, error->level);
         }
 
         VALUE error_argv[] = {
+            ID2SYM(rb_intern(pm_diagnostic_id_human(error->diag_id))),
             rb_enc_str_new_cstr(error->message, encoding),
             rb_class_new_instance(3, location_argv, rb_cPrismLocation),
             level
         };
 
-        rb_ary_push(errors, rb_class_new_instance(3, error_argv, rb_cPrismParseError));
+        rb_ary_push(errors, rb_class_new_instance(4, error_argv, rb_cPrismParseError));
     }
 
     return errors;
@@ -465,17 +499,36 @@ parser_warnings(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
         }
 
         VALUE warning_argv[] = {
+            ID2SYM(rb_intern(pm_diagnostic_id_human(warning->diag_id))),
             rb_enc_str_new_cstr(warning->message, encoding),
             rb_class_new_instance(3, location_argv, rb_cPrismLocation),
             level
         };
 
-        rb_ary_push(warnings, rb_class_new_instance(3, warning_argv, rb_cPrismParseWarning));
+        rb_ary_push(warnings, rb_class_new_instance(4, warning_argv, rb_cPrismParseWarning));
     }
 
     return warnings;
 }
 
+/**
+ * Create a new parse result from the given parser, value, encoding, and source.
+ */
+static VALUE
+parse_result_create(pm_parser_t *parser, VALUE value, rb_encoding *encoding, VALUE source) {
+    VALUE result_argv[] = {
+        value,
+        parser_comments(parser, source),
+        parser_magic_comments(parser, source),
+        parser_data_loc(parser, source),
+        parser_errors(parser, encoding, source),
+        parser_warnings(parser, encoding, source),
+        source
+    };
+
+    return rb_class_new_instance(7, result_argv, rb_cPrismParseResult);
+}
+
 /******************************************************************************/
 /* Lexing Ruby code                                                           */
 /******************************************************************************/
@@ -582,19 +635,11 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
         value = parse_lex_data.tokens;
     }
 
-    VALUE result_argv[] = {
-        value,
-        parser_comments(&parser, source),
-        parser_magic_comments(&parser, source),
-        parser_data_loc(&parser, source),
-        parser_errors(&parser, parse_lex_data.encoding, source),
-        parser_warnings(&parser, parse_lex_data.encoding, source),
-        source
-    };
-
+    VALUE result = parse_result_create(&parser, value, parse_lex_data.encoding, source);
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
-    return rb_class_new_instance(7, result_argv, rb_cPrismParseResult);
+
+    return result;
 }
 
 /**
@@ -654,17 +699,8 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
     rb_encoding *encoding = rb_enc_find(parser.encoding->name);
 
     VALUE source = pm_source_new(&parser, encoding);
-    VALUE result_argv[] = {
-        pm_ast_new(&parser, node, encoding, source),
-        parser_comments(&parser, source),
-        parser_magic_comments(&parser, source),
-        parser_data_loc(&parser, source),
-        parser_errors(&parser, encoding, source),
-        parser_warnings(&parser, encoding, source),
-        source
-    };
-
-    VALUE result = rb_class_new_instance(7, result_argv, rb_cPrismParseResult);
+    VALUE value = pm_ast_new(&parser, node, encoding, source);
+    VALUE result = parse_result_create(&parser, value, encoding, source) ;
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
@@ -679,21 +715,25 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
  * Parse the given string and return a ParseResult instance. The options that
  * are supported are:
  *
- * * `filepath` - the filepath of the source being parsed. This should be a
- *       string or nil
+ * * `command_line` - either nil or a string of the various options that were
+ *       set on the command line. Valid values are combinations of "a", "l",
+ *       "n", "p", and "x".
  * * `encoding` - the encoding of the source being parsed. This should be an
- *       encoding or nil
- * * `line` - the line number that the parse starts on. This should be an
- *       integer or nil. Note that this is 1-indexed.
+ *       encoding or nil.
+ * * `filepath` - the filepath of the source being parsed. This should be a
+ *       string or nil.
  * * `frozen_string_literal` - whether or not the frozen string literal pragma
  *       has been set. This should be a boolean or nil.
- * * `version` - the version of prism that should be used to parse Ruby code. By
- *       default prism assumes you want to parse with the latest vesion of
- *       prism (which you can trigger with `nil` or `"latest"`). If you want to
- *       parse exactly as CRuby 3.3.0 would, then you can pass `"3.3.0"`.
+ * * `line` - the line number that the parse starts on. This should be an
+ *       integer or nil. Note that this is 1-indexed.
  * * `scopes` - the locals that are in scope surrounding the code that is being
  *       parsed. This should be an array of arrays of symbols or nil. Scopes are
  *       ordered from the outermost scope to the innermost one.
+ * * `version` - the version of Ruby syntax that prism should used to parse Ruby
+ *       code. By default prism assumes you want to parse with the latest version
+ *       of Ruby syntax (which you can trigger with `nil` or `"latest"`). You
+ *       may also restrict the syntax to a specific version of Ruby. The
+ *       supported values are `"3.3.0"` and `"3.4.0"`.
  */
 static VALUE
 parse(int argc, VALUE *argv, VALUE self) {
@@ -701,17 +741,17 @@ parse(int argc, VALUE *argv, VALUE self) {
     pm_options_t options = { 0 };
     string_options(argc, argv, &input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
+#ifdef PRISM_BUILD_DEBUG
     size_t length = pm_string_length(&input);
-    char* dup = malloc(length);
+    char* dup = xmalloc(length);
     memcpy(dup, pm_string_source(&input), length);
     pm_string_constant_init(&input, dup, length);
 #endif
 
     VALUE value = parse_input(&input, &options);
 
-#ifdef PRISM_DEBUG_MODE_BUILD
-    free(dup);
+#ifdef PRISM_BUILD_DEBUG
+    xfree(dup);
 #endif
 
     pm_string_free(&input);
@@ -719,6 +759,60 @@ parse(int argc, VALUE *argv, VALUE self) {
     return value;
 }
 
+/**
+ * An implementation of fgets that is suitable for use with Ruby IO objects.
+ */
+static char *
+parse_stream_fgets(char *string, int size, void *stream) {
+    RUBY_ASSERT(size > 0);
+
+    VALUE line = rb_funcall((VALUE) stream, rb_intern("gets"), 1, INT2FIX(size - 1));
+    if (NIL_P(line)) {
+        return NULL;
+    }
+
+    const char *cstr = StringValueCStr(line);
+    size_t length = strlen(cstr);
+
+    memcpy(string, cstr, length);
+    string[length] = '\0';
+
+    return string;
+}
+
+/**
+ * call-seq:
+ *   Prism::parse_stream(stream, **options) -> ParseResult
+ *
+ * Parse the given object that responds to `gets` and return a ParseResult
+ * instance. The options that are supported are the same as Prism::parse.
+ */
+static VALUE
+parse_stream(int argc, VALUE *argv, VALUE self) {
+    VALUE stream;
+    VALUE keywords;
+    rb_scan_args(argc, argv, "1:", &stream, &keywords);
+
+    pm_options_t options = { 0 };
+    extract_options(&options, Qnil, keywords);
+
+    pm_parser_t parser;
+    pm_buffer_t buffer;
+
+    pm_node_t *node = pm_parse_stream(&parser, &buffer, (void *) stream, parse_stream_fgets, &options);
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
+
+    VALUE source = pm_source_new(&parser, encoding);
+    VALUE value = pm_ast_new(&parser, node, encoding, source);
+    VALUE result = parse_result_create(&parser, value, encoding, source);
+
+    pm_node_destroy(&parser, node);
+    pm_buffer_free(&buffer);
+    pm_parser_free(&parser);
+
+    return result;
+}
+
 /**
  * call-seq:
  *   Prism::parse_file(filepath, **options) -> ParseResult
@@ -945,6 +1039,35 @@ named_captures(VALUE self, VALUE source) {
     return names;
 }
 
+/**
+ * call-seq:
+ *   Debug::integer_parse(source) -> [Integer, String]
+ *
+ * Parses the given source string and returns the integer it represents, as well
+ * as a decimal string representation.
+ */
+static VALUE
+integer_parse(VALUE self, VALUE source) {
+    const uint8_t *start = (const uint8_t *) RSTRING_PTR(source);
+    size_t length = RSTRING_LEN(source);
+
+    pm_integer_t integer = { 0 };
+    pm_integer_parse(&integer, PM_INTEGER_BASE_UNKNOWN, start, start + length);
+
+    pm_buffer_t buffer = { 0 };
+    pm_integer_string(&buffer, &integer);
+
+    VALUE string = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer));
+    pm_buffer_free(&buffer);
+
+    VALUE result = rb_ary_new_capa(2);
+    rb_ary_push(result, pm_integer_new(&integer));
+    rb_ary_push(result, string);
+    pm_integer_free(&integer);
+
+    return result;
+}
+
 /**
  * call-seq:
  *   Debug::memsize(source) -> { length: xx, memsize: xx, node_count: xx }
@@ -1010,6 +1133,8 @@ profile_file(VALUE self, VALUE filepath) {
     return Qnil;
 }
 
+#ifndef PRISM_EXCLUDE_PRETTYPRINT
+
 /**
  * call-seq:
  *   Debug::inspect_node(source) -> inspected
@@ -1040,6 +1165,8 @@ inspect_node(VALUE self, VALUE source) {
     return string;
 }
 
+#endif
+
 /**
  * call-seq:
  *   Debug::format_errors(source, colorize) -> String
@@ -1057,7 +1184,7 @@ format_errors(VALUE self, VALUE source, VALUE colorize) {
     pm_node_t *node = pm_parse(&parser);
     pm_buffer_t buffer = { 0 };
 
-    pm_parser_errors_format(&parser, &buffer, RTEST(colorize));
+    pm_parser_errors_format(&parser, &parser.error_list, &buffer, RTEST(colorize), true);
 
     rb_encoding *encoding = rb_enc_find(parser.encoding->name);
     VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding);
@@ -1070,6 +1197,114 @@ format_errors(VALUE self, VALUE source, VALUE colorize) {
     return result;
 }
 
+/**
+ * call-seq:
+ *   Debug::static_inspect(source) -> String
+ *
+ * Inspect the node as it would be inspected by the warnings used in static
+ * literal sets.
+ */
+static VALUE
+static_inspect(int argc, VALUE *argv, VALUE self) {
+    pm_string_t input;
+    pm_options_t options = { 0 };
+    string_options(argc, argv, &input, &options);
+
+    pm_parser_t parser;
+    pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options);
+
+    pm_node_t *program = pm_parse(&parser);
+    pm_node_t *node = ((pm_program_node_t *) program)->statements->body.nodes[0];
+
+    pm_buffer_t buffer = { 0 };
+    pm_static_literal_inspect(&buffer, &parser, node);
+
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
+    VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding);
+
+    pm_buffer_free(&buffer);
+    pm_node_destroy(&parser, program);
+    pm_parser_free(&parser);
+    pm_string_free(&input);
+    pm_options_free(&options);
+
+    return result;
+}
+
+/**
+ * call-seq: Debug::Encoding.all -> Array[Debug::Encoding]
+ *
+ * Return an array of all of the encodings that prism knows about.
+ */
+static VALUE
+encoding_all(VALUE self) {
+    VALUE encodings = rb_ary_new();
+
+    for (size_t index = 0; index < PM_ENCODING_MAXIMUM; index++) {
+        const pm_encoding_t *encoding = &pm_encodings[index];
+
+        VALUE encoding_argv[] = { rb_str_new_cstr(encoding->name), encoding->multibyte ? Qtrue : Qfalse };
+        rb_ary_push(encodings, rb_class_new_instance(2, encoding_argv, rb_cPrismDebugEncoding));
+    }
+
+    return encodings;
+}
+
+static const pm_encoding_t *
+encoding_find(VALUE name) {
+    const uint8_t *source = (const uint8_t *) RSTRING_PTR(name);
+    size_t length = RSTRING_LEN(name);
+
+    const pm_encoding_t *encoding = pm_encoding_find(source, source + length);
+    if (encoding == NULL) { rb_raise(rb_eArgError, "Unknown encoding: %s", source); }
+
+    return encoding;
+}
+
+/**
+ * call-seq: Debug::Encoding.width(source) -> Integer
+ *
+ * Returns the width of the first character in the given string if it is valid
+ * in the encoding. If it is not, this function returns 0.
+ */
+static VALUE
+encoding_char_width(VALUE self, VALUE name, VALUE value) {
+    return ULONG2NUM(encoding_find(name)->char_width((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)));
+}
+
+/**
+ * call-seq: Debug::Encoding.alnum?(source) -> true | false
+ *
+ * Returns true if the first character in the given string is an alphanumeric
+ * character in the encoding.
+ */
+static VALUE
+encoding_alnum_char(VALUE self, VALUE name, VALUE value) {
+    return encoding_find(name)->alnum_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) > 0 ? Qtrue : Qfalse;
+}
+
+/**
+ * call-seq: Debug::Encoding.alpha?(source) -> true | false
+ *
+ * Returns true if the first character in the given string is an alphabetic
+ * character in the encoding.
+ */
+static VALUE
+encoding_alpha_char(VALUE self, VALUE name, VALUE value) {
+    return encoding_find(name)->alpha_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) > 0 ? Qtrue : Qfalse;
+}
+
+/**
+ * call-seq: Debug::Encoding.upper?(source) -> true | false
+ *
+ * Returns true if the first character in the given string is an uppercase
+ * character in the encoding.
+ */
+static VALUE
+encoding_isupper_char(VALUE self, VALUE name, VALUE value) {
+    return encoding_find(name)->isupper_char((const uint8_t *) RSTRING_PTR(value), RSTRING_LEN(value)) ? Qtrue : Qfalse;
+}
+
 /******************************************************************************/
 /* Initialization of the extension                                            */
 /******************************************************************************/
@@ -1107,31 +1342,24 @@ Init_prism(void) {
 
     // Intern all of the options that we support so that we don't have to do it
     // every time we parse.
-    rb_option_id_filepath = rb_intern_const("filepath");
+    rb_option_id_command_line = rb_intern_const("command_line");
     rb_option_id_encoding = rb_intern_const("encoding");
-    rb_option_id_line = rb_intern_const("line");
+    rb_option_id_filepath = rb_intern_const("filepath");
     rb_option_id_frozen_string_literal = rb_intern_const("frozen_string_literal");
-    rb_option_id_version = rb_intern_const("version");
+    rb_option_id_line = rb_intern_const("line");
     rb_option_id_scopes = rb_intern_const("scopes");
+    rb_option_id_version = rb_intern_const("version");
 
     /**
      * The version of the prism library.
      */
     rb_define_const(rb_cPrism, "VERSION", rb_str_new2(EXPECTED_PRISM_VERSION));
 
-    /**
-     * The backend of the parser that prism is using to parse Ruby code. This
-     * can be either :CEXT or :FFI. On runtimes that support C extensions, we
-     * default to :CEXT. Otherwise we use :FFI.
-     */
-    rb_define_const(rb_cPrism, "BACKEND", ID2SYM(rb_intern("CEXT")));
-
     // First, the functions that have to do with lexing and parsing.
-    rb_define_singleton_method(rb_cPrism, "dump", dump, -1);
-    rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1);
     rb_define_singleton_method(rb_cPrism, "lex", lex, -1);
     rb_define_singleton_method(rb_cPrism, "lex_file", lex_file, -1);
     rb_define_singleton_method(rb_cPrism, "parse", parse, -1);
+    rb_define_singleton_method(rb_cPrism, "parse_stream", parse_stream, -1);
     rb_define_singleton_method(rb_cPrism, "parse_file", parse_file, -1);
     rb_define_singleton_method(rb_cPrism, "parse_comments", parse_comments, -1);
     rb_define_singleton_method(rb_cPrism, "parse_file_comments", parse_file_comments, -1);
@@ -1140,14 +1368,33 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrism, "parse_success?", parse_success_p, -1);
     rb_define_singleton_method(rb_cPrism, "parse_file_success?", parse_file_success_p, -1);
 
+#ifndef PRISM_EXCLUDE_SERIALIZATION
+    rb_define_singleton_method(rb_cPrism, "dump", dump, -1);
+    rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1);
+#endif
+
     // Next, the functions that will be called by the parser to perform various
     // internal tasks. We expose these to make them easier to test.
     VALUE rb_cPrismDebug = rb_define_module_under(rb_cPrism, "Debug");
     rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1);
+    rb_define_singleton_method(rb_cPrismDebug, "integer_parse", integer_parse, 1);
     rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1);
     rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
     rb_define_singleton_method(rb_cPrismDebug, "format_errors", format_errors, 2);
+    rb_define_singleton_method(rb_cPrismDebug, "static_inspect", static_inspect, -1);
+
+#ifndef PRISM_EXCLUDE_PRETTYPRINT
+    rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
+#endif
+
+    // Next, define the functions that are exposed through the private
+    // Debug::Encoding class.
+    rb_cPrismDebugEncoding = rb_define_class_under(rb_cPrismDebug, "Encoding", rb_cObject);
+    rb_define_singleton_method(rb_cPrismDebugEncoding, "all", encoding_all, 0);
+    rb_define_singleton_method(rb_cPrismDebugEncoding, "_width", encoding_char_width, 2);
+    rb_define_singleton_method(rb_cPrismDebugEncoding, "_alnum?", encoding_alnum_char, 2);
+    rb_define_singleton_method(rb_cPrismDebugEncoding, "_alpha?", encoding_alpha_char, 2);
+    rb_define_singleton_method(rb_cPrismDebugEncoding, "_upper?", encoding_isupper_char, 2);
 
     // Next, initialize the other APIs.
     Init_prism_api_node();