prism 0.19.0 → 0.24.0

data/ext/prism/extension.c

@@ -1,5 +1,9 @@
 #include "prism/extension.h"
 
+#ifdef _WIN32
+#include <ruby/win32.h>
+#endif
+
 // NOTE: this file should contain only bindings. All non-trivial logic should be
 // in libprism so it can be shared its the various callers.
 
@@ -21,7 +25,7 @@ ID rb_option_id_filepath;
 ID rb_option_id_encoding;
 ID rb_option_id_line;
 ID rb_option_id_frozen_string_literal;
-ID rb_option_id_verbose;
+ID rb_option_id_version;
 ID rb_option_id_scopes;
 
 /******************************************************************************/
@@ -129,8 +133,14 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
         if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value));
     } else if (key_id == rb_option_id_frozen_string_literal) {
         if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, value == Qtrue);
-    } else if (key_id == rb_option_id_verbose) {
-        pm_options_suppress_warnings_set(options, value != Qtrue);
+    } else if (key_id == rb_option_id_version) {
+        if (!NIL_P(value)) {
+            const char *version = check_string(value);
+
+            if (!pm_options_version_set(options, version, RSTRING_LEN(value))) {
+                rb_raise(rb_eArgError, "invalid version: %"PRIsVALUE, value);
+            }
+        }
     } else if (key_id == rb_option_id_scopes) {
         if (!NIL_P(value)) build_options_scopes(options, value);
     } else {
@@ -206,20 +216,29 @@ string_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options)
 /**
  * Read options for methods that look like (filepath, **options).
  */
-static bool
+static void
 file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) {
     VALUE filepath;
     VALUE keywords;
     rb_scan_args(argc, argv, "1:", &filepath, &keywords);
 
+    Check_Type(filepath, T_STRING);
+
     extract_options(options, filepath, keywords);
 
-    if (!pm_string_mapped_init(input, (const char *) pm_string_source(&options->filepath))) {
+    const char * string_source = (const char *) pm_string_source(&options->filepath);
+
+    if (!pm_string_mapped_init(input, string_source)) {
         pm_options_free(options);
-        return false;
-    }
 
-    return true;
+#ifdef _WIN32
+        int e = rb_w32_map_errno(GetLastError());
+#else
+        int e = errno;
+#endif
+
+        rb_syserr_fail(e, string_source);
+    }
 }
 
 /******************************************************************************/
@@ -293,7 +312,8 @@ static VALUE
 dump_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = dump_input(&input, &options);
     pm_string_free(&input);
@@ -393,12 +413,25 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
             LONG2FIX(error->location.end - error->location.start)
         };
 
+        VALUE level = Qnil;
+        switch (error->level) {
+            case PM_ERROR_LEVEL_FATAL:
+                level = ID2SYM(rb_intern("fatal"));
+                break;
+            case PM_ERROR_LEVEL_ARGUMENT:
+                level = ID2SYM(rb_intern("argument"));
+                break;
+            default:
+                rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, error->level);
+        }
+
         VALUE error_argv[] = {
             rb_enc_str_new_cstr(error->message, encoding),
-            rb_class_new_instance(3, location_argv, rb_cPrismLocation)
+            rb_class_new_instance(3, location_argv, rb_cPrismLocation),
+            level
         };
 
-        rb_ary_push(errors, rb_class_new_instance(2, error_argv, rb_cPrismParseError));
+        rb_ary_push(errors, rb_class_new_instance(3, error_argv, rb_cPrismParseError));
     }
 
     return errors;
@@ -419,12 +452,25 @@ parser_warnings(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
             LONG2FIX(warning->location.end - warning->location.start)
         };
 
+        VALUE level = Qnil;
+        switch (warning->level) {
+            case PM_WARNING_LEVEL_DEFAULT:
+                level = ID2SYM(rb_intern("default"));
+                break;
+            case PM_WARNING_LEVEL_VERBOSE:
+                level = ID2SYM(rb_intern("verbose"));
+                break;
+            default:
+                rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, warning->level);
+        }
+
         VALUE warning_argv[] = {
             rb_enc_str_new_cstr(warning->message, encoding),
-            rb_class_new_instance(3, location_argv, rb_cPrismLocation)
+            rb_class_new_instance(3, location_argv, rb_cPrismLocation),
+            level
         };
 
-        rb_ary_push(warnings, rb_class_new_instance(2, warning_argv, rb_cPrismParseWarning));
+        rb_ary_push(warnings, rb_class_new_instance(3, warning_argv, rb_cPrismParseWarning));
     }
 
     return warnings;
@@ -496,8 +542,9 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
     pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
     pm_parser_register_encoding_changed_callback(&parser, parse_lex_encoding_changed_callback);
 
+    VALUE source_string = rb_str_new((const char *) pm_string_source(input), pm_string_length(input));
     VALUE offsets = rb_ary_new();
-    VALUE source_argv[] = { rb_str_new((const char *) pm_string_source(input), pm_string_length(input)), ULONG2NUM(parser.start_line), offsets };
+    VALUE source_argv[] = { source_string, LONG2NUM(parser.start_line), offsets };
     VALUE source = rb_class_new_instance(3, source_argv, rb_cPrismSource);
 
     parse_lex_data_t parse_lex_data = {
@@ -515,17 +562,21 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
     parser.lex_callback = &lex_callback;
     pm_node_t *node = pm_parse(&parser);
 
-    // Here we need to update the source range to have the correct newline
-    // offsets. We do it here because we've already created the object and given
-    // it over to all of the tokens.
+    // Here we need to update the Source object to have the correct
+    // encoding for the source string and the correct newline offsets.
+    // We do it here because we've already created the Source object and given
+    // it over to all of the tokens, and both of these are only set after pm_parse().
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
+    rb_enc_associate(source_string, encoding);
+
     for (size_t index = 0; index < parser.newline_list.size; index++) {
-        rb_ary_push(offsets, INT2FIX(parser.newline_list.offsets[index]));
+        rb_ary_push(offsets, ULONG2NUM(parser.newline_list.offsets[index]));
     }
 
     VALUE value;
     if (return_nodes) {
         value = rb_ary_new_capa(2);
-        rb_ary_push(value, pm_ast_new(&parser, node, parse_lex_data.encoding));
+        rb_ary_push(value, pm_ast_new(&parser, node, parse_lex_data.encoding, source));
         rb_ary_push(value, parse_lex_data.tokens);
     } else {
         value = parse_lex_data.tokens;
@@ -577,7 +628,8 @@ static VALUE
 lex_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = parse_lex_input(&input, &options, false);
     pm_string_free(&input);
@@ -603,7 +655,7 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
 
     VALUE source = pm_source_new(&parser, encoding);
     VALUE result_argv[] = {
-        pm_ast_new(&parser, node, encoding),
+        pm_ast_new(&parser, node, encoding, source),
         parser_comments(&parser, source),
         parser_magic_comments(&parser, source),
         parser_data_loc(&parser, source),
@@ -635,10 +687,13 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
  *       integer or nil. Note that this is 1-indexed.
  * * `frozen_string_literal` - whether or not the frozen string literal pragma
  *       has been set. This should be a boolean or nil.
- * * `verbose` - the current level of verbosity. This controls whether or not
- *       the parser emits warnings. This should be a boolean or nil.
+ * * `version` - the version of prism that should be used to parse Ruby code. By
+ *       default prism assumes you want to parse with the latest vesion of
+ *       prism (which you can trigger with `nil` or `"latest"`). If you want to
+ *       parse exactly as CRuby 3.3.0 would, then you can pass `"3.3.0"`.
  * * `scopes` - the locals that are in scope surrounding the code that is being
- *       parsed. This should be an array of arrays of symbols or nil.
+ *       parsed. This should be an array of arrays of symbols or nil. Scopes are
+ *       ordered from the outermost scope to the innermost one.
  */
 static VALUE
 parse(int argc, VALUE *argv, VALUE self) {
@@ -675,7 +730,8 @@ static VALUE
 parse_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = parse_input(&input, &options);
     pm_string_free(&input);
@@ -735,7 +791,8 @@ static VALUE
 parse_file_comments(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = parse_input_comments(&input, &options);
     pm_string_free(&input);
@@ -789,7 +846,8 @@ static VALUE
 parse_lex_file(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE value = parse_lex_input(&input, &options, true);
     pm_string_free(&input);
@@ -846,7 +904,8 @@ static VALUE
 parse_file_success_p(int argc, VALUE *argv, VALUE self) {
     pm_string_t input;
     pm_options_t options = { 0 };
-    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    file_options(argc, argv, &input, &options);
 
     VALUE result = parse_input_success_p(&input, &options);
     pm_string_free(&input);
@@ -924,7 +983,17 @@ profile_file(VALUE self, VALUE filepath) {
     pm_string_t input;
 
     const char *checked = check_string(filepath);
-    if (!pm_string_mapped_init(&input, checked)) return Qnil;
+    Check_Type(filepath, T_STRING);
+
+    if (!pm_string_mapped_init(&input, checked)) {
+#ifdef _WIN32
+        int e = rb_w32_map_errno(GetLastError());
+#else
+        int e = errno;
+#endif
+
+        rb_syserr_fail(e, checked);
+    }
 
     pm_options_t options = { 0 };
     pm_options_filepath_set(&options, checked);
@@ -971,6 +1040,36 @@ inspect_node(VALUE self, VALUE source) {
     return string;
 }
 
+/**
+ * call-seq:
+ *   Debug::format_errors(source, colorize) -> String
+ *
+ * Format the errors that are found when parsing the given source string.
+ */
+static VALUE
+format_errors(VALUE self, VALUE source, VALUE colorize) {
+    pm_string_t input;
+    input_load_string(&input, source);
+
+    pm_parser_t parser;
+    pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), NULL);
+
+    pm_node_t *node = pm_parse(&parser);
+    pm_buffer_t buffer = { 0 };
+
+    pm_parser_errors_format(&parser, &buffer, RTEST(colorize));
+
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
+    VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding);
+
+    pm_buffer_free(&buffer);
+    pm_node_destroy(&parser, node);
+    pm_parser_free(&parser);
+    pm_string_free(&input);
+
+    return result;
+}
+
 /******************************************************************************/
 /* Initialization of the extension                                            */
 /******************************************************************************/
@@ -1012,7 +1111,7 @@ Init_prism(void) {
     rb_option_id_encoding = rb_intern_const("encoding");
     rb_option_id_line = rb_intern_const("line");
     rb_option_id_frozen_string_literal = rb_intern_const("frozen_string_literal");
-    rb_option_id_verbose = rb_intern_const("verbose");
+    rb_option_id_version = rb_intern_const("version");
     rb_option_id_scopes = rb_intern_const("scopes");
 
     /**
@@ -1048,6 +1147,7 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1);
     rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
     rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
+    rb_define_singleton_method(rb_cPrismDebug, "format_errors", format_errors, 2);
 
     // Next, initialize the other APIs.
     Init_prism_api_node();

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "0.19.0"
+#define EXPECTED_PRISM_VERSION "0.24.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>
@@ -9,7 +9,7 @@
 
 VALUE pm_source_new(pm_parser_t *parser, rb_encoding *encoding);
 VALUE pm_token_new(pm_parser_t *parser, pm_token_t *token, rb_encoding *encoding, VALUE source);
-VALUE pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding);
+VALUE pm_ast_new(pm_parser_t *parser, pm_node_t *node, rb_encoding *encoding, VALUE source);
 
 void Init_prism_api_node(void);
 void Init_prism_pack(void);