prism 0.17.1 → 0.19.0

data/ext/prism/extconf.rb

@@ -1,136 +1,106 @@
 # frozen_string_literal: true
 
-require "rbconfig"
+if ARGV.delete("--help")
+  print(<<~TEXT)
+    USAGE: ruby #{$PROGRAM_NAME} [options]
+
+      Flags that are always valid:
+
+          --enable-debug-mode-build
+              Enable debug mode build.
+              You may also use set PRISM_DEBUG_MODE_BUILD environment variable.
+
+          --help
+              Display this message.
+
+      Environment variables used:
+
+          PRISM_DEBUG_MODE_BUILD
+              Equivalent to `--enable-debug-mode-build` when set, even if nil or blank.
 
-module Prism
-  module ExtConf
-    class << self
-      def configure
-        unless RUBY_ENGINE == "ruby"
-          # On non-CRuby we only need the shared library, so build only that and not the C extension.
-          # We also avoid `require "mkmf"` as that prepends the LLVM toolchain to PATH on TruffleRuby,
-          # but we want to use the native toolchain here since librubyparser is run natively.
-          build_shared_rubyparser
-          File.write("Makefile", "all install clean:\n\t@#{RbConfig::CONFIG["NULLCMD"]}\n")
-          return
-        end
-
-        require "mkmf"
-        configure_c_extension
-        configure_rubyparser
-
-        create_makefile("prism/prism")
-
-        if static_link?
-          File.open('Makefile', 'a') do |mf|
-            mf.puts
-            mf.puts '# Automatically rebuild the extension if librubyparser.a changed'
-            mf.puts '$(TARGET_SO): $(LOCAL_LIBS)'
-          end
-        end
-      end
-
-      def configure_c_extension
-        append_cflags("-DPRISM_DEBUG_MODE_BUILD") if debug_mode_build?
-        append_cflags("-fvisibility=hidden")
-      end
-
-      def configure_rubyparser
-        if static_link?
-          static_archive_path = File.join(build_dir, "librubyparser.a")
-          unless File.exist?(static_archive_path)
-            build_static_rubyparser
-          end
-          $LOCAL_LIBS << " #{static_archive_path}"
-        else
-          shared_library_path = File.join(build_dir, "librubyparser.#{RbConfig::CONFIG["SOEXT"]}")
-          unless File.exist?(shared_library_path)
-            build_shared_rubyparser
-          end
-          unless find_library("rubyparser", "pm_parser_init", build_dir)
-            raise "could not link against #{File.basename(shared_library_path)}"
-          end
-        end
-
-        find_header("prism.h", include_dir) or raise "prism.h is required"
-
-        # Explicitly look for the extension header in the parent directory
-        # because we want to consistently look for prism/extension.h in our
-        # source files to line up with our mirroring in CRuby.
-        find_header("prism/extension.h", File.join(__dir__, "..")) or raise "prism/extension.h is required"
-      end
-
-      def build_shared_rubyparser
-        build_target_rubyparser "build/librubyparser.#{RbConfig::CONFIG["SOEXT"]}"
-      end
-
-      def build_static_rubyparser
-        build_target_rubyparser "build/librubyparser.a"
-      end
-
-      def build_target_rubyparser(target)
-        Dir.chdir(root_dir) do
-          if !File.exist?("include/prism/ast.h") && Dir.exist?(".git")
-            # this block only exists to support building the gem from a "git" source,
-            # normally we package up the configure and other files in the gem itself
-            system("templates/template.rb", exception: true)
-          end
-          system("make", target, exception: true)
-        end
-      end
-
-      def root_dir
-        File.expand_path("../..", __dir__)
-      end
-
-      def include_dir
-        File.join(root_dir, "include")
-      end
-
-      def build_dir
-        File.join(root_dir, "build")
-      end
-
-      def print_help
-        print(<<~TEXT)
-          USAGE: ruby #{$PROGRAM_NAME} [options]
-
-            Flags that are always valid:
-
-                --enable-static
-                --disable-static
-                    Enable or disable static linking against librubyparser.
-                    The default is to statically link.
-
-                --enable-debug-mode-build
-                    Enable debug mode build.
-                    You may also use set PRISM_DEBUG_MODE_BUILD environment variable.
-
-                --help
-                    Display this message.
-
-            Environment variables used:
-
-                PRISM_DEBUG_MODE_BUILD
-                    Equivalent to `--enable-debug-mode-build` when set, even if nil or blank.
-
-        TEXT
-      end
-
-      def static_link?
-        enable_config("static", true)
-      end
-
-      def debug_mode_build?
-        enable_config("debug-mode-build", ENV["PRISM_DEBUG_MODE_BUILD"] || false)
-      end
+  TEXT
+  exit!(0)
+end
+
+# If this gem is being build from a git source, then we need to run
+# templating if it hasn't been run yet. In normal packaging, we would have
+# shipped the templated files with the gem, so this wouldn't be necessary.
+def generate_templates
+  Dir.chdir(File.expand_path("../..", __dir__)) do
+    if !File.exist?("include/prism/ast.h") && Dir.exist?(".git")
+      system("templates/template.rb", exception: true)
     end
   end
 end
 
-if ARGV.delete("--help")
-  Prism::ExtConf.print_help
-  exit!(0)
+# Runs `make` in the root directory of the project. Note that this is the
+# `Makefile` for the overall project, not the `Makefile` that is being generated
+# by this script.`
+def make(target)
+  Dir.chdir(File.expand_path("../..", __dir__)) do
+    system("make", target, exception: true)
+  end
 end
 
-Prism::ExtConf.configure
+require "rbconfig"
+
+# On non-CRuby we only need the shared library since we'll interface with it
+# through FFI, so we'll build only that and not the C extension. We also avoid
+# `require "mkmf"` as that prepends the LLVM toolchain to PATH on TruffleRuby,
+# but we want to use the native toolchain here since libprism is run natively.
+if RUBY_ENGINE != "ruby"
+  generate_templates
+  make("build/libprism.#{RbConfig::CONFIG["SOEXT"]}")
+  File.write("Makefile", "all install clean:\n\t@#{RbConfig::CONFIG["NULLCMD"]}\n")
+  return
+end
+
+require "mkmf"
+
+# First, ensure that we can find the header for the prism library.
+generate_templates # Templates should be generated before find_header.
+unless find_header("prism.h", File.expand_path("../../include", __dir__))
+  raise "prism.h is required"
+end
+
+# Next, ensure we can find the header for the C extension. Explicitly look for
+# the extension header in the parent directory because we want to consistently
+# look for `prism/extension.h` in our source files to line up with our mirroring
+# in CRuby.
+unless find_header("prism/extension.h", File.expand_path("..", __dir__))
+  raise "prism/extension.h is required"
+end
+
+# If `--enable-debug-mode-build` is passed to this script or the
+# `PRISM_DEBUG_MODE_BUILD` environment variable is defined, we'll build with the
+# `PRISM_DEBUG_MODE_BUILD` macro defined. This causes parse functions to
+# duplicate their input so that they have clearly set bounds, which is useful
+# for finding bugs that cause the parser to read off the end of the input.
+if enable_config("debug-mode-build", ENV["PRISM_DEBUG_MODE_BUILD"] || false)
+  append_cflags("-DPRISM_DEBUG_MODE_BUILD")
+end
+
+# By default, all symbols are hidden in the shared library.
+append_cflags("-fvisibility=hidden")
+
+# We need to link against the libprism.a archive, which is built by the
+# project's `Makefile`. We'll build it if it doesn't exist yet, and then add it
+# to `mkmf`'s list of local libraries.
+archive_target = "build/libprism.a"
+archive_path = File.expand_path("../../#{archive_target}", __dir__)
+
+make(archive_target) unless File.exist?(archive_path)
+$LOCAL_LIBS << " #{archive_path}"
+
+# Finally, we'll create the `Makefile` that is going to be used to configure and
+# build the C extension.
+create_makefile("prism/prism")
+
+# Now that the `Makefile` for the C extension is built, we'll append on an extra
+# rule that dictates that the extension should be rebuilt if the archive is
+# updated.
+File.open("Makefile", "a") do |mf|
+  mf.puts
+  mf.puts("# Automatically rebuild the extension if libprism.a changed")
+  mf.puts("$(TARGET_SO): $(LOCAL_LIBS)")
+end

data/ext/prism/extension.c

@@ -1,7 +1,7 @@
 #include "prism/extension.h"
 
 // NOTE: this file should contain only bindings. All non-trivial logic should be
-// in librubyparser so it can be shared its the various callers.
+// in libprism so it can be shared its the various callers.
 
 VALUE rb_cPrism;
 VALUE rb_cPrismNode;
@@ -12,7 +12,6 @@ VALUE rb_cPrismLocation;
 VALUE rb_cPrismComment;
 VALUE rb_cPrismInlineComment;
 VALUE rb_cPrismEmbDocComment;
-VALUE rb_cPrismDATAComment;
 VALUE rb_cPrismMagicComment;
 VALUE rb_cPrismParseError;
 VALUE rb_cPrismParseWarning;
@@ -127,7 +126,7 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
     } else if (key_id == rb_option_id_encoding) {
         if (!NIL_P(value)) pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value)));
     } else if (key_id == rb_option_id_line) {
-        if (!NIL_P(value)) pm_options_line_set(options, NUM2UINT(value));
+        if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value));
     } else if (key_id == rb_option_id_frozen_string_literal) {
         if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, value == Qtrue);
     } else if (key_id == rb_option_id_verbose) {
@@ -167,6 +166,7 @@ build_options(VALUE argument) {
  */
 static void
 extract_options(pm_options_t *options, VALUE filepath, VALUE keywords) {
+    options->line = 1; // default
     if (!NIL_P(keywords)) {
         struct build_options_data data = { .options = options, .keywords = keywords };
         struct build_options_data *argument = &data;
@@ -316,26 +316,11 @@ parser_comments(pm_parser_t *parser, VALUE source) {
     for (pm_comment_t *comment = (pm_comment_t *) parser->comment_list.head; comment != NULL; comment = (pm_comment_t *) comment->node.next) {
         VALUE location_argv[] = {
             source,
-            LONG2FIX(comment->start - parser->start),
-            LONG2FIX(comment->end - comment->start)
+            LONG2FIX(comment->location.start - parser->start),
+            LONG2FIX(comment->location.end - comment->location.start)
         };
 
-        VALUE type;
-        switch (comment->type) {
-            case PM_COMMENT_INLINE:
-                type = rb_cPrismInlineComment;
-                break;
-            case PM_COMMENT_EMBDOC:
-                type = rb_cPrismEmbDocComment;
-                break;
-            case PM_COMMENT___END__:
-                type = rb_cPrismDATAComment;
-                break;
-            default:
-                type = rb_cPrismInlineComment;
-                break;
-        }
-
+        VALUE type = (comment->type == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment;
         VALUE comment_argv[] = { rb_class_new_instance(3, location_argv, rb_cPrismLocation) };
         rb_ary_push(comments, rb_class_new_instance(1, comment_argv, type));
     }
@@ -374,6 +359,25 @@ parser_magic_comments(pm_parser_t *parser, VALUE source) {
     return magic_comments;
 }
 
+/**
+ * Extract out the data location from the parser into a Location instance if one
+ * exists.
+ */
+static VALUE
+parser_data_loc(const pm_parser_t *parser, VALUE source) {
+    if (parser->data_loc.end == NULL) {
+        return Qnil;
+    } else {
+        VALUE argv[] = {
+            source,
+            LONG2FIX(parser->data_loc.start - parser->start),
+            LONG2FIX(parser->data_loc.end - parser->data_loc.start)
+        };
+
+        return rb_class_new_instance(3, argv, rb_cPrismLocation);
+    }
+}
+
 /**
  * Extract the errors out of the parser into an array.
  */
@@ -385,8 +389,8 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
     for (error = (pm_diagnostic_t *) parser->error_list.head; error != NULL; error = (pm_diagnostic_t *) error->node.next) {
         VALUE location_argv[] = {
             source,
-            LONG2FIX(error->start - parser->start),
-            LONG2FIX(error->end - error->start)
+            LONG2FIX(error->location.start - parser->start),
+            LONG2FIX(error->location.end - error->location.start)
         };
 
         VALUE error_argv[] = {
@@ -411,8 +415,8 @@ parser_warnings(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
     for (warning = (pm_diagnostic_t *) parser->warning_list.head; warning != NULL; warning = (pm_diagnostic_t *) warning->node.next) {
         VALUE location_argv[] = {
             source,
-            LONG2FIX(warning->start - parser->start),
-            LONG2FIX(warning->end - warning->start)
+            LONG2FIX(warning->location.start - parser->start),
+            LONG2FIX(warning->location.end - warning->location.start)
         };
 
         VALUE warning_argv[] = {
@@ -465,7 +469,7 @@ parse_lex_token(void *data, pm_parser_t *parser, pm_token_t *token) {
 static void
 parse_lex_encoding_changed_callback(pm_parser_t *parser) {
     parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) parser->lex_callback->data;
-    parse_lex_data->encoding = rb_enc_find(parser->encoding.name);
+    parse_lex_data->encoding = rb_enc_find(parser->encoding->name);
 
     // Since the encoding changed, we need to go back and change the encoding of
     // the tokens that were already lexed. This is only going to end up being
@@ -531,6 +535,7 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
         value,
         parser_comments(&parser, source),
         parser_magic_comments(&parser, source),
+        parser_data_loc(&parser, source),
         parser_errors(&parser, parse_lex_data.encoding, source),
         parser_warnings(&parser, parse_lex_data.encoding, source),
         source
@@ -538,7 +543,7 @@ parse_lex_input(pm_string_t *input, const pm_options_t *options, bool return_nod
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
-    return rb_class_new_instance(6, result_argv, rb_cPrismParseResult);
+    return rb_class_new_instance(7, result_argv, rb_cPrismParseResult);
 }
 
 /**
@@ -594,19 +599,20 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
     pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
 
     pm_node_t *node = pm_parse(&parser);
-    rb_encoding *encoding = rb_enc_find(parser.encoding.name);
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
 
     VALUE source = pm_source_new(&parser, encoding);
     VALUE result_argv[] = {
         pm_ast_new(&parser, node, encoding),
         parser_comments(&parser, source),
         parser_magic_comments(&parser, source),
+        parser_data_loc(&parser, source),
         parser_errors(&parser, encoding, source),
         parser_warnings(&parser, encoding, source),
         source
     };
 
-    VALUE result = rb_class_new_instance(6, result_argv, rb_cPrismParseResult);
+    VALUE result = rb_class_new_instance(7, result_argv, rb_cPrismParseResult);
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
@@ -687,7 +693,7 @@ parse_input_comments(pm_string_t *input, const pm_options_t *options) {
     pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
 
     pm_node_t *node = pm_parse(&parser);
-    rb_encoding *encoding = rb_enc_find(parser.encoding.name);
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
 
     VALUE source = pm_source_new(&parser, encoding);
     VALUE comments = parser_comments(&parser, source);
@@ -792,6 +798,63 @@ parse_lex_file(int argc, VALUE *argv, VALUE self) {
     return value;
 }
 
+/**
+ * Parse the given input and return true if it parses without errors.
+ */
+static VALUE
+parse_input_success_p(pm_string_t *input, const pm_options_t *options) {
+    pm_parser_t parser;
+    pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
+
+    pm_node_t *node = pm_parse(&parser);
+    pm_node_destroy(&parser, node);
+
+    VALUE result = parser.error_list.size == 0 ? Qtrue : Qfalse;
+    pm_parser_free(&parser);
+
+    return result;
+}
+
+/**
+ * call-seq:
+ *   Prism::parse_success?(source, **options) -> Array
+ *
+ * Parse the given string and return true if it parses without errors. For
+ * supported options, see Prism::parse.
+ */
+static VALUE
+parse_success_p(int argc, VALUE *argv, VALUE self) {
+    pm_string_t input;
+    pm_options_t options = { 0 };
+    string_options(argc, argv, &input, &options);
+
+    VALUE result = parse_input_success_p(&input, &options);
+    pm_string_free(&input);
+    pm_options_free(&options);
+
+    return result;
+}
+
+/**
+ * call-seq:
+ *   Prism::parse_file_success?(filepath, **options) -> Array
+ *
+ * Parse the given file and return true if it parses without errors. For
+ * supported options, see Prism::parse.
+ */
+static VALUE
+parse_file_success_p(int argc, VALUE *argv, VALUE self) {
+    pm_string_t input;
+    pm_options_t options = { 0 };
+    if (!file_options(argc, argv, &input, &options)) return Qnil;
+
+    VALUE result = parse_input_success_p(&input, &options);
+    pm_string_free(&input);
+    pm_options_free(&options);
+
+    return result;
+}
+
 /******************************************************************************/
 /* Utility functions exposed to make testing easier                           */
 /******************************************************************************/
@@ -808,7 +871,7 @@ static VALUE
 named_captures(VALUE self, VALUE source) {
     pm_string_list_t string_list = { 0 };
 
-    if (!pm_regexp_named_capture_group_names((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, &pm_encoding_utf_8)) {
+    if (!pm_regexp_named_capture_group_names((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, PM_ENCODING_UTF_8_ENTRY)) {
         pm_string_list_free(&string_list);
         return Qnil;
     }
@@ -898,7 +961,7 @@ inspect_node(VALUE self, VALUE source) {
 
     pm_prettyprint(&buffer, &parser, node);
 
-    rb_encoding *encoding = rb_enc_find(parser.encoding.name);
+    rb_encoding *encoding = rb_enc_find(parser.encoding->name);
     VALUE string = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding);
 
     pm_buffer_free(&buffer);
@@ -938,7 +1001,6 @@ Init_prism(void) {
     rb_cPrismComment = rb_define_class_under(rb_cPrism, "Comment", rb_cObject);
     rb_cPrismInlineComment = rb_define_class_under(rb_cPrism, "InlineComment", rb_cPrismComment);
     rb_cPrismEmbDocComment = rb_define_class_under(rb_cPrism, "EmbDocComment", rb_cPrismComment);
-    rb_cPrismDATAComment = rb_define_class_under(rb_cPrism, "DATAComment", rb_cPrismComment);
     rb_cPrismMagicComment = rb_define_class_under(rb_cPrism, "MagicComment", rb_cObject);
     rb_cPrismParseError = rb_define_class_under(rb_cPrism, "ParseError", rb_cObject);
     rb_cPrismParseWarning = rb_define_class_under(rb_cPrism, "ParseWarning", rb_cObject);
@@ -976,6 +1038,8 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrism, "parse_file_comments", parse_file_comments, -1);
     rb_define_singleton_method(rb_cPrism, "parse_lex", parse_lex, -1);
     rb_define_singleton_method(rb_cPrism, "parse_lex_file", parse_lex_file, -1);
+    rb_define_singleton_method(rb_cPrism, "parse_success?", parse_success_p, -1);
+    rb_define_singleton_method(rb_cPrism, "parse_file_success?", parse_file_success_p, -1);
 
     // Next, the functions that will be called by the parser to perform various
     // internal tasks. We expose these to make them easier to test.

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "0.17.1"
+#define EXPECTED_PRISM_VERSION "0.19.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>