prism 1.2.0 → 1.3.0

data/docs/build_system.md

@@ -16,9 +16,9 @@ The main solution for the second point seems a Makefile, otherwise many of the u
 ## General Design
 
 1. Templates are generated by `templates/template.rb`
-4. The `Makefile` compiles both `libprism.a` and `libprism.{so,dylib,dll}` from the `src/**/*.c` and `include/**/*.h` files
-5. The `Rakefile` `:compile` task ensures the above prerequisites are done, then calls `make`,
-  and uses `Rake::ExtensionTask` to compile the C extension (using its `extconf.rb`), which uses `libprism.a`
+2. The `Makefile` compiles both `libprism.a` and `libprism.{so,dylib,dll}` from the `src/**/*.c` and `include/**/*.h` files
+3. The `Rakefile` `:compile` task ensures the above prerequisites are done, then calls `make`,
+   and uses `Rake::ExtensionTask` to compile the C extension (using its `extconf.rb`)
 
 This way there is minimal duplication, and each layer builds on the previous one and has its own responsibilities.
 
@@ -35,14 +35,11 @@ loaded per process (i.e., at most one version of the prism *gem* loaded in a pro
 ### Building the prism gem by `gem install/bundle install`
 
 The gem contains the pre-generated templates.
-When installing the gem, `extconf.rb` is used and that:
-* runs `make build/libprism.a`
-* compiles the C extension with mkmf
 
-When installing the gem on JRuby and TruffleRuby, no C extension is built, so instead of the last step,
-there is Ruby code using FFI which uses `libprism.{so,dylib,dll}`
-to implement the same methods as the C extension, but using serialization instead of many native calls/accesses
-(JRuby does not support C extensions, serialization is faster on TruffleRuby than the C extension).
+When installing the gem on CRuby, `extconf.rb` is used and that compiles the C extension with mkmf, including both the extension files and the sources of prism itself.
+
+When installing the gem on JRuby and TruffleRuby, no C extension is built, so instead the `extconf.rb` runs `make build/libprism.{so,dylib,dll}`.
+There is Ruby code using FFI which uses `libprism.{so,dylib,dll}` to implement the same methods as the C extension, but using serialization instead of many native calls/accesses (JRuby does not support C extensions, serialization is faster on TruffleRuby than the C extension).
 
 ### Building the prism gem from git, e.g. `gem "prism", github: "ruby/prism"`
 
@@ -66,7 +63,7 @@ The script generates the templates when importing.
 
 Then when `mx build` builds TruffleRuby and the `prism` mx project inside, it runs `make`.
 
-Then the `prism bindings` mx project is built, which contains the [bindings](https://github.com/oracle/truffleruby/blob/master/src/main/c/prism_bindings/src/prism_bindings.c)
+Then the `prism bindings` mx project is built, which contains the [bindings](https://github.com/oracle/truffleruby/blob/vm-24.1.1/src/main/c/yarp_bindings/src/yarp_bindings.c)
 and links to `libprism.a` (to avoid exporting symbols, so no conflict when installing the prism gem).
 
 ### Building prism as part of JRuby

data/docs/relocation.md

@@ -0,0 +1,34 @@
+# Relocation
+
+Prism parses deterministically for the same input. This provides a nice property that is exposed through the `#node_id` API on nodes. Effectively this means that for the same input, these values will remain consistent every time the source is parsed. This means we can reparse the source same with a `#node_id` value and find the exact same node again.
+
+The `Relocation` module provides an API around this property. It allows you to "save" nodes and locations using a minimal amount of memory (just the node_id and a field identifier) and then reify them later. This minimizes the amount of memory you need to allocate to store this information because it does not keep around a pointer to the source string.
+
+## Getting started
+
+To get started with the `Relocation` module, you would first instantiate a `Repository` object. You do this through a DSL that chains method calls for configuration. For example, if for every entry in the repository you want to store the start and end lines, the start and end code unit columns for in UTF-16, and the leading comments, you would:
+
+```ruby
+repository = Prism::Relocation.filepath("path/to/file").lines.code_unit_columns(Encoding::UTF_16).leading_comments
+```
+
+Now that you have the repository, you can pass it into any of the `save*` APIs on nodes or locations to create entries in the repository that will be lazily reified.
+
+```ruby
+# assume that node is a Prism::ClassNode object
+entry = node.constant_path.save(repository)
+```
+
+Now that you have the entry object, you do not need to keep around a reference to the repository, it will be cleaned up on its own when the last entry is reified. Now, whenever you need to, you may call the associated field methods on the entry object, as in:
+
+```ruby
+entry.start_line
+entry.end_line
+
+entry.start_code_units_column
+entry.end_code_units_column
+
+entry.leading_comments
+```
+
+Note that if you had configured other fields to be saved, you would be able to access them as well. The first time one of these fields is accessed, the repository will reify every entry it knows about and then clean itself up. In this way, you can effectively treat them as if you had kept around lightweight versions of `Prism::Node` or `Prism::Location` objects.

data/ext/prism/api_node.c

@@ -6075,7 +6075,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
 #line 172 "prism/templates/ext/prism/api_node.c.erb"
                 case PM_UNTIL_NODE: {
                     pm_until_node_t *cast = (pm_until_node_t *) node;
-                    VALUE argv[8];
+                    VALUE argv[9];
 
                     // source
                     argv[0] = source;
@@ -6093,19 +6093,23 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
 #line 220 "prism/templates/ext/prism/api_node.c.erb"
                     argv[4] = pm_location_new(parser, cast->keyword_loc.start, cast->keyword_loc.end);
 
+                    // do_keyword_loc
+#line 223 "prism/templates/ext/prism/api_node.c.erb"
+                    argv[5] = cast->do_keyword_loc.start == NULL ? Qnil : pm_location_new(parser, cast->do_keyword_loc.start, cast->do_keyword_loc.end);
+
                     // closing_loc
 #line 223 "prism/templates/ext/prism/api_node.c.erb"
-                    argv[5] = cast->closing_loc.start == NULL ? Qnil : pm_location_new(parser, cast->closing_loc.start, cast->closing_loc.end);
+                    argv[6] = cast->closing_loc.start == NULL ? Qnil : pm_location_new(parser, cast->closing_loc.start, cast->closing_loc.end);
 
                     // predicate
 #line 195 "prism/templates/ext/prism/api_node.c.erb"
-                    argv[6] = rb_ary_pop(value_stack);
+                    argv[7] = rb_ary_pop(value_stack);
 
                     // statements
 #line 195 "prism/templates/ext/prism/api_node.c.erb"
-                    argv[7] = rb_ary_pop(value_stack);
+                    argv[8] = rb_ary_pop(value_stack);
 
-                    rb_ary_push(value_stack, rb_class_new_instance(8, argv, rb_cPrismUntilNode));
+                    rb_ary_push(value_stack, rb_class_new_instance(9, argv, rb_cPrismUntilNode));
                     break;
                 }
 #line 172 "prism/templates/ext/prism/api_node.c.erb"
@@ -6150,7 +6154,7 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
 #line 172 "prism/templates/ext/prism/api_node.c.erb"
                 case PM_WHILE_NODE: {
                     pm_while_node_t *cast = (pm_while_node_t *) node;
-                    VALUE argv[8];
+                    VALUE argv[9];
 
                     // source
                     argv[0] = source;
@@ -6168,19 +6172,23 @@ pm_ast_new(const pm_parser_t *parser, const pm_node_t *node, rb_encoding *encodi
 #line 220 "prism/templates/ext/prism/api_node.c.erb"
                     argv[4] = pm_location_new(parser, cast->keyword_loc.start, cast->keyword_loc.end);
 
+                    // do_keyword_loc
+#line 223 "prism/templates/ext/prism/api_node.c.erb"
+                    argv[5] = cast->do_keyword_loc.start == NULL ? Qnil : pm_location_new(parser, cast->do_keyword_loc.start, cast->do_keyword_loc.end);
+
                     // closing_loc
 #line 223 "prism/templates/ext/prism/api_node.c.erb"
-                    argv[5] = cast->closing_loc.start == NULL ? Qnil : pm_location_new(parser, cast->closing_loc.start, cast->closing_loc.end);
+                    argv[6] = cast->closing_loc.start == NULL ? Qnil : pm_location_new(parser, cast->closing_loc.start, cast->closing_loc.end);
 
                     // predicate
 #line 195 "prism/templates/ext/prism/api_node.c.erb"
-                    argv[6] = rb_ary_pop(value_stack);
+                    argv[7] = rb_ary_pop(value_stack);
 
                     // statements
 #line 195 "prism/templates/ext/prism/api_node.c.erb"
-                    argv[7] = rb_ary_pop(value_stack);
+                    argv[8] = rb_ary_pop(value_stack);
 
-                    rb_ary_push(value_stack, rb_class_new_instance(8, argv, rb_cPrismWhileNode));
+                    rb_ary_push(value_stack, rb_class_new_instance(9, argv, rb_cPrismWhileNode));
                     break;
                 }
 #line 172 "prism/templates/ext/prism/api_node.c.erb"

data/ext/prism/extconf.rb

@@ -70,26 +70,6 @@ if RUBY_ENGINE != "ruby"
   return
 end
 
-# We're going to need to run `make` using prism's `Makefile`.
-# We want to use the same toolchain (compiler, flags, etc) to compile libprism.a and
-# the C extension since they will be linked together.
-# The C extension uses RbConfig, which contains values from the toolchain that built the running Ruby.
-env = RbConfig::CONFIG.slice("SOEXT", "CPPFLAGS", "CFLAGS", "CC", "AR", "ARFLAGS", "MAKEDIRS", "RMALL")
-
-# It's possible that the Ruby that is being run wasn't actually compiled on this
-# machine, in which case parts of RbConfig might be incorrect. In this case
-# we'll need to do some additional checks and potentially fall back to defaults.
-if env.key?("CC") && !File.exist?(env["CC"])
-  env.delete("CC")
-  env.delete("CFLAGS")
-  env.delete("CPPFLAGS")
-end
-
-if env.key?("AR") && !File.exist?(env["AR"])
-  env.delete("AR")
-  env.delete("ARFLAGS")
-end
-
 require "mkmf"
 
 # First, ensure that we can find the header for the prism library.
@@ -127,24 +107,21 @@ end
 # By default, all symbols are hidden in the shared library.
 append_cflags("-fvisibility=hidden")
 
-# We need to link against the libprism.a archive, which is built by the
-# project's `Makefile`. We'll build it if it doesn't exist yet, and then add it
-# to `mkmf`'s list of local libraries.
-archive_target = "build/libprism.a"
-archive_path = File.expand_path("../../#{archive_target}", __dir__)
+def src_list(path)
+  srcdir = path.dup
+  RbConfig.expand(srcdir) # mutates srcdir :-/
+  Dir[File.join(srcdir, "*.{#{SRC_EXT.join(%q{,})}}")]
+end
 
-make(env, archive_target) unless File.exist?(archive_path)
-$LOCAL_LIBS << " #{archive_path}"
+def add_libprism_source(path)
+  $VPATH << path
+  src_list path
+end
+
+$srcs = src_list("$(srcdir)") +
+  add_libprism_source("$(srcdir)/../../src") +
+  add_libprism_source("$(srcdir)/../../src/util")
 
 # Finally, we'll create the `Makefile` that is going to be used to configure and
 # build the C extension.
 create_makefile("prism/prism")
-
-# Now that the `Makefile` for the C extension is built, we'll append on an extra
-# rule that dictates that the extension should be rebuilt if the archive is
-# updated.
-File.open("Makefile", "a") do |mf|
-  mf.puts
-  mf.puts("# Automatically rebuild the extension if libprism.a changed")
-  mf.puts("$(TARGET_SO): $(LOCAL_LIBS)")
-end

data/ext/prism/extension.c

@@ -23,6 +23,7 @@ VALUE rb_cPrismResult;
 VALUE rb_cPrismParseResult;
 VALUE rb_cPrismLexResult;
 VALUE rb_cPrismParseLexResult;
+VALUE rb_cPrismStringQuery;
 
 VALUE rb_cPrismDebugEncoding;
 
@@ -1133,6 +1134,68 @@ parse_file_failure_p(int argc, VALUE *argv, VALUE self) {
     return RTEST(parse_file_success_p(argc, argv, self)) ? Qfalse : Qtrue;
 }
 
+/******************************************************************************/
+/* String query methods                                                       */
+/******************************************************************************/
+
+/**
+ * Process the result of a call to a string query method and return an
+ * appropriate value.
+ */
+static VALUE
+string_query(pm_string_query_t result) {
+    switch (result) {
+        case PM_STRING_QUERY_ERROR:
+            rb_raise(rb_eArgError, "Invalid or non ascii-compatible encoding");
+            return Qfalse;
+        case PM_STRING_QUERY_FALSE:
+            return Qfalse;
+        case PM_STRING_QUERY_TRUE:
+            return Qtrue;
+    }
+    return Qfalse;
+}
+
+/**
+ * call-seq:
+ *   Prism::StringQuery::local?(string) -> bool
+ *
+ * Returns true if the string constitutes a valid local variable name. Note that
+ * this means the names that can be set through Binding#local_variable_set, not
+ * necessarily the ones that can be set through a local variable assignment.
+ */
+static VALUE
+string_query_local_p(VALUE self, VALUE string) {
+    const uint8_t *source = (const uint8_t *) check_string(string);
+    return string_query(pm_string_query_local(source, RSTRING_LEN(string), rb_enc_get(string)->name));
+}
+
+/**
+ * call-seq:
+ *   Prism::StringQuery::constant?(string) -> bool
+ *
+ * Returns true if the string constitutes a valid constant name. Note that this
+ * means the names that can be set through Module#const_set, not necessarily the
+ * ones that can be set through a constant assignment.
+ */
+static VALUE
+string_query_constant_p(VALUE self, VALUE string) {
+    const uint8_t *source = (const uint8_t *) check_string(string);
+    return string_query(pm_string_query_constant(source, RSTRING_LEN(string), rb_enc_get(string)->name));
+}
+
+/**
+ * call-seq:
+ *   Prism::StringQuery::method_name?(string) -> bool
+ *
+ * Returns true if the string constitutes a valid method name.
+ */
+static VALUE
+string_query_method_name_p(VALUE self, VALUE string) {
+    const uint8_t *source = (const uint8_t *) check_string(string);
+    return string_query(pm_string_query_method_name(source, RSTRING_LEN(string), rb_enc_get(string)->name));
+}
+
 /******************************************************************************/
 /* Initialization of the extension                                            */
 /******************************************************************************/
@@ -1170,6 +1233,7 @@ Init_prism(void) {
     rb_cPrismParseResult = rb_define_class_under(rb_cPrism, "ParseResult", rb_cPrismResult);
     rb_cPrismLexResult = rb_define_class_under(rb_cPrism, "LexResult", rb_cPrismResult);
     rb_cPrismParseLexResult = rb_define_class_under(rb_cPrism, "ParseLexResult", rb_cPrismResult);
+    rb_cPrismStringQuery = rb_define_class_under(rb_cPrism, "StringQuery", rb_cObject);
 
     // Intern all of the IDs eagerly that we support so that we don't have to do
     // it every time we parse.
@@ -1211,6 +1275,10 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1);
 #endif
 
+    rb_define_singleton_method(rb_cPrismStringQuery, "local?", string_query_local_p, 1);
+    rb_define_singleton_method(rb_cPrismStringQuery, "constant?", string_query_constant_p, 1);
+    rb_define_singleton_method(rb_cPrismStringQuery, "method_name?", string_query_method_name_p, 1);
+
     // Next, initialize the other APIs.
     Init_prism_api_node();
     Init_prism_pack();

data/ext/prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "1.2.0"
+#define EXPECTED_PRISM_VERSION "1.3.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>